feat: add total input tokens

docs/api/openapi.yml

@@ -10286,6 +10286,9 @@ components:
         reasoning:
           minimum: 0
           type: integer
+        totalInput:
+          minimum: 0
+          type: integer
       required:
       - input
       - output

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -11361,6 +11361,11 @@ export interface SpantypesTraceAITokensDTO {
 	 * @minimum 0
 	 */
 	reasoning: number;
+	/**
+	 * @type integer
+	 * @minimum 0
+	 */
+	totalInput?: number;
 }
 
 export interface SpantypesTraceAISummaryDTO {

pkg/modules/tracedetail/impltracedetail/store.go

@@ -138,6 +138,7 @@ func (s *traceStore) GetTraceStats(ctx context.Context, orgID valuer.UUID, trace
 		"toUInt64(coalesce(sum(cache_read_tokens_value), 0)) AS cache_read_tokens",
 		"toUInt64(coalesce(sum(cache_write_tokens_value), 0)) AS cache_write_tokens",
 		"toUInt64(coalesce(sum(reasoning_tokens_value), 0)) AS reasoning_tokens",
+		"if(countIf(total_input_tokens_value IS NULL AND coalesce(input_tokens_value, 0) + coalesce(cache_read_tokens_value, 0) + coalesce(cache_write_tokens_value, 0) > 0) = 0, toUInt64(sum(total_input_tokens_value)), NULL) AS total_input_tokens",
 		"sum(total_cost_value) AS total_cost",
 	)
 	sb.From(sb.BuilderAs(spans, "spans"))
@@ -148,7 +149,7 @@ func (s *traceStore) GetTraceStats(ctx context.Context, orgID valuer.UUID, trace
 		&stats.StartNs, &stats.EndNs, &stats.TotalSpans, &stats.TotalErrorSpans, &stats.HasMissingSpans,
 		&stats.RootServiceName, &stats.RootEntryPoint, &stats.RootSpanStatusCode, &stats.GenAISpanCount,
 		&stats.Tokens.Input, &stats.Tokens.Output, &stats.Tokens.CacheRead, &stats.Tokens.CacheWrite, &stats.Tokens.Reasoning,
-		&stats.TotalCost,
+		&stats.Tokens.TotalInput, &stats.TotalCost,
 	)
 	if err != nil {
 		return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying trace stats")
@@ -168,6 +169,7 @@ func (s *traceStore) genAISpanColumns(ctx context.Context, orgID valuer.UUID, bo
 		{aiobservabilitytypes.GenAIUsageCacheReadInputTokens, "cache_read_tokens_value"},
 		{aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, "cache_write_tokens_value"},
 		{aiobservabilitytypes.GenAIUsageReasoningOutputTokens, "reasoning_tokens_value"},
+		{aiobservabilitytypes.SignozGenAITotalInputTokens, "total_input_tokens_value"},
 		{aiobservabilitytypes.SignozGenAITotalCost, "total_cost_value"},
 	}
 

pkg/telemetryschema/aitelemetryschema/const.go

@@ -21,6 +21,7 @@ var (
 		aiobservabilitytypes.GenAIUsageCacheCreationInputTokens: genAIAttribute(aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, telemetrytypes.FieldDataTypeNumber),
 		aiobservabilitytypes.GenAIUsageReasoningOutputTokens:    genAIAttribute(aiobservabilitytypes.GenAIUsageReasoningOutputTokens, telemetrytypes.FieldDataTypeNumber),
 		aiobservabilitytypes.SignozGenAITotalCost:               genAIAttribute(aiobservabilitytypes.SignozGenAITotalCost, telemetrytypes.FieldDataTypeNumber),
+		aiobservabilitytypes.SignozGenAITotalInputTokens:        genAIAttribute(aiobservabilitytypes.SignozGenAITotalInputTokens, telemetrytypes.FieldDataTypeNumber),
 
 		aiobservabilitytypes.GenAIInputMessages:  genAIAttribute(aiobservabilitytypes.GenAIInputMessages, telemetrytypes.FieldDataTypeString),
 		aiobservabilitytypes.GenAIOutputMessages: genAIAttribute(aiobservabilitytypes.GenAIOutputMessages, telemetrytypes.FieldDataTypeString),

pkg/types/aiobservabilitytypes/keys.go

@@ -21,14 +21,15 @@ const (
 	GenAIOutputMessages = "gen_ai.output.messages"
 )
 
-// Per-span costs the SigNoz LLM pricing processor attaches; SigNoz semconv
-// (signoz-semantic-conventions model/genai/registry.yaml), not OTel semconv.
+// Per-span costs and token totals the SigNoz LLM pricing processor attaches;
+// SigNoz semconv (signoz-semantic-conventions model/genai/registry.yaml), not OTel semconv.
 const (
-	SignozGenAICostInput      = "signoz.gen_ai.usage.input_tokens.cost"
-	SignozGenAICostOutput     = "signoz.gen_ai.usage.output_tokens.cost"
-	SignozGenAICostCacheRead  = "signoz.gen_ai.usage.cache_read.input_tokens.cost"
-	SignozGenAICostCacheWrite = "signoz.gen_ai.usage.cache_write.input_tokens.cost"
-	SignozGenAITotalCost      = "signoz.gen_ai.usage.tokens.cost"
+	SignozGenAICostInput        = "signoz.gen_ai.usage.input_tokens.cost"
+	SignozGenAICostOutput       = "signoz.gen_ai.usage.output_tokens.cost"
+	SignozGenAICostCacheRead    = "signoz.gen_ai.usage.cache_read.input_tokens.cost"
+	SignozGenAICostCacheWrite   = "signoz.gen_ai.usage.cache_write.input_tokens.cost"
+	SignozGenAITotalCost        = "signoz.gen_ai.usage.tokens.cost"
+	SignozGenAITotalInputTokens = "signoz.gen_ai.usage.total_input_tokens"
 )
 
 // GenAISpanGateKeys mark a span as gen_ai: an LLM call, a tool call, or an

pkg/types/llmpricingruletypes/processorconfig.go

@@ -48,13 +48,14 @@ type LLMPricingRuleProcessorCache struct {
 	Write float64 `yaml:"write" json:"write"`
 }
 
-// LLMPricingRuleProcessorOutputAttrs maps the processor's computed cost fields to span attribute names.
+// LLMPricingRuleProcessorOutputAttrs maps the processor's computed cost and token fields to span attribute names.
 type LLMPricingRuleProcessorOutputAttrs struct {
-	In         string `yaml:"in" json:"in"`
-	Out        string `yaml:"out" json:"out"`
-	CacheRead  string `yaml:"cache_read" json:"cache_read"`
-	CacheWrite string `yaml:"cache_write" json:"cache_write"`
-	Total      string `yaml:"total" json:"total"`
+	In               string `yaml:"in" json:"in"`
+	Out              string `yaml:"out" json:"out"`
+	CacheRead        string `yaml:"cache_read" json:"cache_read"`
+	CacheWrite       string `yaml:"cache_write" json:"cache_write"`
+	Total            string `yaml:"total" json:"total"`
+	TotalInputTokens string `yaml:"total_input_tokens" json:"total_input_tokens"`
 }
 
 // buildProcessorConfig converts pricing rules into the signozllmpricing processor config.
@@ -103,6 +104,8 @@ func buildProcessorConfig(rules []*LLMPricingRule) *LLMPricingRuleProcessorConfi
 			CacheRead:  aiobservabilitytypes.SignozGenAICostCacheRead,
 			CacheWrite: aiobservabilitytypes.SignozGenAICostCacheWrite,
 			Total:      aiobservabilitytypes.SignozGenAITotalCost,
+
+			TotalInputTokens: aiobservabilitytypes.SignozGenAITotalInputTokens,
 		},
 	}
 }

pkg/types/llmpricingruletypes/testdata/collector_no_rules.yaml

@@ -18,6 +18,7 @@ processors:
             cache_read: signoz.gen_ai.usage.cache_read.input_tokens.cost
             cache_write: signoz.gen_ai.usage.cache_write.input_tokens.cost
             total: signoz.gen_ai.usage.tokens.cost
+            total_input_tokens: signoz.gen_ai.usage.total_input_tokens
 receivers:
     otlp:
         protocols:

pkg/types/llmpricingruletypes/testdata/collector_rule_cache_mode_unknown.yaml

@@ -26,6 +26,7 @@ processors:
             cache_read: signoz.gen_ai.usage.cache_read.input_tokens.cost
             cache_write: signoz.gen_ai.usage.cache_write.input_tokens.cost
             total: signoz.gen_ai.usage.tokens.cost
+            total_input_tokens: signoz.gen_ai.usage.total_input_tokens
 receivers:
     otlp:
         protocols:

pkg/types/llmpricingruletypes/testdata/collector_rule_without_cache.yaml

@@ -23,6 +23,7 @@ processors:
             cache_read: signoz.gen_ai.usage.cache_read.input_tokens.cost
             cache_write: signoz.gen_ai.usage.cache_write.input_tokens.cost
             total: signoz.gen_ai.usage.tokens.cost
+            total_input_tokens: signoz.gen_ai.usage.total_input_tokens
 receivers:
     otlp:
         protocols:

pkg/types/llmpricingruletypes/testdata/collector_with_rule.yaml

@@ -27,6 +27,7 @@ processors:
             cache_read: signoz.gen_ai.usage.cache_read.input_tokens.cost
             cache_write: signoz.gen_ai.usage.cache_write.input_tokens.cost
             total: signoz.gen_ai.usage.tokens.cost
+            total_input_tokens: signoz.gen_ai.usage.total_input_tokens
 receivers:
     otlp:
         protocols:

pkg/types/spantypes/trace_summary.go

@@ -42,6 +42,9 @@ type TraceAITokens struct {
 	CacheRead  uint64 `json:"cacheRead" required:"true"`
 	CacheWrite uint64 `json:"cacheWrite" required:"true"`
 	Reasoning  uint64 `json:"reasoning" required:"true"`
+	// TotalInput is nil, not 0, when any span with input tokens lacks the key,
+	// since a partial sum misleads and 0 is a valid total.
+	TotalInput *uint64 `json:"totalInput,omitempty" nullable:"false"`
 }
 
 func NewGettableTraceSummary(stats *TraceStats) *GettableTraceSummary {

tests/integration/tests/tracedetail/01_summary.py

@@ -32,6 +32,7 @@ def test_summary_ai_trace(
 ) -> None:
     """The summary carries the waterfall's trace-level fields and, for a trace with gen_ai
     spans, token totals over every LLM span and the cost summed over the spans that carry it.
+    The total input count is dropped because one LLM span was never priced.
     Spans are written to one layout only, so a read from the wrong column sums to zero."""
     use_attribute_backend(attribute_backend)
     write_mode = "json_only" if attribute_backend == "json" else "legacy_only"
@@ -59,6 +60,7 @@ def test_summary_ai_trace(
                     "gen_ai.usage.input_tokens": 100,
                     "gen_ai.usage.output_tokens": 20,
                     "gen_ai.usage.cache_read.input_tokens": 7,
+                    "signoz.gen_ai.usage.total_input_tokens": 100,
                     "signoz.gen_ai.usage.tokens.cost": 0.01,
                 },
                 attribute_write_mode=write_mode,
@@ -168,7 +170,13 @@ def test_summary_ai_trace_across_json_rollout(
                 kind=TracesKind.SPAN_KIND_CLIENT,
                 status_code=TracesStatusCode.STATUS_CODE_OK,
                 resources=resources,
-                attributes={"gen_ai.request.model": "gpt-4o-mini", "gen_ai.usage.input_tokens": 100, "gen_ai.usage.output_tokens": 20, "signoz.gen_ai.usage.tokens.cost": 0.01},
+                attributes={
+                    "gen_ai.request.model": "gpt-4o-mini",
+                    "gen_ai.usage.input_tokens": 100,
+                    "gen_ai.usage.output_tokens": 20,
+                    "signoz.gen_ai.usage.total_input_tokens": 100,
+                    "signoz.gen_ai.usage.tokens.cost": 0.01,
+                },
                 attribute_write_mode="legacy_only",
             ),
             Traces(
@@ -181,7 +189,13 @@ def test_summary_ai_trace_across_json_rollout(
                 kind=TracesKind.SPAN_KIND_CLIENT,
                 status_code=TracesStatusCode.STATUS_CODE_OK,
                 resources=resources,
-                attributes={"gen_ai.request.model": "gpt-4o-mini", "gen_ai.usage.input_tokens": 50, "gen_ai.usage.output_tokens": 5, "signoz.gen_ai.usage.tokens.cost": 0.02},
+                attributes={
+                    "gen_ai.request.model": "gpt-4o-mini",
+                    "gen_ai.usage.input_tokens": 50,
+                    "gen_ai.usage.output_tokens": 5,
+                    "signoz.gen_ai.usage.total_input_tokens": 50,
+                    "signoz.gen_ai.usage.tokens.cost": 0.02,
+                },
                 attribute_write_mode="json_only",
             ),
         ]
@@ -199,7 +213,7 @@ def test_summary_ai_trace_across_json_rollout(
     assert summary["totalSpansCount"] == 3
     assert summary["rootServiceEntryPoint"] == "long agent run"
     assert summary["rootSpanStatusCode"] == "200"
-    assert summary["ai"]["tokens"] == {"input": 150, "output": 25, "cacheRead": 0, "cacheWrite": 0, "reasoning": 0}
+    assert summary["ai"]["tokens"] == {"input": 150, "output": 25, "cacheRead": 0, "cacheWrite": 0, "reasoning": 0, "totalInput": 150}
     assert summary["ai"]["totalCost"] == pytest.approx(0.03)