fix: update implementation

docs/api/openapi.yml

@@ -11006,6 +11006,100 @@ paths:
       tags:
       - logs
       - traces
+  /api/v1/fields/ai_observability/keys:
+    get:
+      deprecated: false
+      description: This endpoint returns the field keys the AI observability explorer
+        can filter on, including the computed per-trace aggregates
+      operationId: GetAIObservabilityFieldsKeys
+      parameters:
+      - in: query
+        name: signal
+        schema:
+          $ref: '#/components/schemas/TelemetrytypesSignal'
+      - in: query
+        name: source
+        schema:
+          $ref: '#/components/schemas/TelemetrytypesSource'
+      - in: query
+        name: type
+        schema:
+          type: string
+      - in: query
+        name: limit
+        schema:
+          type: integer
+      - in: query
+        name: startUnixMilli
+        schema:
+          format: int64
+          type: integer
+      - in: query
+        name: endUnixMilli
+        schema:
+          format: int64
+          type: integer
+      - in: query
+        name: fieldContext
+        schema:
+          $ref: '#/components/schemas/TelemetrytypesFieldContext'
+      - in: query
+        name: fieldDataType
+        schema:
+          $ref: '#/components/schemas/TelemetrytypesFieldDataType'
+      - in: query
+        name: metricName
+        schema:
+          type: string
+      - in: query
+        name: metricNamespace
+        schema:
+          type: string
+      - in: query
+        name: searchText
+        schema:
+          type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/TelemetrytypesGettableFieldKeys'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - VIEWER
+      - tokenizer:
+        - VIEWER
+      summary: Get AI observability field keys
+      tags:
+      - fields
   /api/v1/fields/keys:
     get:
       deprecated: false
@@ -11020,6 +11114,10 @@ paths:
         name: source
         schema:
           $ref: '#/components/schemas/TelemetrytypesSource'
+      - in: query
+        name: type
+        schema:
+          type: string
       - in: query
         name: limit
         schema:
@@ -11109,6 +11207,10 @@ paths:
         name: source
         schema:
           $ref: '#/components/schemas/TelemetrytypesSource'
+      - in: query
+        name: type
+        schema:
+          type: string
       - in: query
         name: limit
         schema:
@@ -21311,6 +21413,10 @@ paths:
         name: source
         schema:
           $ref: '#/components/schemas/TelemetrytypesSource'
+      - in: query
+        name: type
+        schema:
+          type: string
       - in: query
         name: limit
         schema:
@@ -21412,6 +21518,10 @@ paths:
         name: source
         schema:
           $ref: '#/components/schemas/TelemetrytypesSource'
+      - in: query
+        name: type
+        schema:
+          type: string
       - in: query
         name: limit
         schema:

frontend/src/api/generated/services/fields/index.ts

@@ -15,6 +15,8 @@ import type {
 } from 'react-query';
 
 import type {
+	GetAIObservabilityFieldsKeys200,
+	GetAIObservabilityFieldsKeysParams,
 	GetFieldsKeys200,
 	GetFieldsKeysParams,
 	GetFieldsValues200,
@@ -25,6 +27,111 @@ import type {
 import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
 import type { ErrorType } from '../../../generatedAPIInstance';
 
+/**
+ * This endpoint returns the field keys the AI observability explorer can filter on, including the computed per-trace aggregates
+ * @summary Get AI observability field keys
+ */
+export const getAIObservabilityFieldsKeys = (
+	params?: GetAIObservabilityFieldsKeysParams,
+	signal?: AbortSignal,
+) => {
+	return GeneratedAPIInstance<GetAIObservabilityFieldsKeys200>({
+		url: `/api/v1/fields/ai_observability/keys`,
+		method: 'GET',
+		params,
+		signal,
+	});
+};
+
+export const getGetAIObservabilityFieldsKeysQueryKey = (
+	params?: GetAIObservabilityFieldsKeysParams,
+) => {
+	return [
+		`/api/v1/fields/ai_observability/keys`,
+		...(params ? [params] : []),
+	] as const;
+};
+
+export const getGetAIObservabilityFieldsKeysQueryOptions = <
+	TData = Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+	TError = ErrorType<RenderErrorResponseDTO>,
+>(
+	params?: GetAIObservabilityFieldsKeysParams,
+	options?: {
+		query?: UseQueryOptions<
+			Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+			TError,
+			TData
+		>;
+	},
+) => {
+	const { query: queryOptions } = options ?? {};
+
+	const queryKey =
+		queryOptions?.queryKey ?? getGetAIObservabilityFieldsKeysQueryKey(params);
+
+	const queryFn: QueryFunction<
+		Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>
+	> = ({ signal }) => getAIObservabilityFieldsKeys(params, signal);
+
+	return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
+		Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+		TError,
+		TData
+	> & { queryKey: QueryKey };
+};
+
+export type GetAIObservabilityFieldsKeysQueryResult = NonNullable<
+	Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>
+>;
+export type GetAIObservabilityFieldsKeysQueryError =
+	ErrorType<RenderErrorResponseDTO>;
+
+/**
+ * @summary Get AI observability field keys
+ */
+
+export function useGetAIObservabilityFieldsKeys<
+	TData = Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+	TError = ErrorType<RenderErrorResponseDTO>,
+>(
+	params?: GetAIObservabilityFieldsKeysParams,
+	options?: {
+		query?: UseQueryOptions<
+			Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+			TError,
+			TData
+		>;
+	},
+): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
+	const queryOptions = getGetAIObservabilityFieldsKeysQueryOptions(
+		params,
+		options,
+	);
+
+	const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
+		queryKey: QueryKey;
+	};
+
+	return { ...query, queryKey: queryOptions.queryKey };
+}
+
+/**
+ * @summary Get AI observability field keys
+ */
+export const invalidateGetAIObservabilityFieldsKeys = async (
+	queryClient: QueryClient,
+	params?: GetAIObservabilityFieldsKeysParams,
+	options?: InvalidateOptions,
+): Promise<QueryClient> => {
+	await queryClient.invalidateQueries(
+		{ queryKey: getGetAIObservabilityFieldsKeysQueryKey(params) },
+		options,
+	);
+
+	return queryClient;
+};
+
 /**
  * This endpoint returns field keys
  * @summary Get field keys

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -10417,6 +10417,70 @@ export enum HandleExportRawDataPOSTFormat {
 	csv = 'csv',
 	jsonl = 'jsonl',
 }
+export type GetAIObservabilityFieldsKeysParams = {
+	/**
+	 * @description undefined
+	 */
+	signal?: TelemetrytypesSignalDTO;
+	/**
+	 * @description undefined
+	 */
+	source?: TelemetrytypesSourceDTO;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	type?: string;
+	/**
+	 * @type integer
+	 * @description undefined
+	 */
+	limit?: number;
+	/**
+	 * @type integer
+	 * @format int64
+	 * @description undefined
+	 */
+	startUnixMilli?: number;
+	/**
+	 * @type integer
+	 * @format int64
+	 * @description undefined
+	 */
+	endUnixMilli?: number;
+	/**
+	 * @description undefined
+	 */
+	fieldContext?: TelemetrytypesFieldContextDTO;
+	/**
+	 * @description undefined
+	 */
+	fieldDataType?: TelemetrytypesFieldDataTypeDTO;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	metricName?: string;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	metricNamespace?: string;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	searchText?: string;
+};
+
+export type GetAIObservabilityFieldsKeys200 = {
+	data: TelemetrytypesGettableFieldKeysDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
 export type GetFieldsKeysParams = {
 	/**
 	 * @description undefined
@@ -10426,6 +10490,11 @@ export type GetFieldsKeysParams = {
 	 * @description undefined
 	 */
 	source?: TelemetrytypesSourceDTO;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	type?: string;
 	/**
 	 * @type integer
 	 * @description undefined
@@ -10485,6 +10554,11 @@ export type GetFieldsValuesParams = {
 	 * @description undefined
 	 */
 	source?: TelemetrytypesSourceDTO;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	type?: string;
 	/**
 	 * @type integer
 	 * @description undefined
@@ -11792,6 +11866,11 @@ export type GetRuleHistoryFilterKeysParams = {
 	 * @description undefined
 	 */
 	source?: TelemetrytypesSourceDTO;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	type?: string;
 	/**
 	 * @type integer
 	 * @description undefined
@@ -11854,6 +11933,11 @@ export type GetRuleHistoryFilterValuesParams = {
 	 * @description undefined
 	 */
 	source?: TelemetrytypesSourceDTO;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	type?: string;
 	/**
 	 * @type integer
 	 * @description undefined

pkg/apiserver/signozapiserver/fields.go

@@ -28,6 +28,24 @@ func (provider *provider) addFieldsRoutes(router *mux.Router) error {
 		return err
 	}
 
+	if err := router.Handle("/api/v1/fields/ai_observability/keys", handler.New(provider.authzMiddleware.ViewAccess(provider.fieldsHandler.GetAIObservabilityFieldsKeys), handler.OpenAPIDef{
+		ID:                  "GetAIObservabilityFieldsKeys",
+		Tags:                []string{"fields"},
+		Summary:             "Get AI observability field keys",
+		Description:         "This endpoint returns the field keys the AI observability explorer can filter on, including the computed per-trace aggregates",
+		Request:             nil,
+		RequestQuery:        new(telemetrytypes.PostableFieldKeysParams),
+		RequestContentType:  "",
+		Response:            new(telemetrytypes.GettableFieldKeys),
+		ResponseContentType: "application/json",
+		SuccessStatusCode:   http.StatusOK,
+		ErrorStatusCodes:    []int{},
+		Deprecated:          false,
+		SecuritySchemes:     newSecuritySchemes(types.RoleViewer),
+	})).Methods(http.MethodGet).GetError(); err != nil {
+		return err
+	}
+
 	if err := router.Handle("/api/v1/fields/values", handler.New(provider.authzMiddleware.ViewAccess(provider.fieldsHandler.GetFieldsValues), handler.OpenAPIDef{
 		ID:                  "GetFieldsValues",
 		Tags:                []string{"fields"},

pkg/modules/fields/handler.go

@@ -8,4 +8,7 @@ type Handler interface {
 
 	// Gets the fields values for the given field value selector
 	GetFieldsValues(http.ResponseWriter, *http.Request)
+
+	// Gets the fields keys the AI observability explorer can filter on
+	GetAIObservabilityFieldsKeys(http.ResponseWriter, *http.Request)
 }

pkg/modules/fields/implfields/handler.go

@@ -7,6 +7,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/http/binding"
 	"github.com/SigNoz/signoz/pkg/http/render"
 	"github.com/SigNoz/signoz/pkg/modules/fields"
+	"github.com/SigNoz/signoz/pkg/telemetryschema/aitelemetryschema"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/SigNoz/signoz/pkg/valuer"
@@ -51,6 +52,43 @@ func (handler *handler) GetFieldsKeys(rw http.ResponseWriter, req *http.Request)
 	})
 }
 
+func (handler *handler) GetAIObservabilityFieldsKeys(rw http.ResponseWriter, req *http.Request) {
+	ctx := req.Context()
+
+	var params telemetrytypes.PostableFieldKeysParams
+	if err := binding.Query.BindQuery(req.URL.Query(), &params); err != nil {
+		render.Error(rw, err)
+		return
+	}
+	// the explorer lists AI traces, whatever signal the caller asked for
+	params.Signal = telemetrytypes.SignalTraces
+
+	claims, err := authtypes.ClaimsFromContext(ctx)
+	if err != nil {
+		render.Error(rw, err)
+		return
+	}
+	orgID := valuer.MustNewUUID(claims.OrgID)
+
+	fieldKeySelector := telemetrytypes.NewFieldKeySelectorFromPostableFieldKeysParams(params)
+
+	keys := make(map[string][]*telemetrytypes.TelemetryFieldKey)
+	complete := true
+	// the trace context names the computed per-trace aggregates, which no scan can serve
+	if fieldKeySelector.FieldContext != telemetrytypes.FieldContextTrace {
+		keys, complete, err = handler.telemetryMetadataStore.GetKeys(ctx, orgID, fieldKeySelector)
+		if err != nil {
+			render.Error(rw, err)
+			return
+		}
+	}
+
+	render.Success(rw, http.StatusOK, &telemetrytypes.GettableFieldKeys{
+		Keys:     aitelemetryschema.FieldKeys(keys, fieldKeySelector),
+		Complete: complete,
+	})
+}
+
 func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Request) {
 	ctx := req.Context()
 

pkg/modules/llmpricingrule/impllmpricingrule/module.go

@@ -11,6 +11,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/modules/llmpricingrule"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
+	"github.com/SigNoz/signoz/pkg/telemetryschema/aitelemetryschema"
 	"github.com/SigNoz/signoz/pkg/types/featuretypes"
 	"github.com/SigNoz/signoz/pkg/types/llmpricingruletypes"
 	"github.com/SigNoz/signoz/pkg/types/opamptypes"
@@ -213,18 +214,18 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
 					Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
 						Name:   "A",
 						Signal: telemetrytypes.SignalTraces,
-						Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", telemetrytypes.GenAIRequestModel)},
+						Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", aitelemetryschema.GenAIRequestModel)},
 						Aggregations: []qbtypes.TraceAggregation{
 							{Expression: "count()", Alias: "spanCount"},
 						},
 						GroupBy: []qbtypes.GroupByKey{
 							{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-								Name:          telemetrytypes.GenAIRequestModel,
+								Name:          aitelemetryschema.GenAIRequestModel,
 								FieldContext:  telemetrytypes.FieldContextSpan,
 								FieldDataType: telemetrytypes.FieldDataTypeString,
 							}},
 							{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-								Name:          telemetrytypes.GenAIProviderName,
+								Name:          aitelemetryschema.GenAIProviderName,
 								FieldContext:  telemetrytypes.FieldContextSpan,
 								FieldDataType: telemetrytypes.FieldDataTypeString,
 							}},
@@ -254,9 +255,9 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
 		switch c.Type {
 		case qbtypes.ColumnTypeGroup:
 			switch c.Name {
-			case telemetrytypes.GenAIRequestModel:
+			case aitelemetryschema.GenAIRequestModel:
 				modelIdx = i
-			case telemetrytypes.GenAIProviderName:
+			case aitelemetryschema.GenAIProviderName:
 				providerIdx = i
 			}
 		case qbtypes.ColumnTypeAggregation:

pkg/statementbuilder/aistatementbuilder/statement_builder.go

@@ -7,6 +7,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/statementbuilder"
 	scopedtraces "github.com/SigNoz/signoz/pkg/statementbuilder/scopedtracesstatementbuilder"
+	"github.com/SigNoz/signoz/pkg/telemetryschema/aitelemetryschema"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -25,7 +26,7 @@ func NewFactory(
 // Scope describes gen_ai for the scoped trace builder: an AI trace has >=1 gen_ai
 // LLM, tool, or agent span, and its list adds AI/LLM per-trace metrics.
 func Scope() scopedtraces.TraceScope {
-	gateKeyNames := []string{telemetrytypes.GenAIRequestModel, telemetrytypes.GenAIToolName, telemetrytypes.GenAIAgentName}
+	gateKeyNames := []string{aitelemetryschema.GenAIRequestModel, aitelemetryschema.GenAIToolName, aitelemetryschema.GenAIAgentName}
 	gateExprs := make([]string, 0, len(gateKeyNames))
 	gateKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(gateKeyNames))
 	for _, name := range gateKeyNames {
@@ -37,32 +38,33 @@ func Scope() scopedtraces.TraceScope {
 		})
 	}
 
-	defs := telemetrytypes.GenAIFieldDefinitions
-	reqModel := defs[telemetrytypes.GenAIRequestModel]
-	toolName := defs[telemetrytypes.GenAIToolName]
-	inTok := defs[telemetrytypes.GenAIUsageInputTokens]
-	outTok := defs[telemetrytypes.GenAIUsageOutputTokens]
-	cost := defs[telemetrytypes.SignozGenAITotalCost]
-	inMsg := defs[telemetrytypes.GenAIInputMessages]
-	outMsg := defs[telemetrytypes.GenAIOutputMessages]
+	defs := aitelemetryschema.GenAIFields
+	reqModel := defs[aitelemetryschema.GenAIRequestModel]
+	toolName := defs[aitelemetryschema.GenAIToolName]
+	inTok := defs[aitelemetryschema.GenAIUsageInputTokens]
+	outTok := defs[aitelemetryschema.GenAIUsageOutputTokens]
+	cost := defs[aitelemetryschema.SignozGenAITotalCost]
+	inMsg := defs[aitelemetryschema.GenAIInputMessages]
+	outMsg := defs[aitelemetryschema.GenAIOutputMessages]
 
 	str := telemetrytypes.FieldDataTypeString
 	columns := append(scopedtraces.CommonTraceColumns(),
 		// LLM calls only (request model present), not the full gate.
-		scopedtraces.TraceColumn{Alias: "llm_call_count", Orderable: true, Expr: scopedtraces.CountExists(&reqModel)},
-		scopedtraces.TraceColumn{Alias: "tool_call_count", Orderable: true, Expr: scopedtraces.CountExists(&toolName)},
-		scopedtraces.TraceColumn{Alias: "distinct_tool_count", Orderable: true, Expr: scopedtraces.UniqCount(&toolName, str)},
+		scopedtraces.TraceColumn{Alias: "llm_call_count", Orderable: true, Filterable: true, Expr: scopedtraces.CountExists(&reqModel)},
+		scopedtraces.TraceColumn{Alias: "tool_call_count", Orderable: true, Filterable: true, Expr: scopedtraces.CountExists(&toolName)},
+		scopedtraces.TraceColumn{Alias: "distinct_tool_count", Orderable: true, Filterable: true, Expr: scopedtraces.UniqCount(&toolName, str)},
 		// tokens live only on LLM spans, so a plain sum needs no gate scoping.
-		scopedtraces.TraceColumn{Alias: "input_tokens", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &inTok)},
-		scopedtraces.TraceColumn{Alias: "output_tokens", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &outTok)},
-		scopedtraces.TraceColumn{Alias: "total_tokens", Orderable: true, Expr: scopedtraces.SumOfKeys(telemetrytypes.FieldDataTypeFloat64, &inTok, &outTok)},
+		scopedtraces.TraceColumn{Alias: "input_tokens", Orderable: true, Filterable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &inTok)},
+		scopedtraces.TraceColumn{Alias: "output_tokens", Orderable: true, Filterable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &outTok)},
+		scopedtraces.TraceColumn{Alias: "total_tokens", Orderable: true, Filterable: true, Expr: scopedtraces.SumOfKeys(telemetrytypes.FieldDataTypeFloat64, &inTok, &outTok)},
 		// per-span cost attached by the SigNoz LLM pricing processor.
-		scopedtraces.TraceColumn{Alias: "estimated_total_cost", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &cost)},
+		scopedtraces.TraceColumn{Alias: "estimated_total_cost", Orderable: true, Filterable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &cost)},
 		// slowest single LLM call in the trace.
-		scopedtraces.TraceColumn{Alias: "max_llm_duration_nano", Orderable: true, Expr: scopedtraces.ScopedToKeyColumn(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("duration_nano"), &reqModel)},
+		scopedtraces.TraceColumn{Alias: "max_llm_duration_nano", Orderable: true, Filterable: true, Expr: scopedtraces.ScopedToKeyColumn(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("duration_nano"), &reqModel)},
 		// errors across the whole trace (any span), so display-only.
 		scopedtraces.TraceColumn{Alias: "error_count", Expr: scopedtraces.CondCount(scopedtraces.IntrinsicSpanKey("has_error"), qbtypes.FilterOperatorEqual, true)},
-		// timestamp of the last gen_ai span (LLM/tool/agent), hence gate-scoped.
+		// timestamp of the last gen_ai span (LLM/tool/agent), hence gate-scoped;
+		// order-only: a raw-nanos threshold makes no sense in the filter bar.
 		scopedtraces.TraceColumn{Alias: "last_activity_time", Orderable: true, Expr: scopedtraces.ScopedReduce(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("timestamp"))},
 		// previews: first call's input (the prompt), last call's output (the answer).
 		scopedtraces.TraceColumn{Alias: "input", SpanLevel: true, Expr: scopedtraces.PickBy(&inMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickEarliest)},

pkg/statementbuilder/aistatementbuilder/statement_builder_test.go

@@ -3,6 +3,8 @@ package aistatementbuilder
 import (
 	"context"
 	"fmt"
+	"maps"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -11,6 +13,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/statementbuilder"
 	scopedtraces "github.com/SigNoz/signoz/pkg/statementbuilder/scopedtracesstatementbuilder"
+	"github.com/SigNoz/signoz/pkg/telemetryschema/aitelemetryschema"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
@@ -40,8 +43,7 @@ func otelKeysMap() map[string][]*telemetrytypes.TelemetryFieldKey {
 
 	m := make(map[string][]*telemetrytypes.TelemetryFieldKey)
 
-	// mirrors what enrichWithGenAIKeys surfaces in production
-	for name, def := range telemetrytypes.GenAIFieldDefinitions {
+	for name, def := range aitelemetryschema.GenAIFields {
 		keyCopy := def
 		m[name] = []*telemetrytypes.TelemetryFieldKey{&keyCopy}
 	}
@@ -976,8 +978,8 @@ func TestBuild_UnsupportedRequestType(t *testing.T) {
 // mask, OR-combined.
 func TestBuild_TraceList_MultiVariantGateKey(t *testing.T) {
 	keys := otelKeysMap()
-	keys[telemetrytypes.GenAIToolName] = append(keys[telemetrytypes.GenAIToolName], &telemetrytypes.TelemetryFieldKey{
-		Name:          telemetrytypes.GenAIToolName,
+	keys[aitelemetryschema.GenAIToolName] = append(keys[aitelemetryschema.GenAIToolName], &telemetrytypes.TelemetryFieldKey{
+		Name:          aitelemetryschema.GenAIToolName,
 		Signal:        telemetrytypes.SignalTraces,
 		FieldContext:  telemetrytypes.FieldContextAttribute,
 		FieldDataType: telemetrytypes.FieldDataTypeFloat64,
@@ -1056,3 +1058,17 @@ func TestBuild_TraceList_VariableInAggregateFilter(t *testing.T) {
 	_, err = build("trace.output_tokens > $missing", map[string]qbtypes.VariableItem{"other": {Value: 1}})
 	require.Error(t, err)
 }
+
+// the schema declares the aggregates the API suggests, this Scope declares the SQL
+// that computes them; a filterable column with no definition is unreachable, and a
+// definition with no filterable column is a dead suggestion
+func TestScope_FilterableColumnsMatchSchemaAggregates(t *testing.T) {
+	var filterable []string
+	for _, c := range Scope().Columns {
+		if c.Orderable && c.Filterable {
+			filterable = append(filterable, c.Alias)
+		}
+	}
+
+	assert.ElementsMatch(t, slices.Collect(maps.Keys(aitelemetryschema.TraceAggregateFields)), filterable)
+}

pkg/statementbuilder/scopedtracesstatementbuilder/scope.go

@@ -24,9 +24,10 @@ type TraceColumn struct {
 	// ClickHouse resolves bare identifiers to same-SELECT aliases first, so any
 	// expression referencing that column would silently bind to the alias.
 	Alias string
-	// Orderable columns can be used in ORDER BY and the aggregate filter; all-span
-	// aggregates are display-only and set false.
-	Orderable bool
+	// Orderable columns can be used in ORDER BY, Filterable ones in the aggregate
+	// filter; all-span aggregates are display-only and set neither.
+	Orderable  bool
+	Filterable bool
 	// SpanLevel columns surface a real span/resource attribute; a filter on them is
 	// applied span-level, so they are excluded from the trace-level aliases.
 	SpanLevel bool

pkg/statementbuilder/scopedtracesstatementbuilder/statement_builder.go

@@ -185,18 +185,19 @@ func (b *scopedTraceStatementBuilder) buildTraceListQuery(
 		return nil, err
 	}
 	orderableSet := orderableAliasSet(resolved)
+	filterableSet := filterableAliasSet(resolved)
 
 	resourceFrag, resourceArgs, resourcePred, err := b.maybeAttachResourceFilter(ctx, orgID, query, start, end, variables)
 	if err != nil {
 		return nil, err
 	}
 
-	fp, err := b.splitFilter(ctx, orgID, query, b.aggregateAliasSet(), orderableSet, start, end, variables, matchedSB)
+	fp, err := b.splitFilter(ctx, orgID, query, b.aggregateAliasSet(), filterableSet, start, end, variables, matchedSB)
 	if err != nil {
 		return nil, err
 	}
 
-	matchedFrag, matchedArgs, err := b.buildMatchedCTE(matchedSB, start, end, startBucket, endBucket, resolved, orders, orderableSet, maskExpr, fp, resourcePred, limit, query.Offset)
+	matchedFrag, matchedArgs, err := b.buildMatchedCTE(matchedSB, start, end, startBucket, endBucket, resolved, orders, orderableSet, filterableSet, maskExpr, fp, resourcePred, limit, query.Offset)
 	if err != nil {
 		return nil, err
 	}
@@ -326,9 +327,10 @@ func (b *scopedTraceStatementBuilder) resolveMask(ctx context.Context, orgID val
 }
 
 type resolvedColumn struct {
-	alias     string
-	expr      string
-	orderable bool
+	alias      string
+	expr       string
+	orderable  bool
+	filterable bool
 }
 
 func (b *scopedTraceStatementBuilder) resolveColumns(ctx context.Context, orgID valuer.UUID, start, end uint64, cols *columnResolver, preds *predicateResolver) ([]resolvedColumn, error) {
@@ -338,7 +340,7 @@ func (b *scopedTraceStatementBuilder) resolveColumns(ctx context.Context, orgID
 		if err != nil {
 			return nil, err
 		}
-		out = append(out, resolvedColumn{alias: c.Alias, expr: expr, orderable: c.Orderable})
+		out = append(out, resolvedColumn{alias: c.Alias, expr: expr, orderable: c.Orderable, filterable: c.Filterable})
 	}
 	return out, nil
 }
@@ -393,7 +395,7 @@ type filterParts struct {
 // splitFilter splits query.Filter into a span-level predicate (args bound into sb)
 // and a trace-level HAVING (explicit query.Having ANDed on), then validates the
 // trace-level part against the matched-pass aggregates.
-func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID valuer.UUID, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], classifySet, orderableSet map[string]struct{}, start, end uint64, variables map[string]qbtypes.VariableItem, sb *sqlbuilder.SelectBuilder) (filterParts, error) {
+func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID valuer.UUID, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], classifySet, filterableSet map[string]struct{}, start, end uint64, variables map[string]qbtypes.VariableItem, sb *sqlbuilder.SelectBuilder) (filterParts, error) {
 	var fp filterParts
 	if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
 		spanExpr, traceExpr, err := querybuilder.SplitFilterForAggregates(query.Filter.Expression, classifySet)
@@ -429,7 +431,7 @@ func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID val
 		}
 		fp.havingExpr = replaced
 	}
-	if err := validateAggregateFilter(fp.havingExpr, orderableSet); err != nil {
+	if err := validateAggregateFilter(fp.havingExpr, filterableSet); err != nil {
 		return fp, err
 	}
 	return fp, nil
@@ -473,7 +475,7 @@ func (b *scopedTraceStatementBuilder) resolveSpanPredicate(ctx context.Context,
 // span filter + HAVING + ORDER BY + LIMIT/OFFSET, selecting only the aliases ORDER BY
 // / HAVING reference. Expressions carry $n markers bound to sb, so each can appear
 // several times and every occurrence resolves to the same arg.
-func (b *scopedTraceStatementBuilder) buildMatchedCTE(sb *sqlbuilder.SelectBuilder, start, end, startBucket, endBucket uint64, resolved []resolvedColumn, orders []listOrder, orderableSet map[string]struct{}, maskExpr string, fp filterParts, resourcePred string, limit, offset int) (string, []any, error) {
+func (b *scopedTraceStatementBuilder) buildMatchedCTE(sb *sqlbuilder.SelectBuilder, start, end, startBucket, endBucket uint64, resolved []resolvedColumn, orders []listOrder, orderableSet, filterableSet map[string]struct{}, maskExpr string, fp filterParts, resourcePred string, limit, offset int) (string, []any, error) {
 	needed := neededMatchedAliases(orders, fp.havingExpr, orderableSet)
 	selects := []string{"trace_id"}
 	for _, rc := range resolved {
@@ -513,8 +515,8 @@ func (b *scopedTraceStatementBuilder) buildMatchedCTE(sb *sqlbuilder.SelectBuild
 	}
 	if strings.TrimSpace(fp.havingExpr) != "" {
 		// the rewriter matches raw key text, so map the trace. form alongside the bare name
-		columnMap := make(map[string]string, len(orderableSet)*2)
-		for a := range orderableSet {
+		columnMap := make(map[string]string, len(filterableSet)*2)
+		for a := range filterableSet {
 			columnMap[a] = quoteAlias(a)
 			columnMap[telemetrytypes.FieldContextTrace.StringValue()+"."+a] = quoteAlias(a)
 		}
@@ -603,6 +605,17 @@ func orderableAliasSet(resolved []resolvedColumn) map[string]struct{} {
 	return set
 }
 
+// filterableAliasSet is the subset of aliases usable in the trace-level filter.
+func filterableAliasSet(resolved []resolvedColumn) map[string]struct{} {
+	set := make(map[string]struct{})
+	for _, rc := range resolved {
+		if rc.filterable {
+			set[rc.alias] = struct{}{}
+		}
+	}
+	return set
+}
+
 // neededMatchedAliases is the minimal alias set the matched pass must select: those
 // in ORDER BY plus those in the aggregate HAVING.
 func neededMatchedAliases(orders []listOrder, havingExpr string, orderableSet map[string]struct{}) map[string]struct{} {
@@ -630,19 +643,19 @@ func traceAggregateNames(havingExpr string) []string {
 	return names
 }
 
-// validateAggregateFilter rejects a trace-level filter referencing an aggregate not
-// computable in the matched pass.
-func validateAggregateFilter(havingExpr string, orderableSet map[string]struct{}) error {
+// validateAggregateFilter rejects a trace-level filter referencing an aggregate that
+// is not filterable.
+func validateAggregateFilter(havingExpr string, filterableSet map[string]struct{}) error {
 	if strings.TrimSpace(havingExpr) == "" {
 		return nil
 	}
-	allowed := make([]string, 0, len(orderableSet))
-	for a := range orderableSet {
+	allowed := make([]string, 0, len(filterableSet))
+	for a := range filterableSet {
 		allowed = append(allowed, a)
 	}
 	sort.Strings(allowed)
 	for _, name := range traceAggregateNames(havingExpr) {
-		if _, ok := orderableSet[name]; !ok {
+		if _, ok := filterableSet[name]; !ok {
 			return errors.NewInvalidInputf(errors.CodeInvalidInput,
 				"aggregate %q cannot be used in the trace-list filter; filterable aggregates: %s", name, strings.Join(allowed, ", "))
 		}

pkg/telemetrymetadata/metadata.go

@@ -1168,27 +1168,6 @@ func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFi
 	return keys
 }
 
-// enrichWithGenAIKeys adds keys that can be queried for GenAI signals, even though they have not been ingested yet.
-func enrichWithGenAIKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
-	for _, selector := range selectors {
-		if selector.Signal != telemetrytypes.SignalTraces && selector.Signal != telemetrytypes.SignalUnspecified {
-			continue
-		}
-		for name, def := range telemetrytypes.GenAIFieldDefinitions {
-			if len(keys[name]) > 0 {
-				continue // already resolved from ingested data
-			}
-			if !selectorMatchesIntrinsicField(selector, def) {
-				continue
-			}
-			keyCopy := def
-			keys[name] = []*telemetrytypes.TelemetryFieldKey{&keyCopy}
-		}
-	}
-
-	return keys
-}
-
 func selectorMatchesIntrinsicField(selector *telemetrytypes.FieldKeySelector, definition telemetrytypes.TelemetryFieldKey) bool {
 	if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != definition.FieldContext {
 		return false
@@ -1274,9 +1253,6 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, orgID valuer.UUID, fie
 
 	applyBackwardCompatibleKeys(mapOfKeys)
 	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, selectors)
-	if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
-		mapOfKeys = enrichWithGenAIKeys(mapOfKeys, selectors)
-	}
 
 	return mapOfKeys, complete, nil
 }
@@ -1355,9 +1331,6 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, orgID valuer.UUID
 
 	applyBackwardCompatibleKeys(mapOfKeys)
 	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, fieldKeySelectors)
-	if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
-		mapOfKeys = enrichWithGenAIKeys(mapOfKeys, fieldKeySelectors)
-	}
 
 	return mapOfKeys, complete, nil
 }

pkg/telemetryschema/aitelemetryschema/const.go

@@ -0,0 +1,69 @@
+package aitelemetryschema
+
+import (
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// OpenTelemetry gen_ai semantic-convention attribute keys. Single source of truth
+// shared by the AI query builder and the LLM pricing pipeline.
+const (
+	GenAIRequestModel = "gen_ai.request.model"
+	GenAIToolName     = "gen_ai.tool.name"
+	GenAIAgentName    = "gen_ai.agent.name"
+	GenAIProviderName = "gen_ai.provider.name"
+
+	GenAIUsageInputTokens              = "gen_ai.usage.input_tokens"
+	GenAIUsageOutputTokens             = "gen_ai.usage.output_tokens"
+	GenAIUsageCacheReadInputTokens     = "gen_ai.usage.cache_read.input_tokens"
+	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
+
+	GenAIInputMessages  = "gen_ai.input.messages"
+	GenAIOutputMessages = "gen_ai.output.messages"
+
+	// SignozGenAITotalCost is not OTel semconv: it is the per-span cost the SigNoz
+	// LLM pricing processor attaches.
+	SignozGenAITotalCost = "_signoz.gen_ai.total_cost"
+)
+
+var (
+	// GenAIFields are the gen_ai span attributes the AI query builder relies on,
+	// suggested before ingestion so the filter bar works on a fresh install.
+	GenAIFields = map[string]telemetrytypes.TelemetryFieldKey{
+		GenAIRequestModel: {Name: GenAIRequestModel, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
+		GenAIToolName:     {Name: GenAIToolName, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
+		GenAIAgentName:    {Name: GenAIAgentName, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
+		GenAIProviderName: {Name: GenAIProviderName, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
+
+		GenAIUsageInputTokens:              {Name: GenAIUsageInputTokens, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeFloat64},
+		GenAIUsageOutputTokens:             {Name: GenAIUsageOutputTokens, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeFloat64},
+		GenAIUsageCacheReadInputTokens:     {Name: GenAIUsageCacheReadInputTokens, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeFloat64},
+		GenAIUsageCacheCreationInputTokens: {Name: GenAIUsageCacheCreationInputTokens, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeFloat64},
+		SignozGenAITotalCost:               {Name: SignozGenAITotalCost, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeFloat64},
+
+		GenAIInputMessages:  {Name: GenAIInputMessages, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
+		GenAIOutputMessages: {Name: GenAIOutputMessages, Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
+	}
+
+	// TraceAggregateFields are the per-trace aggregates the AI trace list computes.
+	// They are never ingested, so only this definition can surface them; the query
+	// builder marks the matching columns filterable.
+	TraceAggregateFields = map[string]telemetrytypes.TelemetryFieldKey{
+		"llm_call_count":        traceAggregate("llm_call_count"),
+		"tool_call_count":       traceAggregate("tool_call_count"),
+		"distinct_tool_count":   traceAggregate("distinct_tool_count"),
+		"input_tokens":          traceAggregate("input_tokens"),
+		"output_tokens":         traceAggregate("output_tokens"),
+		"total_tokens":          traceAggregate("total_tokens"),
+		"estimated_total_cost":  traceAggregate("estimated_total_cost"),
+		"max_llm_duration_nano": traceAggregate("max_llm_duration_nano"),
+	}
+)
+
+func traceAggregate(name string) telemetrytypes.TelemetryFieldKey {
+	return telemetrytypes.TelemetryFieldKey{
+		Name:          name,
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextTrace,
+		FieldDataType: telemetrytypes.FieldDataTypeFloat64,
+	}
+}

pkg/telemetryschema/aitelemetryschema/field_keys.go

@@ -0,0 +1,27 @@
+package aitelemetryschema
+
+import (
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// FieldKeys merges in the keys the metadata store cannot serve: the per-trace
+// aggregates always, since they are computed, and the gen_ai attributes only where
+// ingested metadata has not already resolved them.
+func FieldKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selector *telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
+	for name, def := range TraceAggregateFields {
+		if selector.MatchesKey(&def) {
+			keys[name] = append(keys[name], &def)
+		}
+	}
+
+	for name, def := range GenAIFields {
+		if len(keys[name]) > 0 {
+			continue
+		}
+		if selector.MatchesKey(&def) {
+			keys[name] = []*telemetrytypes.TelemetryFieldKey{&def}
+		}
+	}
+
+	return keys
+}

pkg/telemetryschema/aitelemetryschema/field_keys_test.go

@@ -0,0 +1,26 @@
+package aitelemetryschema
+
+import (
+	"testing"
+
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/stretchr/testify/assert"
+)
+
+// an ingested variant wins, so a materialized or differently typed key is not
+// shadowed by the static definition
+func TestFieldKeys_IngestedGenAIAttributeKept(t *testing.T) {
+	ingested := &telemetrytypes.TelemetryFieldKey{
+		Name:          GenAIRequestModel,
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextAttribute,
+		FieldDataType: telemetrytypes.FieldDataTypeString,
+		Materialized:  true,
+	}
+
+	keys := FieldKeys(map[string][]*telemetrytypes.TelemetryFieldKey{
+		GenAIRequestModel: {ingested},
+	}, &telemetrytypes.FieldKeySelector{SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy})
+
+	assert.Equal(t, []*telemetrytypes.TelemetryFieldKey{ingested}, keys[GenAIRequestModel])
+}

pkg/types/llmpricingruletypes/processorconfig.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 
 	"github.com/SigNoz/signoz/pkg/errors"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/telemetryschema/aitelemetryschema"
 	"gopkg.in/yaml.v3"
 )
 
@@ -84,11 +84,11 @@ func buildProcessorConfig(rules []*LLMPricingRule) *LLMPricingRuleProcessorConfi
 
 	return &LLMPricingRuleProcessorConfig{
 		Attrs: LLMPricingRuleProcessorAttrs{
-			Model:      telemetrytypes.GenAIRequestModel,
-			In:         telemetrytypes.GenAIUsageInputTokens,
-			Out:        telemetrytypes.GenAIUsageOutputTokens,
-			CacheRead:  telemetrytypes.GenAIUsageCacheReadInputTokens,
-			CacheWrite: telemetrytypes.GenAIUsageCacheCreationInputTokens,
+			Model:      aitelemetryschema.GenAIRequestModel,
+			In:         aitelemetryschema.GenAIUsageInputTokens,
+			Out:        aitelemetryschema.GenAIUsageOutputTokens,
+			CacheRead:  aitelemetryschema.GenAIUsageCacheReadInputTokens,
+			CacheWrite: aitelemetryschema.GenAIUsageCacheCreationInputTokens,
 		},
 		DefaultPricing: LLMPricingRuleProcessorDefaultPricing{
 			Rules: pricingRules,
@@ -98,7 +98,7 @@ func buildProcessorConfig(rules []*LLMPricingRule) *LLMPricingRuleProcessorConfi
 			Out:        SignozGenAICostOutput,
 			CacheRead:  SignozGenAICostCacheRead,
 			CacheWrite: SignozGenAICostCacheWrite,
-			Total:      telemetrytypes.SignozGenAITotalCost,
+			Total:      aitelemetryschema.SignozGenAITotalCost,
 		},
 	}
 }

pkg/types/telemetrytypes/field.go

@@ -238,6 +238,7 @@ type FieldKeySelector struct {
 	EndUnixMilli      int64                  `json:"endUnixMilli"`
 	Signal            Signal                 `json:"signal"`
 	Source            Source                 `json:"source"`
+	QueryType         string                 `json:"queryType"`
 	FieldContext      FieldContext           `json:"fieldContext"`
 	FieldDataType     FieldDataType          `json:"fieldDataType"`
 	Name              string                 `json:"name"`
@@ -246,6 +247,27 @@ type FieldKeySelector struct {
 	MetricContext     *MetricContext         `json:"metricContext,omitempty"`
 }
 
+// MatchesKey reports whether a statically defined key satisfies the selector, so
+// callers can suggest keys that were never ingested.
+func (s *FieldKeySelector) MatchesKey(key *TelemetryFieldKey) bool {
+	if s.FieldContext != FieldContextUnspecified && s.FieldContext != key.FieldContext {
+		return false
+	}
+
+	if s.FieldDataType != FieldDataTypeUnspecified && s.FieldDataType != key.FieldDataType {
+		return false
+	}
+
+	if s.Name == "" {
+		return true
+	}
+
+	if s.SelectorMatchType == FieldSelectorMatchTypeExact {
+		return strings.EqualFold(s.Name, key.Name)
+	}
+	return strings.Contains(strings.ToLower(key.Name), strings.ToLower(s.Name))
+}
+
 type FieldValueSelector struct {
 	*FieldKeySelector
 	ExistingQuery string `json:"existingQuery"`
@@ -261,6 +283,7 @@ type GettableFieldKeys struct {
 type PostableFieldKeysParams struct {
 	Signal          Signal        `query:"signal"`
 	Source          Source        `query:"source"`
+	Type            string        `query:"type"`
 	Limit           int           `query:"limit"`
 	StartUnixMilli  int64         `query:"startUnixMilli"`
 	EndUnixMilli    int64         `query:"endUnixMilli"`
@@ -297,6 +320,7 @@ func NewFieldKeySelectorFromPostableFieldKeysParams(params PostableFieldKeysPara
 
 	req.Signal = params.Signal
 	req.Source = params.Source
+	req.QueryType = params.Type
 	req.FieldContext = params.FieldContext
 	req.FieldDataType = params.FieldDataType
 	req.SelectorMatchType = FieldSelectorMatchTypeFuzzy

pkg/types/telemetrytypes/genai_semconv.go

@@ -1,41 +0,0 @@
-package telemetrytypes
-
-// OpenTelemetry gen_ai semantic-convention attribute keys. Single source of truth
-// shared by the AI query builder and the LLM pricing pipeline.
-const (
-	GenAIRequestModel = "gen_ai.request.model"
-	GenAIToolName     = "gen_ai.tool.name"
-	GenAIAgentName    = "gen_ai.agent.name"
-	GenAIProviderName = "gen_ai.provider.name"
-
-	GenAIUsageInputTokens              = "gen_ai.usage.input_tokens"
-	GenAIUsageOutputTokens             = "gen_ai.usage.output_tokens"
-	GenAIUsageCacheReadInputTokens     = "gen_ai.usage.cache_read.input_tokens"
-	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
-
-	GenAIInputMessages  = "gen_ai.input.messages"
-	GenAIOutputMessages = "gen_ai.output.messages"
-
-	// SignozGenAITotalCost is not OTel semconv: it is the per-span cost the SigNoz
-	// LLM pricing processor attaches.
-	SignozGenAITotalCost = "_signoz.gen_ai.total_cost"
-)
-
-// GenAIFieldDefinitions are the gen_ai span attributes the AI query builder relies
-// on, surfaced by the metadata store even before ingestion so the AI gate/columns
-// resolve on a fresh install.
-var GenAIFieldDefinitions = map[string]TelemetryFieldKey{
-	GenAIRequestModel: {Name: GenAIRequestModel, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIToolName:     {Name: GenAIToolName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIAgentName:    {Name: GenAIAgentName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIProviderName: {Name: GenAIProviderName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-
-	GenAIUsageInputTokens:              {Name: GenAIUsageInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageOutputTokens:             {Name: GenAIUsageOutputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageCacheReadInputTokens:     {Name: GenAIUsageCacheReadInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageCacheCreationInputTokens: {Name: GenAIUsageCacheCreationInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	SignozGenAITotalCost:               {Name: SignozGenAITotalCost, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-
-	GenAIInputMessages:  {Name: GenAIInputMessages, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIOutputMessages: {Name: GenAIOutputMessages, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-}

tests/fixtures/metadata.py

@@ -6,6 +6,7 @@ from typing import Any
 
 import numpy as np
 import pytest
+import requests
 
 from fixtures import types
 from fixtures.fingerprint import LogsOrTracesFingerprint
@@ -106,6 +107,24 @@ def truncate_attributes_metadata_table(conn, cluster: str) -> None:
     conn.query(f"TRUNCATE TABLE signoz_metadata.attributes_metadata ON CLUSTER '{cluster}' SYNC")
 
 
+def get_field_keys(signoz: types.SigNoz, token: str, params: dict) -> requests.Response:
+    return requests.get(
+        signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
+        timeout=5,
+        headers={"authorization": f"Bearer {token}"},
+        params=params,
+    )
+
+
+def get_ai_observability_field_keys(signoz: types.SigNoz, token: str, params: dict) -> requests.Response:
+    return requests.get(
+        signoz.self.host_configs["8080"].get("/api/v1/fields/ai_observability/keys"),
+        timeout=5,
+        headers={"authorization": f"Bearer {token}"},
+        params=params,
+    )
+
+
 @pytest.fixture(name="insert_attributes_metadata", scope="function")
 def insert_attributes_metadata(
     clickhouse: types.TestContainerClickhouse,

tests/integration/tests/querierai/03_ai_metadata.py

@@ -0,0 +1,81 @@
+from collections.abc import Callable
+from http import HTTPStatus
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.metadata import get_ai_observability_field_keys, get_field_keys
+
+# The filterable per-trace aggregates; the display-only columns (error_count,
+# last_activity_time, span_count, input, output) must not be suggested.
+AI_TRACE_AGGREGATES = {
+    "llm_call_count",
+    "tool_call_count",
+    "distinct_tool_count",
+    "input_tokens",
+    "output_tokens",
+    "total_tokens",
+    "estimated_total_cost",
+    "max_llm_duration_nano",
+}
+
+
+def test_ai_fields_lists_filterable_aggregates(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_ai_observability_field_keys(signoz, token, {})
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert response.json()["status"] == "success"
+
+    keys = response.json()["data"]["keys"]
+    trace_context_names = {name for name, variants in keys.items() if any(key["fieldContext"] == "trace" for key in variants)}
+    assert trace_context_names == AI_TRACE_AGGREGATES, keys
+
+
+def test_ai_fields_trace_prefix_search(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    # `trace.output` in the filter bar parses into the trace context
+    response = get_ai_observability_field_keys(signoz, token, {"searchText": "trace.output"})
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    keys = response.json()["data"]["keys"]
+    trace_context_names = {name for name, variants in keys.items() if any(key["fieldContext"] == "trace" for key in variants)}
+    assert trace_context_names == {"output_tokens"}, keys
+
+
+def test_ai_fields_bare_prefix_suggests_aggregate_and_attribute(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_ai_observability_field_keys(signoz, token, {"searchText": "output_tok"})
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    keys = response.json()["data"]["keys"]
+    assert any(key["fieldContext"] == "trace" for key in keys["output_tokens"]), keys
+    assert any(key["fieldContext"] == "attribute" for key in keys["gen_ai.usage.output_tokens"]), keys
+
+
+def test_ai_fields_are_not_served_by_the_generic_endpoint(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_field_keys(signoz, token, {"signal": "traces", "searchText": "output_tok"})
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    keys = response.json()["data"]["keys"]
+    assert "output_tokens" not in keys, keys
+    assert "gen_ai.usage.output_tokens" not in keys, keys

tests/integration/tests/querierai/conftest.py

@@ -0,0 +1,31 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz_ai(
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.SigNoz:
+    # AI observability ships behind this flag, so the suite runs with it on
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz-ai",
+        env_overrides={
+            "SIGNOZ_FLAGGER_CONFIG_BOOLEAN_ENABLE__AI__OBSERVABILITY": True,
+        },
+    )