fix: support for related values in ai field values

docs/api/openapi.yml

@@ -2559,7 +2559,6 @@ components:
       - factor-api-key
       - license
       - subscription
-      - deployment-host
       - logs
       - traces
       - metrics
@@ -9223,6 +9222,10 @@ paths:
         name: name
         schema:
           type: string
+      - in: query
+        name: existingQuery
+        schema:
+          type: string
       responses:
         "200":
           content:
@@ -23943,9 +23946,9 @@ paths:
           description: Internal Server Error
       security:
       - api_key:
-        - deployment-host:list
+        - VIEWER
       - tokenizer:
-        - deployment-host:list
+        - VIEWER
       summary: Get host info from Zeus.
       tags:
       - zeus
@@ -23999,9 +24002,9 @@ paths:
           description: Internal Server Error
       security:
       - api_key:
-        - deployment-host:update
+        - ADMIN
       - tokenizer:
-        - deployment-host:update
+        - ADMIN
       summary: Put host in Zeus for a deployment.
       tags:
       - zeus

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -2175,7 +2175,6 @@ export enum CoretypesKindDTO {
 	'factor-api-key' = 'factor-api-key',
 	license = 'license',
 	subscription = 'subscription',
-	'deployment-host' = 'deployment-host',
 	logs = 'logs',
 	traces = 'traces',
 	metrics = 'metrics',
@@ -10249,6 +10248,11 @@ export type GetAIObservabilityFieldsValuesParams = {
 	 * @description undefined
 	 */
 	name?: string;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	existingQuery?: string;
 };
 
 export type GetAIObservabilityFieldsValues200 = {

frontend/src/components/QueryBuilderV2/QueryBuilderV2.tsx

@@ -241,29 +241,28 @@ export const QueryBuilderV2 = memo(function QueryBuilderV2({
 						))
 					)}
 
-					{!showOnlyWhereClause &&
-						currentQuery.builder.queryFormulas?.length > 0 && (
-							<div className="qb-formulas-container">
-								{currentQuery.builder.queryFormulas.map((formula, index) => {
-									const query =
-										currentQuery.builder.queryData[index] ||
-										currentQuery.builder.queryData[0];
+					{!showOnlyWhereClause && currentQuery.builder.queryFormulas.length > 0 && (
+						<div className="qb-formulas-container">
+							{currentQuery.builder.queryFormulas.map((formula, index) => {
+								const query =
+									currentQuery.builder.queryData[index] ||
+									currentQuery.builder.queryData[0];
 
-									return (
-										<div key={formula.queryName} className="qb-formula">
-											<Formula
-												filterConfigs={filterConfigs}
-												query={query}
-												formula={formula}
-												index={index}
-												isAdditionalFilterEnable={false}
-												isQBV2
-											/>
-										</div>
-									);
-								})}
-							</div>
-						)}
+								return (
+									<div key={formula.queryName} className="qb-formula">
+										<Formula
+											filterConfigs={filterConfigs}
+											query={query}
+											formula={formula}
+											index={index}
+											isAdditionalFilterEnable={false}
+											isQBV2
+										/>
+									</div>
+								);
+							})}
+						</div>
+					)}
 
 					{shouldShowFooter && (
 						<QueryFooter
@@ -291,7 +290,7 @@ export const QueryBuilderV2 = memo(function QueryBuilderV2({
 							</div>
 						))}
 
-						{currentQuery.builder.queryFormulas?.map((formula) => (
+						{currentQuery.builder.queryFormulas.map((formula) => (
 							<div key={formula.queryName} className="formula-name">
 								{formula.queryName}
 							</div>

frontend/src/components/QueryBuilderV2/QueryV2/__tests__/QueryV2.test.tsx

@@ -212,32 +212,6 @@ describe('QueryBuilderV2 + QueryV2 - base render', () => {
 		expect(handleRunQueryMock).toHaveBeenCalled();
 	});
 
-	it('does not crash when builder.queryFormulas/queryTraceOperator are missing (partial/legacy query)', () => {
-		const currentQueryBase = baseQBContext.currentQuery as Query;
-
-		mockedUseQueryBuilder.mockReturnValue({
-			...baseQBContext,
-			currentQuery: {
-				...currentQueryBase,
-				builder: {
-					queryData: currentQueryBase.builder.queryData,
-					queryFormulas: undefined as unknown as [],
-					queryTraceOperator: undefined as unknown as [],
-				},
-			},
-		});
-
-		expect(() =>
-			render(<QueryBuilderV2 panelType={PANEL_TYPES.TABLE} version="v4" />),
-		).not.toThrow();
-
-		// query list still renders from queryData, formulas block is skipped
-		expect(document.querySelector('.query-names-section')).toBeInTheDocument();
-		expect(
-			document.querySelector('.qb-formulas-container'),
-		).not.toBeInTheDocument();
-	});
-
 	it('fx button is disabled when functions already exist', () => {
 		const currentQueryBase = baseQBContext.currentQuery as Query;
 		const supersetQueryBase = baseQBContext.supersetQuery as Query;

frontend/src/components/ServiceAccountDrawer/__tests__/ServiceAccountDrawer.authz.test.tsx

@@ -4,7 +4,7 @@ import {
 } from 'mocks-server/__mockdata__/roles';
 import { rest, server } from 'mocks-server/server';
 import { NuqsTestingAdapter } from 'nuqs/adapters/testing';
-import { render, screen, waitFor } from 'tests/test-utils';
+import { fireEvent, render, screen, waitFor } from 'tests/test-utils';
 import {
 	setupAuthzAdmin,
 	setupAuthzDeny,
@@ -110,7 +110,10 @@ describe('ServiceAccountDrawer — permissions', () => {
 	it('shows PermissionDeniedCallout in Keys tab when list-keys permission is denied', async () => {
 		server.use(setupAuthzDeny(APIKeyListPermission));
 
-		renderDrawer({ account: 'sa-1', tab: 'keys' });
+		renderDrawer();
+		await screen.findByDisplayValue('CI Bot');
+
+		fireEvent.click(screen.getByRole('radio', { name: /keys/i }));
 
 		await waitFor(() => {
 			expect(screen.getByText(/list:factor-api-key/)).toBeInTheDocument();

frontend/src/container/FormAlertRules/utils.ts

@@ -53,24 +53,17 @@ export const getUpdatedStepInterval = (evalWindow?: string): number => {
 };
 
 export const getSelectedQueryOptions = (
-	queries:
-		| Array<
-				| IBuilderQuery
-				| IBuilderTraceOperator
-				| IBuilderFormula
-				| IClickHouseQuery
-				| IPromQLQuery
-		  >
-		| undefined
-		| null,
-): SelectProps['options'] => {
-	if (!queries) {
-		return [];
-	}
-	return queries
+	queries: Array<
+		| IBuilderQuery
+		| IBuilderTraceOperator
+		| IBuilderFormula
+		| IClickHouseQuery
+		| IPromQLQuery
+	>,
+): SelectProps['options'] =>
+	queries
 		.filter((query) => !query.disabled)
 		.map((query) => ({
 			label: 'queryName' in query ? query.queryName : query.name,
 			value: 'queryName' in query ? query.queryName : query.name,
 		}));
-};

frontend/src/hooks/queryBuilder/useGetCompositeQueryParam.ts

@@ -27,14 +27,6 @@ export const useGetCompositeQueryParam = (): Query | null => {
 				decodeURIComponent(compositeQuery.replace(/\+/g, ' ')),
 			);
 
-			// Add default values for optional fields if empty
-			if (parsedCompositeQuery?.builder) {
-				parsedCompositeQuery.builder.queryFormulas =
-					parsedCompositeQuery.builder.queryFormulas ?? [];
-				parsedCompositeQuery.builder.queryTraceOperator =
-					parsedCompositeQuery.builder.queryTraceOperator ?? [];
-			}
-
 			// Convert old format to new format for each query in builder.queryData
 			if (parsedCompositeQuery?.builder?.queryData) {
 				parsedCompositeQuery.builder.queryData =

frontend/src/providers/QueryBuilder.tsx

@@ -163,23 +163,20 @@ export function QueryBuilderProvider({
 	const prepareQueryBuilderData = useCallback(
 		(query: Query): Query => {
 			const builder: QueryBuilderData = {
-				queryData:
-					query.builder.queryData?.map((item) => ({
-						...initialQueryBuilderFormValuesMap[
-							initialDataSource || DataSource.METRICS
-						],
-						...item,
-					})) ?? [],
-				queryFormulas:
-					query.builder.queryFormulas?.map((item) => ({
-						...initialFormulaBuilderFormValues,
-						...item,
-					})) ?? [],
-				queryTraceOperator:
-					query.builder.queryTraceOperator?.map((item) => ({
-						...initialQueryBuilderFormTraceOperatorValues,
-						...item,
-					})) ?? [],
+				queryData: query.builder.queryData?.map((item) => ({
+					...initialQueryBuilderFormValuesMap[
+						initialDataSource || DataSource.METRICS
+					],
+					...item,
+				})),
+				queryFormulas: query.builder.queryFormulas?.map((item) => ({
+					...initialFormulaBuilderFormValues,
+					...item,
+				})),
+				queryTraceOperator: query.builder.queryTraceOperator?.map((item) => ({
+					...initialQueryBuilderFormTraceOperatorValues,
+					...item,
+				})),
 			};
 
 			const setupedQueryData = builder.queryData.map((item) => {
@@ -212,17 +209,15 @@ export function QueryBuilderProvider({
 				return currentElement;
 			});
 
-			const promql: IPromQLQuery[] =
-				query.promql?.map((item) => ({
-					...initialQueryPromQLData,
-					...item,
-				})) ?? [];
+			const promql: IPromQLQuery[] = query.promql.map((item) => ({
+				...initialQueryPromQLData,
+				...item,
+			}));
 
-			const clickHouse: IClickHouseQuery[] =
-				query.clickhouse_sql?.map((item) => ({
-					...initialClickHouseData,
-					...item,
-				})) ?? [];
+			const clickHouse: IClickHouseQuery[] = query.clickhouse_sql.map((item) => ({
+				...initialClickHouseData,
+				...item,
+			}));
 
 			const newQueryState: QueryState = {
 				clickhouse_sql: clickHouse,

frontend/src/schemas/generated/transactionGroups.schema.json

@@ -66,7 +66,6 @@
         "factor-api-key",
         "license",
         "subscription",
-        "deployment-host",
         "logs",
         "traces",
         "metrics",

pkg/apiserver/signozapiserver/zeus.go

@@ -5,8 +5,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/http/handler"
 	"github.com/SigNoz/signoz/pkg/types"
-	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/types/coretypes"
 	"github.com/SigNoz/signoz/pkg/types/zeustypes"
 	"github.com/gorilla/mux"
 )
@@ -29,7 +27,7 @@ func (provider *provider) addZeusRoutes(router *mux.Router) error {
 		return err
 	}
 
-	if err := router.Handle("/api/v2/zeus/hosts", handler.New(provider.authzMiddleware.CheckResources(provider.zeusHandler.GetHosts, authtypes.SigNozAdminRoleName, authtypes.SigNozEditorRoleName, authtypes.SigNozViewerRoleName), handler.OpenAPIDef{
+	if err := router.Handle("/api/v2/zeus/hosts", handler.New(provider.authzMiddleware.ViewAccess(provider.zeusHandler.GetHosts), handler.OpenAPIDef{
 		ID:                  "GetHosts",
 		Tags:                []string{"zeus"},
 		Summary:             "Get host info from Zeus.",
@@ -41,17 +39,12 @@ func (provider *provider) addZeusRoutes(router *mux.Router) error {
 		SuccessStatusCode:   http.StatusOK,
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound},
 		Deprecated:          false,
-		SecuritySchemes:     newScopedSecuritySchemes([]string{coretypes.ResourceMetaResourceDeploymentHost.Scope(coretypes.VerbList)}),
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceDeploymentHost,
-		Verb:     coretypes.VerbList,
-		Category: coretypes.ActionCategoryDataAccess,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodGet).GetError(); err != nil {
+		SecuritySchemes:     newSecuritySchemes(types.RoleViewer),
+	})).Methods(http.MethodGet).GetError(); err != nil {
 		return err
 	}
 
-	if err := router.Handle("/api/v2/zeus/hosts", handler.New(provider.authzMiddleware.CheckResources(provider.zeusHandler.PutHost, authtypes.SigNozAdminRoleName), handler.OpenAPIDef{
+	if err := router.Handle("/api/v2/zeus/hosts", handler.New(provider.authzMiddleware.AdminAccess(provider.zeusHandler.PutHost), handler.OpenAPIDef{
 		ID:                  "PutHost",
 		Tags:                []string{"zeus"},
 		Summary:             "Put host in Zeus for a deployment.",
@@ -63,14 +56,8 @@ func (provider *provider) addZeusRoutes(router *mux.Router) error {
 		SuccessStatusCode:   http.StatusNoContent,
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusConflict},
 		Deprecated:          false,
-		SecuritySchemes:     newScopedSecuritySchemes([]string{coretypes.ResourceMetaResourceDeploymentHost.Scope(coretypes.VerbUpdate)}),
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceDeploymentHost,
-		Verb:     coretypes.VerbUpdate,
-		Category: coretypes.ActionCategoryConfigurationChange,
-		ID:       coretypes.BodyJSONPath("name"),
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodPut).GetError(); err != nil {
+		SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+	})).Methods(http.MethodPut).GetError(); err != nil {
 		return err
 	}
 

pkg/modules/aiobservability/implaiobservability/handler.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/http/binding"
 	"github.com/SigNoz/signoz/pkg/http/render"
 	"github.com/SigNoz/signoz/pkg/modules/aiobservability"
@@ -66,13 +65,6 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques
 	ctx, cancel := context.WithTimeout(req.Context(), 10*time.Second)
 	defer cancel()
 
-	// binding ignores query params the struct does not declare, so an unsupported
-	// existingQuery would silently return values it did not narrow
-	if req.URL.Query().Has("existingQuery") {
-		render.Error(rw, errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "existingQuery is not supported"))
-		return
-	}
-
 	var params aiobservabilitytypes.PostableFieldValueParams
 	if err := binding.Query.BindQuery(req.URL.Query(), &params); err != nil {
 		render.Error(rw, err)
@@ -84,18 +76,29 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques
 		render.Error(rw, err)
 		return
 	}
+	orgID := valuer.MustNewUUID(claims.OrgID)
 
+	params.ExistingQuery = aitelemetryschema.ScopedExistingQuery(params.ExistingQuery)
 	fieldValueSelector := aiobservabilitytypes.NewFieldValueSelectorFromPostableFieldValueParams(params)
 
 	values := &telemetrytypes.TelemetryFieldValues{}
 	complete := true
 	// the trace context names the computed per-trace aggregates, which are never ingested
 	if fieldValueSelector.FieldContext != telemetrytypes.FieldContextTrace {
-		values, complete, err = handler.telemetryMetadataStore.GetAllValues(ctx, valuer.MustNewUUID(claims.OrgID), fieldValueSelector)
+		values, complete, err = handler.telemetryMetadataStore.GetAllValues(ctx, orgID, fieldValueSelector)
 		if err != nil {
 			render.Error(rw, err)
 			return
 		}
+
+		// related values are best-effort: on failure the plain values still serve
+		// the filter bar
+		relatedValues, relatedComplete, err := handler.telemetryMetadataStore.GetRelatedValues(ctx, orgID, fieldValueSelector)
+		if err != nil {
+			relatedValues = []string{}
+		}
+		values.RelatedValues = relatedValues
+		complete = complete && relatedComplete
 	}
 
 	render.Success(rw, http.StatusOK, &telemetrytypes.GettableFieldValues{

pkg/querybuilder/exists_expr.go

@@ -43,7 +43,7 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
 		if len(evolutionsEntries) > 0 && evolutionsEntries[0] != nil {
 			columnName = evolutionsEntries[0].ColumnName
 		}
-		rawPath := fmt.Sprintf("%s.%s", columnName, ClickHouseIdentifier(key.Name))
+		rawPath := fmt.Sprintf("%s.`%s`", columnName, key.Name)
 		if exists {
 			return rawPath + " IS NOT NULL", nil
 		}

pkg/signoz/provider.go

@@ -244,7 +244,6 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewDeleteOrphanUserRolesFactory(),
 		sqlmigration.NewMigrateLambdaDashboardsFactory(),
 		sqlmigration.NewAddAuthDomainTuplesFactory(sqlstore),
-		sqlmigration.NewAddDeploymentHostTuplesFactory(sqlstore),
 	)
 }
 

pkg/sqlmigration/118_add_deployment_host_tuples.go

@@ -1,161 +0,0 @@
-package sqlmigration
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"time"
-
-	"github.com/SigNoz/signoz/pkg/factory"
-	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/types/coretypes"
-	"github.com/oklog/ulid/v2"
-	"github.com/uptrace/bun"
-	"github.com/uptrace/bun/dialect"
-	"github.com/uptrace/bun/migrate"
-)
-
-type addDeploymentHostTuples struct {
-	sqlstore sqlstore.SQLStore
-}
-
-func NewAddDeploymentHostTuplesFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
-	return factory.NewProviderFactory(factory.MustNewName("add_deployment_host_tuples"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
-		return &addDeploymentHostTuples{sqlstore: sqlstore}, nil
-	})
-}
-
-func (migration *addDeploymentHostTuples) Register(migrations *migrate.Migrations) error {
-	return migrations.Register(migration.Up, migration.Down)
-}
-
-func (migration *addDeploymentHostTuples) Up(ctx context.Context, db *bun.DB) error {
-	tx, err := db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-	defer func() { _ = tx.Rollback() }()
-
-	var storeID string
-	err = tx.QueryRowContext(ctx, `SELECT id FROM store WHERE name = ? LIMIT 1`, "signoz").Scan(&storeID)
-	if err != nil {
-		return err
-	}
-
-	var orgIDs []string
-	err = tx.NewSelect().
-		Table("organizations").
-		Column("id").
-		Scan(ctx, &orgIDs)
-	if err != nil && err != sql.ErrNoRows {
-		return err
-	}
-
-	isPG := migration.sqlstore.BunDB().Dialect().Name() == dialect.PG
-
-	// zeus hosts moved from the legacy ViewAccess/AdminAccess role gates to
-	// CheckResources, which on enterprise requires real tuples -- existing orgs
-	// never had these written, only new orgs get them from the registry at bootstrap.
-	tuples := []migrationTuple{
-		{authtypes.SigNozAdminRoleName, "metaresource", "deployment-host", "list"},
-		{authtypes.SigNozAdminRoleName, "metaresource", "deployment-host", "update"},
-		{authtypes.SigNozEditorRoleName, "metaresource", "deployment-host", "list"},
-		{authtypes.SigNozViewerRoleName, "metaresource", "deployment-host", "list"},
-	}
-
-	for _, orgID := range orgIDs {
-		for _, tuple := range tuples {
-			entropy := ulid.DefaultEntropy()
-			now := time.Now().UTC()
-			tupleID := ulid.MustNew(ulid.Timestamp(now), entropy).String()
-
-			objectID := "organization/" + orgID + "/" + tuple.objectName + "/*"
-			roleSubject := "organization/" + orgID + "/role/" + tuple.roleName
-
-			if isPG {
-				user := "role:" + roleSubject + "#assignee"
-				result, err := tx.ExecContext(ctx, `
-					INSERT INTO tuple (store, object_type, object_id, relation, _user, user_type, ulid, inserted_at)
-					VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-					ON CONFLICT (store, object_type, object_id, relation, _user) DO NOTHING`,
-					storeID, tuple.objectType, objectID, tuple.relation, user, "userset", tupleID, now,
-				)
-				if err != nil {
-					return err
-				}
-				rowsAffected, err := result.RowsAffected()
-				if err != nil {
-					return err
-				}
-				if rowsAffected == 0 {
-					continue
-				}
-				_, err = tx.ExecContext(ctx, `
-					INSERT INTO changelog (store, object_type, object_id, relation, _user, operation, ulid, inserted_at)
-					VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-					ON CONFLICT (store, ulid, object_type) DO NOTHING`,
-					storeID, tuple.objectType, objectID, tuple.relation, user, 0, tupleID, now,
-				)
-				if err != nil {
-					return err
-				}
-			} else {
-				result, err := tx.ExecContext(ctx, `
-					INSERT INTO tuple (store, object_type, object_id, relation, user_object_type, user_object_id, user_relation, user_type, ulid, inserted_at)
-					VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-					ON CONFLICT (store, object_type, object_id, relation, user_object_type, user_object_id, user_relation) DO NOTHING`,
-					storeID, tuple.objectType, objectID, tuple.relation, "role", roleSubject, "assignee", "userset", tupleID, now,
-				)
-				if err != nil {
-					return err
-				}
-				rowsAffected, err := result.RowsAffected()
-				if err != nil {
-					return err
-				}
-				if rowsAffected == 0 {
-					continue
-				}
-				_, err = tx.ExecContext(ctx, `
-					INSERT INTO changelog (store, object_type, object_id, relation, user_object_type, user_object_id, user_relation, operation, ulid, inserted_at)
-					VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-					ON CONFLICT (store, ulid, object_type) DO NOTHING`,
-					storeID, tuple.objectType, objectID, tuple.relation, "role", roleSubject, "assignee", 0, tupleID, now,
-				)
-				if err != nil {
-					return err
-				}
-			}
-		}
-	}
-
-	managedRoleGroups := make(map[string]string, len(coretypes.ManagedRoleToTransactions))
-	for roleName, transactions := range coretypes.ManagedRoleToTransactions {
-		data, err := json.Marshal(authtypes.NewTransactionGroupsFromTransactions(transactions))
-		if err != nil {
-			return err
-		}
-		managedRoleGroups[roleName] = string(data)
-	}
-
-	for _, orgID := range orgIDs {
-		for roleName, data := range managedRoleGroups {
-			if _, err := tx.NewUpdate().
-				Model(new(roles)).
-				Set("transaction_groups = ?", data).
-				Where("org_id = ?", orgID).
-				Where("type = ?", authtypes.RoleTypeManaged.StringValue()).
-				Where("name = ?", roleName).
-				Exec(ctx); err != nil {
-				return err
-			}
-		}
-	}
-
-	return tx.Commit()
-}
-
-func (migration *addDeploymentHostTuples) Down(context.Context, *bun.DB) error {
-	return nil
-}

pkg/statementbuilder/aistatementbuilder/statement_builder.go

@@ -1,8 +1,6 @@
 package aistatementbuilder
 
 import (
-	"strings"
-
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/statementbuilder"
@@ -27,11 +25,8 @@ func NewFactory(
 // Scope describes gen_ai for the scoped trace builder: an AI trace has >=1 gen_ai
 // LLM, tool, or agent span, and its list adds AI/LLM per-trace metrics.
 func Scope() scopedtraces.TraceScope {
-	gateKeyNames := []string{aiobservabilitytypes.GenAIRequestModel, aiobservabilitytypes.GenAIToolName, aiobservabilitytypes.GenAIAgentName}
-	gateExprs := make([]string, 0, len(gateKeyNames))
-	gateKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(gateKeyNames))
-	for _, name := range gateKeyNames {
-		gateExprs = append(gateExprs, name+" EXISTS")
+	gateKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(aiobservabilitytypes.GenAISpanGateKeys))
+	for _, name := range aiobservabilitytypes.GenAISpanGateKeys {
 		gateKeys = append(gateKeys, &telemetrytypes.TelemetryFieldKey{
 			Name:         name,
 			Signal:       telemetrytypes.SignalTraces,
@@ -79,7 +74,7 @@ func Scope() scopedtraces.TraceScope {
 	}
 
 	return scopedtraces.TraceScope{
-		FilterExpression:  strings.Join(gateExprs, " OR "),
+		FilterExpression:  aiobservabilitytypes.GenAISpanFilterExpression(),
 		FieldKeys:         gateKeys,
 		Columns:           columns,
 		DefaultOrderAlias: "last_activity_time",

pkg/telemetryschema/aitelemetryschema/existing_query.go

@@ -0,0 +1,36 @@
+package aitelemetryschema
+
+import (
+	"github.com/SigNoz/signoz/pkg/querybuilder"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
+)
+
+var (
+	traceAggregateNames = func() map[string]struct{} {
+		names := make(map[string]struct{}, len(TraceAggregateFields))
+		for name := range TraceAggregateFields {
+			names[name] = struct{}{}
+		}
+		return names
+	}()
+
+	genAISpanGate = "(" + aiobservabilitytypes.GenAISpanFilterExpression() + ")"
+)
+
+// ScopedExistingQuery narrows value suggestions to gen_ai spans: the caller's
+// filter minus its per-trace aggregate atoms (never ingested, so nothing can
+// narrow on them), ANDed with the gen_ai span gate. An unparseable filter is
+// dropped, matching how the metadata store treats it downstream.
+func ScopedExistingQuery(existingQuery string) string {
+	spanExpr := ""
+	if existingQuery != "" {
+		if expr, _, err := querybuilder.SplitFilterForAggregates(existingQuery, traceAggregateNames); err == nil {
+			spanExpr = expr
+		}
+	}
+
+	if spanExpr == "" {
+		return genAISpanGate
+	}
+	return genAISpanGate + " AND (" + spanExpr + ")"
+}

pkg/telemetryschema/aitelemetryschema/existing_query_test.go

@@ -0,0 +1,94 @@
+package aitelemetryschema
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestScopedExistingQuery(t *testing.T) {
+	gate := "(gen_ai.request.model EXISTS OR gen_ai.tool.name EXISTS OR gen_ai.agent.name EXISTS)"
+
+	testCases := []struct {
+		name          string
+		existingQuery string
+		expected      string
+	}{
+		{
+			name:          "empty query returns the gate alone",
+			existingQuery: "",
+			expected:      gate,
+		},
+		{
+			name:          "span filter is preserved under the gate",
+			existingQuery: "service.name = 'checkout'",
+			expected:      gate + " AND (service.name = 'checkout')",
+		},
+		{
+			name:          "trace aggregate filter is stripped",
+			existingQuery: "llm_call_count > 5",
+			expected:      gate,
+		},
+		{
+			name:          "mixed filter keeps only the span part",
+			existingQuery: "llm_call_count > 5 AND gen_ai.request.model = 'gpt-4'",
+			expected:      gate + " AND (gen_ai.request.model = 'gpt-4')",
+		},
+		{
+			name:          "trace context filter is stripped",
+			existingQuery: "trace.total_tokens > 100 AND service.name = 'checkout'",
+			expected:      gate + " AND (service.name = 'checkout')",
+		},
+		{
+			name:          "unparseable filter is dropped",
+			existingQuery: "service.name = ",
+			expected:      gate,
+		},
+		{
+			name:          "multiple span conditions survive as one AND chain",
+			existingQuery: "service.name = 'checkout' AND gen_ai.request.model = 'gpt-4' AND llm_call_count > 5",
+			expected:      gate + " AND (service.name = 'checkout' AND gen_ai.request.model = 'gpt-4')",
+		},
+		{
+			name:          "span OR group is kept whole and parenthesized against the AND join",
+			existingQuery: "service.name = 'a' OR service.name = 'b'",
+			expected:      gate + " AND ((service.name = 'a' OR service.name = 'b'))",
+		},
+		{
+			name:          "parenthesized span OR group ANDed with an aggregate keeps only the group",
+			existingQuery: "(service.name = 'a' OR service.name = 'b') AND llm_call_count > 5",
+			expected:      gate + " AND ((service.name = 'a' OR service.name = 'b'))",
+		},
+		{
+			name:          "OR group of trace aggregates is stripped whole",
+			existingQuery: "llm_call_count > 5 OR total_tokens > 100",
+			expected:      gate,
+		},
+		{
+			name:          "OR mixing aggregate and span atoms drops the whole filter",
+			existingQuery: "llm_call_count > 5 OR service.name = 'checkout'",
+			expected:      gate,
+		},
+		{
+			name:          "parenthesized AND group is split, not routed whole",
+			existingQuery: "(llm_call_count > 5 AND service.name = 'checkout') AND gen_ai.request.model = 'gpt-4'",
+			expected:      gate + " AND (service.name = 'checkout' AND gen_ai.request.model = 'gpt-4')",
+		},
+		{
+			name:          "NOT over an aggregate group is stripped",
+			existingQuery: "NOT (llm_call_count > 5) AND service.name = 'checkout'",
+			expected:      gate + " AND (service.name = 'checkout')",
+		},
+		{
+			name:          "NOT over a span group is kept",
+			existingQuery: "NOT (service.name = 'checkout')",
+			expected:      gate + " AND (NOT (service.name = 'checkout'))",
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			assert.Equal(t, testCase.expected, ScopedExistingQuery(testCase.existingQuery))
+		})
+	}
+}

pkg/telemetryschema/tracestelemetryschema/const.go

@@ -43,7 +43,6 @@ const (
 	SpanAttributesStringColumn = "attributes_string"
 	SpanAttributesNumberColumn = "attributes_number"
 	SpanAttributesBoolColumn   = "attributes_bool"
-	SpanAttributesColumn       = "attributes"
 	SpanResourcesStringColumn  = "resources_string"
 )
 

pkg/telemetryschema/tracestelemetryschema/field_mapper.go

@@ -52,9 +52,8 @@ var (
 			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
 			ValueType: schema.ColumnTypeString,
 		}},
-		"resource":   {Name: "resource", Type: schema.JSONColumnType{}},
-		"scope":      {Name: "scope", Type: schema.JSONColumnType{}},
-		"attributes": {Name: "attributes", Type: schema.JSONColumnType{}},
+		"resource": {Name: "resource", Type: schema.JSONColumnType{}},
+		"scope":    {Name: "scope", Type: schema.JSONColumnType{}},
 
 		"events": {Name: "events", Type: schema.ArrayColumnType{
 			ElementType: schema.ColumnTypeString,
@@ -185,32 +184,16 @@ func (m *fieldMapper) getColumn(
 	case telemetrytypes.FieldContextScope:
 		return []*schema.Column{indexV3Columns["scope"]}, nil
 	case telemetrytypes.FieldContextAttribute:
-		// Only typed keys resolve here: a data-type-unspecified attribute key
-		// falls through to ErrColumnNotFound so bare keys keep taking the legacy
-		// CandidateKeys/synthesis path rather than flipping to metadata-first.
-		var mapCol *schema.Column
 		switch key.FieldDataType {
 		case telemetrytypes.FieldDataTypeString:
-			mapCol = indexV3Columns["attributes_string"]
+			return []*schema.Column{indexV3Columns["attributes_string"]}, nil
 		case telemetrytypes.FieldDataTypeInt64,
 			telemetrytypes.FieldDataTypeFloat64,
 			telemetrytypes.FieldDataTypeNumber:
-			mapCol = indexV3Columns["attributes_number"]
+			return []*schema.Column{indexV3Columns["attributes_number"]}, nil
 		case telemetrytypes.FieldDataTypeBool:
-			mapCol = indexV3Columns["attributes_bool"]
-		default:
-			return nil, qbtypes.ErrColumnNotFound
+			return []*schema.Column{indexV3Columns["attributes_bool"]}, nil
 		}
-		// Dual-read from the JSON column only once the attributes evolution entry
-		// is registered for this key; without it, resolution is byte-for-byte the
-		// legacy Map path. The JSON column is returned first so it wins the multiIf
-		// when both homes hold the key; SelectEvolutionsForColumns narrows the pair
-		// by the query's time range. This makes the evolution entry the rollout
-		// control, exactly as it is for resource/scope.
-		if attributeJSONEvolutionRegistered(key) {
-			return []*schema.Column{indexV3Columns["attributes"], mapCol}, nil
-		}
-		return []*schema.Column{mapCol}, nil
 	case telemetrytypes.FieldContextSpan:
 		// Check if this is a span scope field
 		if strings.ToLower(key.Name) == SpanSearchScopeRoot || strings.ToLower(key.Name) == SpanSearchScopeEntryPoint {
@@ -296,7 +279,6 @@ func (m *fieldMapper) resolveColumnExprs(
 		return nil, nil, nil, err
 	}
 
-	key = narrowEvolutionsToColumns(key, columns)
 	newColumns, evolutionsEntries, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
 	if err != nil {
 		return nil, nil, nil, err
@@ -327,17 +309,6 @@ func (m *fieldMapper) resolveColumnExprs(
 					exprs = append(exprs, fmt.Sprintf("%s.attributes.%s::String", columnName, querybuilder.ClickHouseIdentifier(attributeName)))
 					existExprs = append(existExprs, fmt.Sprintf("%s.attributes.%s IS NOT NULL", columnName, querybuilder.ClickHouseIdentifier(attributeName)))
 				}
-			case telemetrytypes.FieldContextAttribute:
-				// Span attributes are flat shared-data paths keyed by the attribute name
-				// verbatim (no nested object), so the name addresses the path directly.
-				// String casts to ::String so an absent path folds to '' — matching the
-				// Map column's default and preserving negative-operator parity; typed
-				// numeric/bool cast to Nullable so a missing or wrong-typed path reads
-				// NULL rather than 0/false. Existence tests the raw path (the cast folds
-				// NULL) and is index-eligible via attributes_paths_tokenbf.
-				path := fmt.Sprintf("%s.%s", columnName, querybuilder.ClickHouseIdentifier(key.Name))
-				exprs = append(exprs, fmt.Sprintf("%s::%s", path, attributeJSONCast(key.FieldDataType)))
-				existExprs = append(existExprs, fmt.Sprintf("%s IS NOT NULL", path))
 			default:
 				return nil, nil, nil, errors.NewInternalf(errors.CodeInternal, "only resource and scope context fields are supported for json columns, got %s", key.FieldContext.String)
 			}
@@ -382,66 +353,6 @@ func (m *fieldMapper) resolveColumnExprs(
 	return exprs, existExprs, columns, nil
 }
 
-// attributeJSONEvolutionRegistered reports whether the key carries an evolution entry for the
-// `attributes` JSON column. Until that entry is registered, attribute resolution stays on the
-// legacy Map column and the JSON column is never touched.
-func attributeJSONEvolutionRegistered(key *telemetrytypes.TelemetryFieldKey) bool {
-	for _, e := range key.Evolutions {
-		if e != nil && e.ColumnName == SpanAttributesColumn {
-			return true
-		}
-	}
-	return false
-}
-
-// narrowEvolutionsToColumns returns key.Evolutions filtered to entries whose column is in cols.
-// A metadata attribute key carries the `__all__` evolutions for every attribute-context column
-// (all three legacy maps plus the JSON column), but getColumn resolves a typed key to only its
-// own map + the JSON column; without this filter SelectEvolutionsForColumns would reject the
-// in-range sibling-map entries as columns not present in the slice. A no-op for every other
-// context, where getColumn already returns exactly the columns the evolutions name.
-func narrowEvolutionsToColumns(key *telemetrytypes.TelemetryFieldKey, cols []*schema.Column) *telemetrytypes.TelemetryFieldKey {
-	if len(key.Evolutions) == 0 {
-		return key
-	}
-	allowed := make(map[string]struct{}, len(cols))
-	for _, c := range cols {
-		allowed[c.Name] = struct{}{}
-	}
-	filtered := make([]*telemetrytypes.EvolutionEntry, 0, len(key.Evolutions))
-	for _, e := range key.Evolutions {
-		if e == nil {
-			continue
-		}
-		if _, ok := allowed[e.ColumnName]; ok {
-			filtered = append(filtered, e)
-		}
-	}
-	if len(filtered) == len(key.Evolutions) {
-		return key
-	}
-	narrowed := *key
-	narrowed.Evolutions = filtered
-	return &narrowed
-}
-
-// attributeJSONCast returns the ClickHouse cast target for a span attribute read from the
-// JSON column. String (and data-type-unspecified) casts to non-nullable String so an absent
-// path folds to ” the way the Map column's default does, keeping negative-operator parity;
-// numeric and bool cast to Nullable so an absent or wrong-typed path reads NULL instead of a
-// spurious 0/false. GROUP BY accepts these Nullable scalar casts (unlike a raw Dynamic).
-func attributeJSONCast(dataType telemetrytypes.FieldDataType) string {
-	switch dataType {
-	case telemetrytypes.FieldDataTypeInt64,
-		telemetrytypes.FieldDataTypeFloat64,
-		telemetrytypes.FieldDataTypeNumber,
-		telemetrytypes.FieldDataTypeBool:
-		return fmt.Sprintf("Nullable(%s)", telemetrytypes.MappingFieldDataTypeToJSONDataType[dataType].StringValue())
-	default:
-		return "String"
-	}
-}
-
 // upgradeToFamilies swaps single-member candidates for their family when the
 // metadata map proves membership. Candidate order and every non-family
 // candidate stay exactly as the legacy flow produced them; sibling candidates
@@ -605,7 +516,6 @@ func (m *fieldMapper) columnIsTemporal(ctx context.Context, startNs, endNs uint6
 	if err != nil {
 		return false, err
 	}
-	key = narrowEvolutionsToColumns(key, columns)
 	newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
 	if err != nil {
 		return false, err
@@ -728,7 +638,6 @@ func (m *fieldMapper) ExistsFor(
 	if err != nil {
 		return "", err
 	}
-	key = narrowEvolutionsToColumns(key, columns)
 	fieldExpression, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, key)
 	if err != nil {
 		return "", err

pkg/telemetryschema/tracestelemetryschema/field_mapper_attributes_json_test.go

@@ -1,254 +0,0 @@
-package tracestelemetryschema
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
-	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
-	"github.com/huandu/go-sqlbuilder"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-var (
-	attrJSONRelease = time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
-
-	attrWindowBefore   = [2]uint64{tsNano(2024, 1), tsNano(2024, 6)}
-	attrWindowAfter    = [2]uint64{tsNano(2025, 6), tsNano(2025, 7)}
-	attrWindowStraddle = [2]uint64{tsNano(2024, 6), tsNano(2025, 6)}
-)
-
-func tsNano(y int, m time.Month) uint64 {
-	return uint64(time.Date(y, m, 1, 0, 0, 0, 0, time.UTC).UnixNano())
-}
-
-func attrKey(name string, dt telemetrytypes.FieldDataType, evo []*telemetrytypes.EvolutionEntry) telemetrytypes.TelemetryFieldKey {
-	return telemetrytypes.TelemetryFieldKey{
-		Name:          name,
-		FieldContext:  telemetrytypes.FieldContextAttribute,
-		FieldDataType: dt,
-		Evolutions:    evo,
-	}
-}
-
-// TestFieldForAttributeJSONEvolution asserts the value expression across the rollout window:
-// before release the legacy Map lookup (byte-for-byte today), after release the type-aware JSON
-// cast, straddling a dual-read multiIf with the JSON column first.
-func TestFieldForAttributeJSONEvolution(t *testing.T) {
-	ctx := context.Background()
-	fm := NewFieldMapper(flaggertest.New(t))
-	evo := MockAttributeEvolutionData(attrJSONRelease)
-
-	testCases := []struct {
-		name     string
-		dataType telemetrytypes.FieldDataType
-		window   [2]uint64
-		expected string
-	}{
-		{"string before -> map", telemetrytypes.FieldDataTypeString, attrWindowBefore, "attributes_string['user.id']"},
-		{"string after -> json", telemetrytypes.FieldDataTypeString, attrWindowAfter, "attributes.`user.id`::String"},
-		{"string straddle -> dual", telemetrytypes.FieldDataTypeString, attrWindowStraddle, "multiIf(attributes.`user.id` IS NOT NULL, attributes.`user.id`::String, mapContains(attributes_string, 'user.id'), attributes_string['user.id'], NULL)"},
-		{"number before -> map", telemetrytypes.FieldDataTypeNumber, attrWindowBefore, "attributes_number['user.id']"},
-		{"number after -> json", telemetrytypes.FieldDataTypeNumber, attrWindowAfter, "attributes.`user.id`::Nullable(Float64)"},
-		{"number straddle -> dual", telemetrytypes.FieldDataTypeNumber, attrWindowStraddle, "multiIf(attributes.`user.id` IS NOT NULL, attributes.`user.id`::Nullable(Float64), mapContains(attributes_number, 'user.id'), attributes_number['user.id'], NULL)"},
-		{"int64 after -> json", telemetrytypes.FieldDataTypeInt64, attrWindowAfter, "attributes.`user.id`::Nullable(Int64)"},
-		{"bool before -> map", telemetrytypes.FieldDataTypeBool, attrWindowBefore, "attributes_bool['user.id']"},
-		{"bool after -> json", telemetrytypes.FieldDataTypeBool, attrWindowAfter, "attributes.`user.id`::Nullable(Bool)"},
-		{"bool straddle -> dual", telemetrytypes.FieldDataTypeBool, attrWindowStraddle, "multiIf(attributes.`user.id` IS NOT NULL, attributes.`user.id`::Nullable(Bool), mapContains(attributes_bool, 'user.id'), attributes_bool['user.id'], NULL)"},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			key := attrKey("user.id", tc.dataType, evo)
-			got, err := fm.FieldFor(ctx, valuer.UUID{}, tc.window[0], tc.window[1], &key)
-			require.NoError(t, err)
-			assert.Equal(t, tc.expected, got)
-		})
-	}
-}
-
-// TestFieldForAttributeNoEvolutionParity proves the JSON column is untouched until the evolution
-// entry is registered: a key with no evolutions resolves to the Map column for every window.
-func TestFieldForAttributeNoEvolutionParity(t *testing.T) {
-	ctx := context.Background()
-	fm := NewFieldMapper(flaggertest.New(t))
-
-	for _, dt := range []struct {
-		dataType telemetrytypes.FieldDataType
-		expected string
-	}{
-		{telemetrytypes.FieldDataTypeString, "attributes_string['user.id']"},
-		{telemetrytypes.FieldDataTypeNumber, "attributes_number['user.id']"},
-		{telemetrytypes.FieldDataTypeBool, "attributes_bool['user.id']"},
-	} {
-		key := attrKey("user.id", dt.dataType, nil)
-		got, err := fm.FieldFor(ctx, valuer.UUID{}, attrWindowAfter[0], attrWindowAfter[1], &key)
-		require.NoError(t, err)
-		assert.Equal(t, dt.expected, got, "no evolution entry must keep the Map path")
-	}
-}
-
-// TestConditionForAttributeJSON asserts the emitted WHERE fragment per operator against the JSON
-// column (window fully after release). Positive operators carry the raw-path existence guard;
-// numeric comparisons keep numeric semantics; existence never tests the ::String cast.
-func TestConditionForAttributeJSON(t *testing.T) {
-	ctx := context.Background()
-	fm := NewFieldMapper(flaggertest.New(t))
-	cb := NewConditionBuilder(fm, flaggertest.New(t))
-	evo := MockAttributeEvolutionData(attrJSONRelease)
-
-	testCases := []struct {
-		name     string
-		key      telemetrytypes.TelemetryFieldKey
-		operator qbtypes.FilterOperator
-		value    any
-		expected string
-	}{
-		{
-			name:     "equal string",
-			key:      attrKey("user.id", telemetrytypes.FieldDataTypeString, evo),
-			operator: qbtypes.FilterOperatorEqual, value: "admin",
-			expected: "(attributes.`user.id`::String = ? AND attributes.`user.id` IS NOT NULL)",
-		},
-		{
-			name:     "not equal string has no exists guard",
-			key:      attrKey("user.id", telemetrytypes.FieldDataTypeString, evo),
-			operator: qbtypes.FilterOperatorNotEqual, value: "admin",
-			expected: "attributes.`user.id`::String <> ?",
-		},
-		{
-			name:     "greater than number",
-			key:      attrKey("http.status_code", telemetrytypes.FieldDataTypeInt64, evo),
-			operator: qbtypes.FilterOperatorGreaterThan, value: float64(200),
-			expected: "toFloat64(attributes.`http.status_code`::Nullable(Int64)) > ?",
-		},
-		{
-			name:     "ilike string",
-			key:      attrKey("user.id", telemetrytypes.FieldDataTypeString, evo),
-			operator: qbtypes.FilterOperatorILike, value: "%adm%",
-			expected: "LOWER(attributes.`user.id`::String) LIKE LOWER(?)",
-		},
-		{
-			name:     "exists uses raw path",
-			key:      attrKey("user.id", telemetrytypes.FieldDataTypeString, evo),
-			operator: qbtypes.FilterOperatorExists, value: nil,
-			expected: "attributes.`user.id` IS NOT NULL",
-		},
-		{
-			name:     "not exists uses raw path",
-			key:      attrKey("user.id", telemetrytypes.FieldDataTypeString, evo),
-			operator: qbtypes.FilterOperatorNotExists, value: nil,
-			expected: "attributes.`user.id` IS NULL",
-		},
-		{
-			name:     "in string",
-			key:      attrKey("user.id", telemetrytypes.FieldDataTypeString, evo),
-			operator: qbtypes.FilterOperatorIn, value: []any{"a", "b"},
-			expected: "((attributes.`user.id`::String = ? OR attributes.`user.id`::String = ?) AND attributes.`user.id` IS NOT NULL)",
-		},
-		{
-			name:     "not in string has no exists guard",
-			key:      attrKey("user.id", telemetrytypes.FieldDataTypeString, evo),
-			operator: qbtypes.FilterOperatorNotIn, value: []any{"a", "b"},
-			expected: "(attributes.`user.id`::String <> ? AND attributes.`user.id`::String <> ?)",
-		},
-		{
-			name:     "between number",
-			key:      attrKey("latency", telemetrytypes.FieldDataTypeNumber, evo),
-			operator: qbtypes.FilterOperatorBetween, value: []any{float64(1), float64(9)},
-			expected: "toFloat64(attributes.`latency`::Nullable(Float64)) BETWEEN ? AND ?",
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			sb := sqlbuilder.NewSelectBuilder()
-			conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, attrWindowAfter[0], attrWindowAfter[1], &tc.key,
-				map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, tc.value, sb)
-			require.NoError(t, err)
-			sb.Where(conds...)
-			sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
-			assert.Contains(t, sql, tc.expected)
-		})
-	}
-}
-
-// TestConditionForAttributeJSONNotExistsDualRead covers NOT EXISTS across both homes during the
-// dual-read window: it must AND the JSON IS NULL with NOT mapContains so a row present in either
-// home is excluded (De Morgan), including rows that predate the JSON column.
-func TestConditionForAttributeJSONNotExistsDualRead(t *testing.T) {
-	ctx := context.Background()
-	fm := NewFieldMapper(flaggertest.New(t))
-	cb := NewConditionBuilder(fm, flaggertest.New(t))
-	evo := MockAttributeEvolutionData(attrJSONRelease)
-
-	key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
-	sb := sqlbuilder.NewSelectBuilder()
-	conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, attrWindowStraddle[0], attrWindowStraddle[1], &key,
-		map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotExists, nil, sb)
-	require.NoError(t, err)
-	sb.Where(conds...)
-	sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
-	// the value multiIf resolves the row's home; NOT EXISTS negates the whole thing to IS NULL
-	assert.Contains(t, sql, "IS NULL")
-	assert.Contains(t, sql, "attributes.`user.id` IS NOT NULL")
-	assert.Contains(t, sql, "mapContains(attributes_string, 'user.id')")
-}
-
-// TestColumnExpressionForAttributeJSON covers group-by (coerced to String) and aggregation
-// (coerced to Float64) over a JSON attribute after release: both are exists-guarded so an absent
-// path is NULL rather than a spurious ”/0, and the numeric branch keeps its toFloat64 coercion.
-func TestColumnExpressionForAttributeJSON(t *testing.T) {
-	ctx := context.Background()
-	fm := NewFieldMapper(flaggertest.New(t))
-	evo := MockAttributeEvolutionData(attrJSONRelease)
-
-	t.Run("group by string", func(t *testing.T) {
-		key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
-		got, err := fm.ColumnExpressionFor(ctx, valuer.UUID{}, attrWindowAfter[0], attrWindowAfter[1], &key, telemetrytypes.FieldDataTypeString, nil)
-		require.NoError(t, err)
-		assert.Equal(t, "multiIf(attributes.`user.id` IS NOT NULL, attributes.`user.id`::String, NULL)", got)
-	})
-
-	t.Run("aggregation numeric", func(t *testing.T) {
-		key := attrKey("latency", telemetrytypes.FieldDataTypeNumber, evo)
-		got, err := fm.ColumnExpressionFor(ctx, valuer.UUID{}, attrWindowAfter[0], attrWindowAfter[1], &key, telemetrytypes.FieldDataTypeFloat64, nil)
-		require.NoError(t, err)
-		assert.Equal(t, "multiIf(attributes.`latency` IS NOT NULL, toFloat64(attributes.`latency`::Nullable(Float64)), NULL)", got)
-	})
-}
-
-// TestAttributeJSONNoAmbiguityWarning guards against a visible regression: the JSON column is a
-// second physical home for the same logical field, not a second logical field, so a plain
-// attribute filter must not emit the "ambiguous key" warning.
-func TestAttributeJSONNoAmbiguityWarning(t *testing.T) {
-	ctx := context.Background()
-	fm := NewFieldMapper(flaggertest.New(t))
-	cb := NewConditionBuilder(fm, flaggertest.New(t))
-	evo := MockAttributeEvolutionData(attrJSONRelease)
-
-	key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
-	sb := sqlbuilder.NewSelectBuilder()
-	_, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, attrWindowAfter[0], attrWindowAfter[1], &key,
-		map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "x", sb)
-	require.NoError(t, err)
-	assert.Empty(t, warnings, "a plain attribute filter must not emit an ambiguity warning")
-}
-
-// TestColumnForUnspecifiedAttributeNoBranchFlip pins the branch-flip decision: a
-// data-type-unspecified attribute key resolves to no column (even with the evolution present), so
-// bare attribute keys keep taking the legacy CandidateKeys/synthesis path rather than becoming
-// metadata-first resolvable.
-func TestColumnForUnspecifiedAttributeNoBranchFlip(t *testing.T) {
-	ctx := context.Background()
-	fm := NewFieldMapper(flaggertest.New(t))
-	evo := MockAttributeEvolutionData(attrJSONRelease)
-
-	key := attrKey("user.id", telemetrytypes.FieldDataTypeUnspecified, evo)
-	_, err := fm.ColumnFor(ctx, valuer.UUID{}, attrWindowAfter[0], attrWindowAfter[1], &key)
-	assert.ErrorIs(t, err, qbtypes.ErrColumnNotFound)
-}

pkg/telemetryschema/tracestelemetryschema/test_data.go

@@ -154,29 +154,6 @@ func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
 	return keysMap
 }
 
-// MockAttributeEvolutionData returns the attribute-context evolution timeline: the three legacy
-// map columns at epoch 0 and the JSON `attributes` column released at releaseTime. Every entry
-// is field_name "__all__", so a typed attribute key carries all four; getColumn keeps only its
-// own map plus the JSON column and narrowEvolutionsToColumns drops the rest before selection.
-func MockAttributeEvolutionData(releaseTime time.Time) []*telemetrytypes.EvolutionEntry {
-	entry := func(col, typ string, rt time.Time) *telemetrytypes.EvolutionEntry {
-		return &telemetrytypes.EvolutionEntry{
-			Signal:       telemetrytypes.SignalTraces,
-			ColumnName:   col,
-			ColumnType:   typ,
-			FieldContext: telemetrytypes.FieldContextAttribute,
-			FieldName:    "__all__",
-			ReleaseTime:  rt,
-		}
-	}
-	return []*telemetrytypes.EvolutionEntry{
-		entry("attributes_string", "Map(LowCardinality(String), String)", time.Unix(0, 0)),
-		entry("attributes_number", "Map(LowCardinality(String), Float64)", time.Unix(0, 0)),
-		entry("attributes_bool", "Map(LowCardinality(String), Bool)", time.Unix(0, 0)),
-		entry("attributes", "JSON()", releaseTime),
-	}
-}
-
 // MockEvolutionData returns the canonical resource-column evolution timeline used in tests:
 // the legacy resources_string map at epoch 0 and the JSON resource column released at releaseTime.
 func MockEvolutionData(releaseTime time.Time) []*telemetrytypes.EvolutionEntry {

pkg/types/aiobservabilitytypes/field.go

@@ -15,11 +15,12 @@ type PostableFieldKeysParams struct {
 	Limit          int                          `query:"limit"`
 }
 
-// existingQuery is unsupported until the computed per-trace aggregates it may
-// reference can be narrowed on.
+// existingQuery may reference the computed per-trace aggregates, which are
+// never ingested; those filters are stripped before narrowing values.
 type PostableFieldValueParams struct {
 	PostableFieldKeysParams
-	Name string `query:"name"`
+	Name          string `query:"name"`
+	ExistingQuery string `query:"existingQuery"`
 }
 
 func NewFieldKeySelectorFromPostableFieldKeysParams(params PostableFieldKeysParams) *telemetrytypes.FieldKeySelector {
@@ -30,6 +31,7 @@ func NewFieldValueSelectorFromPostableFieldValueParams(params PostableFieldValue
 	return telemetrytypes.NewFieldValueSelectorFromPostableFieldValueParams(telemetrytypes.PostableFieldValueParams{
 		PostableFieldKeysParams: params.telemetryParams(),
 		Name:                    params.Name,
+		ExistingQuery:           params.ExistingQuery,
 	})
 }
 

pkg/types/aiobservabilitytypes/keys.go

@@ -1,5 +1,7 @@
 package aiobservabilitytypes
 
+import "strings"
+
 // OpenTelemetry gen_ai semantic-convention attribute keys. Single source of truth
 // shared by the AI query builder and the LLM pricing pipeline.
 const (
@@ -18,6 +20,20 @@ const (
 	GenAIOutputMessages = "gen_ai.output.messages"
 )
 
+// GenAISpanGateKeys mark a span as gen_ai: an LLM call, a tool call, or an
+// agent span. A trace belongs to the AI explorer when any span carries one.
+var GenAISpanGateKeys = []string{GenAIRequestModel, GenAIToolName, GenAIAgentName}
+
+// GenAISpanFilterExpression renders the gate as a query-builder filter
+// expression: each gate key ORed on EXISTS.
+func GenAISpanFilterExpression() string {
+	exprs := make([]string, 0, len(GenAISpanGateKeys))
+	for _, key := range GenAISpanGateKeys {
+		exprs = append(exprs, key+" EXISTS")
+	}
+	return strings.Join(exprs, " OR ")
+}
+
 // Per-span costs the SigNoz LLM pricing processor attaches; not OTel semconv.
 const (
 	SignozGenAICostInput      = "_signoz.gen_ai.cost_input"

pkg/types/coretypes/kind.go

@@ -71,7 +71,6 @@ func (name Kind) Enum() []any {
 		KindFactorAPIKey,
 		KindLicense,
 		KindSubscription,
-		KindDeploymentHost,
 		KindLogs,
 		KindTraces,
 		KindMetrics,

pkg/types/coretypes/registry_kind.go

@@ -31,7 +31,6 @@ var Kinds = []Kind{
 	KindFactorAPIKey,
 	KindLicense,
 	KindSubscription,
-	KindDeploymentHost,
 	KindLogs,
 	KindTraces,
 	KindMetrics,
@@ -72,7 +71,6 @@ var (
 	KindFactorAPIKey                 = MustNewKind("factor-api-key")
 	KindLicense                      = MustNewKind("license")
 	KindSubscription                 = MustNewKind("subscription")
-	KindDeploymentHost               = MustNewKind("deployment-host")
 	KindLogs                         = MustNewKind("logs")
 	KindTraces                       = MustNewKind("traces")
 	KindMetrics                      = MustNewKind("metrics")

pkg/types/coretypes/registry_managed_role.go

@@ -191,9 +191,6 @@ var ManagedRoleToTransactions = map[string][]Transaction{
 		{Verb: VerbRead, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindTTLSetting}, WildCardSelectorString)},
 		{Verb: VerbUpdate, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindTTLSetting}, WildCardSelectorString)},
 		{Verb: VerbList, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindTTLSetting}, WildCardSelectorString)},
-		// deployment-host — admin updates, viewer lists
-		{Verb: VerbList, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindDeploymentHost}, WildCardSelectorString)},
-		{Verb: VerbUpdate, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindDeploymentHost}, WildCardSelectorString)},
 		// user-preference — every authenticated user can read+update their own
 		{Verb: VerbRead, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindUserPreference}, WildCardSelectorString)},
 		{Verb: VerbUpdate, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindUserPreference}, WildCardSelectorString)},
@@ -286,8 +283,6 @@ var ManagedRoleToTransactions = map[string][]Transaction{
 		// ttl-setting — read only (admin updates)
 		{Verb: VerbRead, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindTTLSetting}, WildCardSelectorString)},
 		{Verb: VerbList, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindTTLSetting}, WildCardSelectorString)},
-		// deployment-host — list only (admin updates)
-		{Verb: VerbList, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindDeploymentHost}, WildCardSelectorString)},
 		// user-preference — every authenticated user can read+update their own
 		{Verb: VerbRead, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindUserPreference}, WildCardSelectorString)},
 		{Verb: VerbUpdate, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindUserPreference}, WildCardSelectorString)},
@@ -346,8 +341,6 @@ var ManagedRoleToTransactions = map[string][]Transaction{
 		// ttl-setting — read only
 		{Verb: VerbRead, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindTTLSetting}, WildCardSelectorString)},
 		{Verb: VerbList, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindTTLSetting}, WildCardSelectorString)},
-		// deployment-host — list only
-		{Verb: VerbList, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindDeploymentHost}, WildCardSelectorString)},
 		// user-preference — every authenticated user can read+update their own
 		{Verb: VerbRead, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindUserPreference}, WildCardSelectorString)},
 		{Verb: VerbUpdate, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindUserPreference}, WildCardSelectorString)},

pkg/types/coretypes/registry_resource.go

@@ -31,7 +31,6 @@ var Resources = []Resource{
 	ResourceMetaResourceFactorAPIKey,
 	ResourceMetaResourceLicense,
 	ResourceMetaResourceSubscription,
-	ResourceMetaResourceDeploymentHost,
 	ResourceTelemetryResourceLogs,
 	ResourceTelemetryResourceTraces,
 	ResourceTelemetryResourceMetrics,
@@ -72,7 +71,6 @@ var (
 	ResourceMetaResourceFactorAPIKey                     = NewResourceMetaResource(KindFactorAPIKey, VerbCreate, VerbList, VerbRead, VerbUpdate, VerbDelete)
 	ResourceMetaResourceLicense                          = NewResourceMetaResource(KindLicense)
 	ResourceMetaResourceSubscription                     = NewResourceMetaResource(KindSubscription)
-	ResourceMetaResourceDeploymentHost                   = NewResourceMetaResource(KindDeploymentHost, VerbList, VerbUpdate)
 	ResourceTelemetryResourceLogs                        = NewResourceTelemetryResource(KindLogs)
 	ResourceTelemetryResourceTraces                      = NewResourceTelemetryResource(KindTraces)
 	ResourceTelemetryResourceMetrics                     = NewResourceTelemetryResource(KindMetrics)

tests/integration/testdata/role/managed_role_transactions.json

@@ -295,14 +295,6 @@
         "list"
       ]
     },
-    {
-      "type": "metaresource",
-      "kind": "deployment-host",
-      "verbs": [
-        "list",
-        "update"
-      ]
-    },
     {
       "type": "metaresource",
       "kind": "user-preference",
@@ -506,13 +498,6 @@
         "list"
       ]
     },
-    {
-      "type": "metaresource",
-      "kind": "deployment-host",
-      "verbs": [
-        "list"
-      ]
-    },
     {
       "type": "metaresource",
       "kind": "user-preference",
@@ -669,13 +654,6 @@
         "list"
       ]
     },
-    {
-      "type": "metaresource",
-      "kind": "deployment-host",
-      "verbs": [
-        "list"
-      ]
-    },
     {
       "type": "metaresource",
       "kind": "user-preference",

tests/integration/tests/querierai/03_ai_metadata.py

@@ -2,10 +2,12 @@ from collections.abc import Callable
 from datetime import UTC, datetime
 from http import HTTPStatus
 
+import pytest
+
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.metadata import get_field_keys, get_field_values
-from fixtures.querierai import ai_trace
+from fixtures.metadata import AttributesMetadata, get_field_keys, get_field_values
+from fixtures.querierai import ai_trace, ai_trace_mixed_spans
 from fixtures.traces import Traces
 
 AI_KEYS_PATH = "/api/v1/ai_observability/fields/keys"
@@ -106,20 +108,94 @@ def test_ai_field_values_suggests_ingested_attribute_values(
     assert values["stringValues"] == ["gpt-it-values"], values
 
 
-def test_ai_field_values_reject_existing_query(
+@pytest.mark.parametrize(
+    "existing_query,search_text,expected",
+    [
+        pytest.param(None, "", {"ai-rel-a", "ai-rel-b", "ai-rel-c"}, id="no_query_scopes_to_gen_ai_spans"),
+        pytest.param("gen_ai.user.id = 'alice'", "", {"ai-rel-a"}, id="span_filter_narrows_under_the_gate"),
+        pytest.param("llm_call_count > 0", "", {"ai-rel-a", "ai-rel-b", "ai-rel-c"}, id="pure_trace_aggregate_filter_is_stripped"),
+        pytest.param("llm_call_count > 0 AND gen_ai.user.id = 'alice'", "", {"ai-rel-a"}, id="mixed_filter_keeps_only_the_span_part"),
+        pytest.param(
+            "llm_call_count > 0 OR gen_ai.user.id = 'alice'",
+            "",
+            {"ai-rel-a", "ai-rel-b", "ai-rel-c"},
+            id="class_mixing_or_drops_the_filter_not_the_request",
+        ),
+        pytest.param(
+            "gen_ai.user.id = ",
+            "",
+            {"ai-rel-a", "ai-rel-b", "ai-rel-c"},
+            id="unparseable_filter_falls_back_to_the_gate",
+        ),
+        pytest.param(None, "ai-rel-a", {"ai-rel-a"}, id="search_text_narrows_related_values"),
+        # http.request.method lives on the root span's metadata row, gen_ai.* on
+        # the LLM/tool/agent rows; rows are per span-shape, so the gate AND a
+        # cross-span attribute filter can match no single row
+        pytest.param("http.request.method = 'POST'", "", set(), id="cross_span_attribute_filter_matches_no_row"),
+    ],
+)
+def test_ai_field_values_related_values(
     signoz: types.SigNoz,
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+    insert_attributes_metadata: Callable[[list[AttributesMetadata]], None],
+    existing_query: str | None,
+    search_text: str,
+    expected: set[str],
 ) -> None:
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+
+    # existingQuery key resolution reads the trace keys tables, not
+    # attributes_metadata; a mixed trace registers the gate keys (model/tool/
+    # agent) plus gen_ai.user.id and http.request.method
+    insert_traces(ai_trace_mixed_spans(now=now, service="ai-rel-a", user="alice"))
+
+    # related values are served from attributes_metadata; one row per gate key,
+    # the traces row without any gate attribute and the logs row (wrong
+    # data_source, gate attribute present) must never surface
+    insert_attributes_metadata(
+        [
+            AttributesMetadata(
+                data_source="traces",
+                resource_attributes={"service.name": "ai-rel-a"},
+                attributes={"gen_ai.request.model": "gpt-rel", "gen_ai.user.id": "alice"},
+            ),
+            AttributesMetadata(
+                data_source="traces",
+                resource_attributes={"service.name": "ai-rel-b"},
+                attributes={"gen_ai.tool.name": "get_weather", "gen_ai.user.id": "bob"},
+            ),
+            AttributesMetadata(
+                data_source="traces",
+                resource_attributes={"service.name": "ai-rel-c"},
+                attributes={"gen_ai.agent.name": "chat-agent"},
+            ),
+            AttributesMetadata(
+                data_source="traces",
+                resource_attributes={"service.name": "plain-rel"},
+                attributes={"http.request.method": "POST"},
+            ),
+            AttributesMetadata(
+                data_source="logs",
+                resource_attributes={"service.name": "ai-rel-logs"},
+                attributes={"gen_ai.request.model": "gpt-rel"},
+            ),
+        ]
+    )
+
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    response = get_field_values(
-        signoz,
-        token,
-        {"name": "gen_ai.request.model", "existingQuery": "service.name = 'ai-it-values'"},
-        AI_VALUES_PATH,
-    )
-    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    params = {"name": "service.name", "searchText": search_text}
+    if existing_query is not None:
+        params["existingQuery"] = existing_query
+
+    response = get_field_values(signoz, token, params, AI_VALUES_PATH)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert response.json()["status"] == "success"
+
+    related = response.json()["data"]["values"].get("relatedValues") or []
+    assert set(related) == expected, related
 
 
 def test_ai_field_values_of_computed_aggregate_are_empty(