fix: support for related values in ai field values

docs/api/openapi.yml

@@ -9222,6 +9222,10 @@ paths:
         name: name
         schema:
           type: string
+      - in: query
+        name: existingQuery
+        schema:
+          type: string
       responses:
         "200":
           content:

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -10248,6 +10248,11 @@ export type GetAIObservabilityFieldsValuesParams = {
 	 * @description undefined
 	 */
 	name?: string;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	existingQuery?: string;
 };
 
 export type GetAIObservabilityFieldsValues200 = {

pkg/modules/aiobservability/implaiobservability/handler.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/http/binding"
 	"github.com/SigNoz/signoz/pkg/http/render"
 	"github.com/SigNoz/signoz/pkg/modules/aiobservability"
@@ -66,13 +65,6 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques
 	ctx, cancel := context.WithTimeout(req.Context(), 10*time.Second)
 	defer cancel()
 
-	// binding ignores query params the struct does not declare, so an unsupported
-	// existingQuery would silently return values it did not narrow
-	if req.URL.Query().Has("existingQuery") {
-		render.Error(rw, errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "existingQuery is not supported"))
-		return
-	}
-
 	var params aiobservabilitytypes.PostableFieldValueParams
 	if err := binding.Query.BindQuery(req.URL.Query(), &params); err != nil {
 		render.Error(rw, err)
@@ -84,18 +76,29 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques
 		render.Error(rw, err)
 		return
 	}
+	orgID := valuer.MustNewUUID(claims.OrgID)
 
+	params.ExistingQuery = aitelemetryschema.ScopedExistingQuery(params.ExistingQuery)
 	fieldValueSelector := aiobservabilitytypes.NewFieldValueSelectorFromPostableFieldValueParams(params)
 
 	values := &telemetrytypes.TelemetryFieldValues{}
 	complete := true
 	// the trace context names the computed per-trace aggregates, which are never ingested
 	if fieldValueSelector.FieldContext != telemetrytypes.FieldContextTrace {
-		values, complete, err = handler.telemetryMetadataStore.GetAllValues(ctx, valuer.MustNewUUID(claims.OrgID), fieldValueSelector)
+		values, complete, err = handler.telemetryMetadataStore.GetAllValues(ctx, orgID, fieldValueSelector)
 		if err != nil {
 			render.Error(rw, err)
 			return
 		}
+
+		// related values are best-effort: on failure the plain values still serve
+		// the filter bar
+		relatedValues, relatedComplete, err := handler.telemetryMetadataStore.GetRelatedValues(ctx, orgID, fieldValueSelector)
+		if err != nil {
+			relatedValues = []string{}
+		}
+		values.RelatedValues = relatedValues
+		complete = complete && relatedComplete
 	}
 
 	render.Success(rw, http.StatusOK, &telemetrytypes.GettableFieldValues{

pkg/statementbuilder/aistatementbuilder/statement_builder.go

@@ -1,8 +1,6 @@
 package aistatementbuilder
 
 import (
-	"strings"
-
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/statementbuilder"
@@ -27,11 +25,8 @@ func NewFactory(
 // Scope describes gen_ai for the scoped trace builder: an AI trace has >=1 gen_ai
 // LLM, tool, or agent span, and its list adds AI/LLM per-trace metrics.
 func Scope() scopedtraces.TraceScope {
-	gateKeyNames := []string{aiobservabilitytypes.GenAIRequestModel, aiobservabilitytypes.GenAIToolName, aiobservabilitytypes.GenAIAgentName}
-	gateExprs := make([]string, 0, len(gateKeyNames))
-	gateKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(gateKeyNames))
-	for _, name := range gateKeyNames {
-		gateExprs = append(gateExprs, name+" EXISTS")
+	gateKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(aiobservabilitytypes.GenAISpanGateKeys))
+	for _, name := range aiobservabilitytypes.GenAISpanGateKeys {
 		gateKeys = append(gateKeys, &telemetrytypes.TelemetryFieldKey{
 			Name:         name,
 			Signal:       telemetrytypes.SignalTraces,
@@ -79,7 +74,7 @@ func Scope() scopedtraces.TraceScope {
 	}
 
 	return scopedtraces.TraceScope{
-		FilterExpression:  strings.Join(gateExprs, " OR "),
+		FilterExpression:  aiobservabilitytypes.GenAISpanFilterExpression(),
 		FieldKeys:         gateKeys,
 		Columns:           columns,
 		DefaultOrderAlias: "last_activity_time",

pkg/telemetryschema/aitelemetryschema/existing_query.go

@@ -0,0 +1,36 @@
+package aitelemetryschema
+
+import (
+	"github.com/SigNoz/signoz/pkg/querybuilder"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
+)
+
+var (
+	traceAggregateNames = func() map[string]struct{} {
+		names := make(map[string]struct{}, len(TraceAggregateFields))
+		for name := range TraceAggregateFields {
+			names[name] = struct{}{}
+		}
+		return names
+	}()
+
+	genAISpanGate = "(" + aiobservabilitytypes.GenAISpanFilterExpression() + ")"
+)
+
+// ScopedExistingQuery narrows value suggestions to gen_ai spans: the caller's
+// filter minus its per-trace aggregate atoms (never ingested, so nothing can
+// narrow on them), ANDed with the gen_ai span gate. An unparseable filter is
+// dropped, matching how the metadata store treats it downstream.
+func ScopedExistingQuery(existingQuery string) string {
+	spanExpr := ""
+	if existingQuery != "" {
+		if expr, _, err := querybuilder.SplitFilterForAggregates(existingQuery, traceAggregateNames); err == nil {
+			spanExpr = expr
+		}
+	}
+
+	if spanExpr == "" {
+		return genAISpanGate
+	}
+	return genAISpanGate + " AND (" + spanExpr + ")"
+}

pkg/telemetryschema/aitelemetryschema/existing_query_test.go

@@ -0,0 +1,94 @@
+package aitelemetryschema
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestScopedExistingQuery(t *testing.T) {
+	gate := "(gen_ai.request.model EXISTS OR gen_ai.tool.name EXISTS OR gen_ai.agent.name EXISTS)"
+
+	testCases := []struct {
+		name          string
+		existingQuery string
+		expected      string
+	}{
+		{
+			name:          "empty query returns the gate alone",
+			existingQuery: "",
+			expected:      gate,
+		},
+		{
+			name:          "span filter is preserved under the gate",
+			existingQuery: "service.name = 'checkout'",
+			expected:      gate + " AND (service.name = 'checkout')",
+		},
+		{
+			name:          "trace aggregate filter is stripped",
+			existingQuery: "llm_call_count > 5",
+			expected:      gate,
+		},
+		{
+			name:          "mixed filter keeps only the span part",
+			existingQuery: "llm_call_count > 5 AND gen_ai.request.model = 'gpt-4'",
+			expected:      gate + " AND (gen_ai.request.model = 'gpt-4')",
+		},
+		{
+			name:          "trace context filter is stripped",
+			existingQuery: "trace.total_tokens > 100 AND service.name = 'checkout'",
+			expected:      gate + " AND (service.name = 'checkout')",
+		},
+		{
+			name:          "unparseable filter is dropped",
+			existingQuery: "service.name = ",
+			expected:      gate,
+		},
+		{
+			name:          "multiple span conditions survive as one AND chain",
+			existingQuery: "service.name = 'checkout' AND gen_ai.request.model = 'gpt-4' AND llm_call_count > 5",
+			expected:      gate + " AND (service.name = 'checkout' AND gen_ai.request.model = 'gpt-4')",
+		},
+		{
+			name:          "span OR group is kept whole and parenthesized against the AND join",
+			existingQuery: "service.name = 'a' OR service.name = 'b'",
+			expected:      gate + " AND ((service.name = 'a' OR service.name = 'b'))",
+		},
+		{
+			name:          "parenthesized span OR group ANDed with an aggregate keeps only the group",
+			existingQuery: "(service.name = 'a' OR service.name = 'b') AND llm_call_count > 5",
+			expected:      gate + " AND ((service.name = 'a' OR service.name = 'b'))",
+		},
+		{
+			name:          "OR group of trace aggregates is stripped whole",
+			existingQuery: "llm_call_count > 5 OR total_tokens > 100",
+			expected:      gate,
+		},
+		{
+			name:          "OR mixing aggregate and span atoms drops the whole filter",
+			existingQuery: "llm_call_count > 5 OR service.name = 'checkout'",
+			expected:      gate,
+		},
+		{
+			name:          "parenthesized AND group is split, not routed whole",
+			existingQuery: "(llm_call_count > 5 AND service.name = 'checkout') AND gen_ai.request.model = 'gpt-4'",
+			expected:      gate + " AND (service.name = 'checkout' AND gen_ai.request.model = 'gpt-4')",
+		},
+		{
+			name:          "NOT over an aggregate group is stripped",
+			existingQuery: "NOT (llm_call_count > 5) AND service.name = 'checkout'",
+			expected:      gate + " AND (service.name = 'checkout')",
+		},
+		{
+			name:          "NOT over a span group is kept",
+			existingQuery: "NOT (service.name = 'checkout')",
+			expected:      gate + " AND (NOT (service.name = 'checkout'))",
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			assert.Equal(t, testCase.expected, ScopedExistingQuery(testCase.existingQuery))
+		})
+	}
+}

pkg/types/aiobservabilitytypes/field.go

@@ -15,11 +15,12 @@ type PostableFieldKeysParams struct {
 	Limit          int                          `query:"limit"`
 }
 
-// existingQuery is unsupported until the computed per-trace aggregates it may
-// reference can be narrowed on.
+// existingQuery may reference the computed per-trace aggregates, which are
+// never ingested; those filters are stripped before narrowing values.
 type PostableFieldValueParams struct {
 	PostableFieldKeysParams
-	Name string `query:"name"`
+	Name          string `query:"name"`
+	ExistingQuery string `query:"existingQuery"`
 }
 
 func NewFieldKeySelectorFromPostableFieldKeysParams(params PostableFieldKeysParams) *telemetrytypes.FieldKeySelector {
@@ -30,6 +31,7 @@ func NewFieldValueSelectorFromPostableFieldValueParams(params PostableFieldValue
 	return telemetrytypes.NewFieldValueSelectorFromPostableFieldValueParams(telemetrytypes.PostableFieldValueParams{
 		PostableFieldKeysParams: params.telemetryParams(),
 		Name:                    params.Name,
+		ExistingQuery:           params.ExistingQuery,
 	})
 }
 

pkg/types/aiobservabilitytypes/keys.go

@@ -1,5 +1,7 @@
 package aiobservabilitytypes
 
+import "strings"
+
 // OpenTelemetry gen_ai semantic-convention attribute keys. Single source of truth
 // shared by the AI query builder and the LLM pricing pipeline.
 const (
@@ -18,6 +20,20 @@ const (
 	GenAIOutputMessages = "gen_ai.output.messages"
 )
 
+// GenAISpanGateKeys mark a span as gen_ai: an LLM call, a tool call, or an
+// agent span. A trace belongs to the AI explorer when any span carries one.
+var GenAISpanGateKeys = []string{GenAIRequestModel, GenAIToolName, GenAIAgentName}
+
+// GenAISpanFilterExpression renders the gate as a query-builder filter
+// expression: each gate key ORed on EXISTS.
+func GenAISpanFilterExpression() string {
+	exprs := make([]string, 0, len(GenAISpanGateKeys))
+	for _, key := range GenAISpanGateKeys {
+		exprs = append(exprs, key+" EXISTS")
+	}
+	return strings.Join(exprs, " OR ")
+}
+
 // Per-span costs the SigNoz LLM pricing processor attaches; not OTel semconv.
 const (
 	SignozGenAICostInput      = "_signoz.gen_ai.cost_input"

tests/integration/tests/querierai/03_ai_metadata.py

@@ -2,10 +2,12 @@ from collections.abc import Callable
 from datetime import UTC, datetime
 from http import HTTPStatus
 
+import pytest
+
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.metadata import get_field_keys, get_field_values
-from fixtures.querierai import ai_trace
+from fixtures.metadata import AttributesMetadata, get_field_keys, get_field_values
+from fixtures.querierai import ai_trace, ai_trace_mixed_spans
 from fixtures.traces import Traces
 
 AI_KEYS_PATH = "/api/v1/ai_observability/fields/keys"
@@ -106,20 +108,94 @@ def test_ai_field_values_suggests_ingested_attribute_values(
     assert values["stringValues"] == ["gpt-it-values"], values
 
 
-def test_ai_field_values_reject_existing_query(
+@pytest.mark.parametrize(
+    "existing_query,search_text,expected",
+    [
+        pytest.param(None, "", {"ai-rel-a", "ai-rel-b", "ai-rel-c"}, id="no_query_scopes_to_gen_ai_spans"),
+        pytest.param("gen_ai.user.id = 'alice'", "", {"ai-rel-a"}, id="span_filter_narrows_under_the_gate"),
+        pytest.param("llm_call_count > 0", "", {"ai-rel-a", "ai-rel-b", "ai-rel-c"}, id="pure_trace_aggregate_filter_is_stripped"),
+        pytest.param("llm_call_count > 0 AND gen_ai.user.id = 'alice'", "", {"ai-rel-a"}, id="mixed_filter_keeps_only_the_span_part"),
+        pytest.param(
+            "llm_call_count > 0 OR gen_ai.user.id = 'alice'",
+            "",
+            {"ai-rel-a", "ai-rel-b", "ai-rel-c"},
+            id="class_mixing_or_drops_the_filter_not_the_request",
+        ),
+        pytest.param(
+            "gen_ai.user.id = ",
+            "",
+            {"ai-rel-a", "ai-rel-b", "ai-rel-c"},
+            id="unparseable_filter_falls_back_to_the_gate",
+        ),
+        pytest.param(None, "ai-rel-a", {"ai-rel-a"}, id="search_text_narrows_related_values"),
+        # http.request.method lives on the root span's metadata row, gen_ai.* on
+        # the LLM/tool/agent rows; rows are per span-shape, so the gate AND a
+        # cross-span attribute filter can match no single row
+        pytest.param("http.request.method = 'POST'", "", set(), id="cross_span_attribute_filter_matches_no_row"),
+    ],
+)
+def test_ai_field_values_related_values(
     signoz: types.SigNoz,
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+    insert_attributes_metadata: Callable[[list[AttributesMetadata]], None],
+    existing_query: str | None,
+    search_text: str,
+    expected: set[str],
 ) -> None:
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+
+    # existingQuery key resolution reads the trace keys tables, not
+    # attributes_metadata; a mixed trace registers the gate keys (model/tool/
+    # agent) plus gen_ai.user.id and http.request.method
+    insert_traces(ai_trace_mixed_spans(now=now, service="ai-rel-a", user="alice"))
+
+    # related values are served from attributes_metadata; one row per gate key,
+    # the traces row without any gate attribute and the logs row (wrong
+    # data_source, gate attribute present) must never surface
+    insert_attributes_metadata(
+        [
+            AttributesMetadata(
+                data_source="traces",
+                resource_attributes={"service.name": "ai-rel-a"},
+                attributes={"gen_ai.request.model": "gpt-rel", "gen_ai.user.id": "alice"},
+            ),
+            AttributesMetadata(
+                data_source="traces",
+                resource_attributes={"service.name": "ai-rel-b"},
+                attributes={"gen_ai.tool.name": "get_weather", "gen_ai.user.id": "bob"},
+            ),
+            AttributesMetadata(
+                data_source="traces",
+                resource_attributes={"service.name": "ai-rel-c"},
+                attributes={"gen_ai.agent.name": "chat-agent"},
+            ),
+            AttributesMetadata(
+                data_source="traces",
+                resource_attributes={"service.name": "plain-rel"},
+                attributes={"http.request.method": "POST"},
+            ),
+            AttributesMetadata(
+                data_source="logs",
+                resource_attributes={"service.name": "ai-rel-logs"},
+                attributes={"gen_ai.request.model": "gpt-rel"},
+            ),
+        ]
+    )
+
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    response = get_field_values(
-        signoz,
-        token,
-        {"name": "gen_ai.request.model", "existingQuery": "service.name = 'ai-it-values'"},
-        AI_VALUES_PATH,
-    )
-    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    params = {"name": "service.name", "searchText": search_text}
+    if existing_query is not None:
+        params["existingQuery"] = existing_query
+
+    response = get_field_values(signoz, token, params, AI_VALUES_PATH)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert response.json()["status"] == "success"
+
+    related = response.json()["data"]["values"].get("relatedValues") or []
+    assert set(related) == expected, related
 
 
 def test_ai_field_values_of_computed_aggregate_are_empty(