refactor(sqlmigration): re-encode kept quick filter entries with the JSON encoder

Uses a named entry type, encodes the kept entries without HTML escaping
instead of joining bytes, and keeps non-object entries instead of
skipping the whole row.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

pkg/signoz/provider.go

@@ -271,6 +271,7 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewRenameSavedViewDeprecatedTraceFieldsFactory(),
 		sqlmigration.NewAddDowntimeTuplesFactory(sqlstore),
 		sqlmigration.NewAddRuleTuplesFactory(sqlstore),
+		sqlmigration.NewRemoveUnsupportedQuickFiltersFactory(),
 	)
 }
 

pkg/sqlmigration/030_create_quick_filters.go

@@ -58,7 +58,6 @@ func (m *createQuickFilters) Up(ctx context.Context, db *bun.DB) error {
 			{"key": "httpMethod", "dataType": "string", "type": "tag"},
 			{"key": "httpRoute", "dataType": "string", "type": "tag"},
 			{"key": "httpUrl", "dataType": "string", "type": "tag"},
-			{"key": "traceID", "dataType": "string", "type": "tag"},
 		}},
 		{"logs", []map[string]any{
 			{"key": "severity_text", "dataType": "string", "type": "resource"},

pkg/sqlmigration/031_update_quick_filters.go

@@ -58,7 +58,6 @@ func (migration *updateQuickFilters) Up(ctx context.Context, db *bun.DB) error {
 			{"key": "http.method", "dataType": "string", "type": "tag"},
 			{"key": "http.route", "dataType": "string", "type": "tag"},
 			{"key": "http_url", "dataType": "string", "type": "tag"},
-			{"key": "trace_id", "dataType": "string", "type": "tag"},
 		}},
 		{"logs", []map[string]any{
 			{"key": "severity_text", "dataType": "string", "type": "resource"},

pkg/sqlmigration/144_remove_unsupported_quick_filters.go

@@ -0,0 +1,192 @@
+package sqlmigration
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log/slog"
+	"strings"
+
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/migrate"
+
+	"github.com/SigNoz/signoz/pkg/factory"
+)
+
+var unsupportedQuickFilterNames = map[string]struct{}{
+	// IDs and per-record values
+	"trace_id":            {},
+	"traceID":             {},
+	"span_id":             {},
+	"spanID":              {},
+	"parent_span_id":      {},
+	"parentSpanID":        {},
+	"timestamp":           {},
+	"body":                {},
+	"gen_ai.tool.call.id": {},
+
+	// Free-text payloads
+	"gen_ai.input.messages":       {},
+	"gen_ai.output.messages":      {},
+	"gen_ai.system_instructions":  {},
+	"gen_ai.prompt":               {},
+	"gen_ai.completion":           {},
+	"gen_ai.tool.call.arguments":  {},
+	"gen_ai.tool.call.result":     {},
+	"ai.prompt.messages":          {},
+	"ai.toolCall.result":          {},
+	"llm.input_messages":          {},
+	"input.value":                 {},
+	"output.value":                {},
+	"langfuse.observation.input":  {},
+	"langfuse.observation.output": {},
+	"traceloop.entity.input":      {},
+	"traceloop.entity.output":     {},
+	"final_result":                {},
+	"exception.message":           {},
+	"exception.stacktrace":        {},
+	"db.statement":                {},
+	"db.query.text":               {},
+
+	// Token counts
+	"gen_ai.usage.input_tokens":                              {},
+	"gen_ai.usage.output_tokens":                             {},
+	"gen_ai.usage.total_tokens":                              {},
+	"gen_ai.usage.prompt_tokens":                             {},
+	"gen_ai.usage.completion_tokens":                         {},
+	"gen_ai.usage.cached_input_tokens":                       {},
+	"gen_ai.usage.input_tokens.cached":                       {},
+	"gen_ai.usage.cache_read.input_tokens":                   {},
+	"gen_ai.usage.cache_read_input_tokens":                   {},
+	"gen_ai.usage.cache_creation.input_tokens":               {},
+	"gen_ai.usage.cache_creation_input_tokens":               {},
+	"gen_ai.usage.cache_write.input_tokens":                  {},
+	"gen_ai.usage.reasoning.output_tokens":                   {},
+	"gen_ai.usage.reasoning_tokens":                          {},
+	"gen_ai.usage.prompt_token_details.cached_read_tokens":   {},
+	"gen_ai.usage.completion_token_details.reasoning_tokens": {},
+	"ai.usage.inputTokens":                                   {},
+	"ai.usage.outputTokens":                                  {},
+	"ai.usage.promptTokens":                                  {},
+	"ai.usage.completionTokens":                              {},
+	"llm.token_count.prompt":                                 {},
+	"llm.token_count.completion":                             {},
+	"llm.token_count.total":                                  {},
+	"llm.token_count.prompt_details.cache_read":              {},
+	"llm.token_count.prompt_details.cache_write":             {},
+	"llm.token_count.completion_details.reasoning":           {},
+	"llm.usage.prompt_tokens":                                {},
+	"llm.usage.completion_tokens":                            {},
+	"llm.usage.total_tokens":                                 {},
+	"input_tokens":                                           {},
+	"output_tokens":                                          {},
+	"cache_read_tokens":                                      {},
+	"cache_creation_tokens":                                  {},
+
+	// Per-span costs
+	"signoz.gen_ai.usage.input_tokens.cost":             {},
+	"signoz.gen_ai.usage.output_tokens.cost":            {},
+	"signoz.gen_ai.usage.cache_read.input_tokens.cost":  {},
+	"signoz.gen_ai.usage.cache_write.input_tokens.cost": {},
+	"signoz.gen_ai.usage.tokens.cost":                   {},
+}
+
+type unsupportedQuickFilterRow struct {
+	bun.BaseModel `bun:"table:quick_filter"`
+
+	ID     string `bun:"id,pk"`
+	Filter string `bun:"filter"`
+}
+
+type unsupportedQuickFilterEntry struct {
+	Name string `json:"name"`
+}
+
+type removeUnsupportedQuickFilters struct {
+	settings factory.ProviderSettings
+}
+
+func NewRemoveUnsupportedQuickFiltersFactory() factory.ProviderFactory[SQLMigration, Config] {
+	return factory.NewProviderFactory(factory.MustNewName("drop_unsupported_quick_filters"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
+		return &removeUnsupportedQuickFilters{settings: ps}, nil
+	})
+}
+
+func (migration *removeUnsupportedQuickFilters) Register(migrations *migrate.Migrations) error {
+	return migrations.Register(migration.Up, migration.Down)
+}
+
+func (migration *removeUnsupportedQuickFilters) Up(ctx context.Context, db *bun.DB) error {
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	var rows []*unsupportedQuickFilterRow
+	if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
+		return err
+	}
+
+	var migrated, skipped int
+	for _, row := range rows {
+		filtered, changed, ok := removeUnsupportedQuickFilterEntries(row.Filter)
+		if !ok {
+			migration.settings.Logger.WarnContext(ctx, "quick filter could not be parsed, leaving it untouched", slog.String("quick_filter_id", row.ID), slog.String("raw_filter", row.Filter))
+			skipped++
+			continue
+		}
+		if !changed {
+			continue
+		}
+
+		migrated++
+		if _, err := tx.NewUpdate().Model((*unsupportedQuickFilterRow)(nil)).Set("filter = ?", filtered).Where("id = ?", row.ID).Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	migration.settings.Logger.InfoContext(ctx, "removed unsupported quick filters", slog.Int("total", len(rows)), slog.Int("migrated", migrated), slog.Int("skipped", skipped))
+
+	return tx.Commit()
+}
+
+func (migration *removeUnsupportedQuickFilters) Down(context.Context, *bun.DB) error {
+	return nil
+}
+
+// removeUnsupportedQuickFilterEntries drops entries whose name is in
+// unsupportedQuickFilterNames and re-encodes the rest unchanged. changed=false
+// means nothing was dropped and the row must not be written; ok=false means
+// unparseable.
+func removeUnsupportedQuickFilterEntries(filter string) (filtered string, changed bool, ok bool) {
+	var entries []json.RawMessage
+	if err := json.Unmarshal([]byte(filter), &entries); err != nil {
+		return "", false, false
+	}
+
+	kept := make([]json.RawMessage, 0, len(entries))
+	for _, rawEntry := range entries {
+		var entry unsupportedQuickFilterEntry
+		if err := json.Unmarshal(rawEntry, &entry); err != nil {
+			kept = append(kept, rawEntry)
+			continue
+		}
+		if _, unsupported := unsupportedQuickFilterNames[entry.Name]; unsupported {
+			continue
+		}
+		kept = append(kept, rawEntry)
+	}
+
+	if len(kept) == len(entries) {
+		return "", false, true
+	}
+
+	var buf bytes.Buffer
+	encoder := json.NewEncoder(&buf)
+	encoder.SetEscapeHTML(false)
+	if err := encoder.Encode(kept); err != nil {
+		return "", false, false
+	}
+	return strings.TrimSuffix(buf.String(), "\n"), true, true
+}

pkg/types/quickfiltertypes/filter.go

@@ -40,6 +40,86 @@ var (
 	SourceAiObservability = Source{valuer.NewString("ai_observability")}
 )
 
+// unsupportedFilterNames are fields whose values are (near-)unique per span or
+// log, so a quick filter on them would list nothing useful to pick from.
+var unsupportedFilterNames = map[string]struct{}{
+	// IDs and per-record values
+	"trace_id":            {},
+	"traceID":             {},
+	"span_id":             {},
+	"spanID":              {},
+	"parent_span_id":      {},
+	"parentSpanID":        {},
+	"timestamp":           {},
+	"body":                {},
+	"gen_ai.tool.call.id": {},
+
+	// Free-text payloads
+	"gen_ai.input.messages":       {},
+	"gen_ai.output.messages":      {},
+	"gen_ai.system_instructions":  {},
+	"gen_ai.prompt":               {},
+	"gen_ai.completion":           {},
+	"gen_ai.tool.call.arguments":  {},
+	"gen_ai.tool.call.result":     {},
+	"ai.prompt.messages":          {},
+	"ai.toolCall.result":          {},
+	"llm.input_messages":          {},
+	"input.value":                 {},
+	"output.value":                {},
+	"langfuse.observation.input":  {},
+	"langfuse.observation.output": {},
+	"traceloop.entity.input":      {},
+	"traceloop.entity.output":     {},
+	"final_result":                {},
+	"exception.message":           {},
+	"exception.stacktrace":        {},
+	"db.statement":                {},
+	"db.query.text":               {},
+
+	// Token counts
+	"gen_ai.usage.input_tokens":                              {},
+	"gen_ai.usage.output_tokens":                             {},
+	"gen_ai.usage.total_tokens":                              {},
+	"gen_ai.usage.prompt_tokens":                             {},
+	"gen_ai.usage.completion_tokens":                         {},
+	"gen_ai.usage.cached_input_tokens":                       {},
+	"gen_ai.usage.input_tokens.cached":                       {},
+	"gen_ai.usage.cache_read.input_tokens":                   {},
+	"gen_ai.usage.cache_read_input_tokens":                   {},
+	"gen_ai.usage.cache_creation.input_tokens":               {},
+	"gen_ai.usage.cache_creation_input_tokens":               {},
+	"gen_ai.usage.cache_write.input_tokens":                  {},
+	"gen_ai.usage.reasoning.output_tokens":                   {},
+	"gen_ai.usage.reasoning_tokens":                          {},
+	"gen_ai.usage.prompt_token_details.cached_read_tokens":   {},
+	"gen_ai.usage.completion_token_details.reasoning_tokens": {},
+	"ai.usage.inputTokens":                                   {},
+	"ai.usage.outputTokens":                                  {},
+	"ai.usage.promptTokens":                                  {},
+	"ai.usage.completionTokens":                              {},
+	"llm.token_count.prompt":                                 {},
+	"llm.token_count.completion":                             {},
+	"llm.token_count.total":                                  {},
+	"llm.token_count.prompt_details.cache_read":              {},
+	"llm.token_count.prompt_details.cache_write":             {},
+	"llm.token_count.completion_details.reasoning":           {},
+	"llm.usage.prompt_tokens":                                {},
+	"llm.usage.completion_tokens":                            {},
+	"llm.usage.total_tokens":                                 {},
+	"input_tokens":                                           {},
+	"output_tokens":                                          {},
+	"cache_read_tokens":                                      {},
+	"cache_creation_tokens":                                  {},
+
+	// Per-span costs
+	"signoz.gen_ai.usage.input_tokens.cost":             {},
+	"signoz.gen_ai.usage.output_tokens.cost":            {},
+	"signoz.gen_ai.usage.cache_read.input_tokens.cost":  {},
+	"signoz.gen_ai.usage.cache_write.input_tokens.cost": {},
+	"signoz.gen_ai.usage.tokens.cost":                   {},
+}
+
 func (Source) Enum() []any {
 	return []any{
 		SourceTraces,
@@ -184,7 +264,6 @@ func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
 		{Name: "http.method", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
 		{Name: "http.route", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
 		{Name: "http_url", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
-		{Name: "trace_id", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
 	}
 
 	logsFilters := []telemetrytypes.TelemetryFieldKey{
@@ -263,6 +342,9 @@ func validateFilters(filters []telemetrytypes.TelemetryFieldKey) error {
 		if filter.Name == "" {
 			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "filter name is required")
 		}
+		if _, unsupported := unsupportedFilterNames[filter.Name]; unsupported {
+			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "%s is not supported as a quick filter as it has too many unique values, use the search bar to filter by it instead", filter.Name)
+		}
 	}
 	return nil
 }

pkg/types/quickfiltertypes/filter_test.go

@@ -0,0 +1,58 @@
+package quickfiltertypes
+
+import (
+	"testing"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewStorableQuickFilter(t *testing.T) {
+	serviceName := telemetrytypes.TelemetryFieldKey{Name: "service.name", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString}
+
+	testCases := []struct {
+		name        string
+		filter      telemetrytypes.TelemetryFieldKey
+		rejectedKey string
+	}{
+		{name: "SupportedNumberField_Accepted", filter: telemetrytypes.TelemetryFieldKey{Name: "duration_nano", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeNumber}},
+		{name: "TraceID_Rejected", filter: telemetrytypes.TelemetryFieldKey{Name: "trace_id", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString}, rejectedKey: "trace_id"},
+		{name: "LogSpanID_Rejected", filter: telemetrytypes.TelemetryFieldKey{Name: "span_id", FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString}, rejectedKey: "span_id"},
+		{name: "LegacyParentSpanID_Rejected", filter: telemetrytypes.TelemetryFieldKey{Name: "parentSpanID", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString}, rejectedKey: "parentSpanID"},
+		{name: "LogBody_Rejected", filter: telemetrytypes.TelemetryFieldKey{Name: "body", FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString}, rejectedKey: "body"},
+		{name: "GenAIInputTokens_Rejected", filter: telemetrytypes.TelemetryFieldKey{Name: "gen_ai.usage.input_tokens", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber}, rejectedKey: "gen_ai.usage.input_tokens"},
+		{name: "OpenInferenceTokenCount_Rejected", filter: telemetrytypes.TelemetryFieldKey{Name: "llm.token_count.prompt", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber}, rejectedKey: "llm.token_count.prompt"},
+		{name: "GenAITotalCost_Rejected", filter: telemetrytypes.TelemetryFieldKey{Name: "signoz.gen_ai.usage.tokens.cost", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber}, rejectedKey: "signoz.gen_ai.usage.tokens.cost"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			_, err := NewStorableQuickFilter(valuer.GenerateUUID(), SourceTraces, []telemetrytypes.TelemetryFieldKey{serviceName, testCase.filter})
+			if testCase.rejectedKey == "" {
+				assert.NoError(t, err)
+				return
+			}
+
+			require.Error(t, err)
+			assert.True(t, errors.Ast(err, errors.TypeInvalidInput))
+			assert.Contains(t, err.Error(), testCase.rejectedKey)
+		})
+	}
+}
+
+func TestNewDefaultQuickFilterHasNoUnsupportedFields(t *testing.T) {
+	defaults, err := NewDefaultQuickFilter(valuer.GenerateUUID())
+	require.NoError(t, err)
+
+	for _, storable := range defaults {
+		sourceFilters, err := NewSourceFilterFromStorableQuickFilter(storable)
+		require.NoError(t, err)
+		for _, filter := range sourceFilters.Filters {
+			_, unsupported := unsupportedFilterNames[filter.Name]
+			assert.False(t, unsupported, "default %s quick filters contain %s", storable.Source.StringValue(), filter.Name)
+		}
+	}
+}

tests/integration/tests/quickfilter/01_quick_filter.py

@@ -266,6 +266,18 @@ def test_update_quick_filters_rejects_invalid_input(
             {"filters": [{"key": "service.name", "dataType": "string", "type": "resource"}]},
         ),
         ("invalid", {"filters": []}),
+        (
+            "traces",
+            {"filters": [{"name": "trace_id", "fieldContext": "span", "fieldDataType": "string"}]},
+        ),
+        (
+            "logs",
+            {"filters": [{"name": "span_id", "fieldContext": "log", "fieldDataType": "string"}]},
+        ),
+        (
+            "ai_observability",
+            {"filters": [{"name": "gen_ai.usage.input_tokens", "fieldContext": "attribute", "fieldDataType": "number"}]},
+        ),
     ]:
         response = requests.put(
             signoz.self.host_configs["8080"].get(f"/api/v2/quick_filters/{source}"),
@@ -292,6 +304,7 @@ def test_default_traces_filters_are_served_as_the_fields_api_serves_them(
     filters = {field_key["name"]: field_key for field_key in response.json()["data"]["filters"]}
 
     assert "hasError" not in filters
+    assert "trace_id" not in filters
     assert (filters["has_error"]["fieldContext"], filters["has_error"]["fieldDataType"]) == ("span", "bool")
     assert (filters["name"]["fieldContext"], filters["name"]["fieldDataType"]) == ("span", "string")
     assert (filters["duration_nano"]["fieldContext"], filters["duration_nano"]["fieldDataType"]) == ("span", "number")