Compare commits

...

3 Commits

Author SHA1 Message Date
nikhilmantri0902
f945b5f513 refactor(dashboard): port the list filter to the shared sqlcompiler
The visitor moves to a key-policy resolver plus an error-code wrap;
emitted SQL is unchanged, pinned by the existing exact-SQL unit suite.
2026-09-09 11:33:35 +05:30
nikhilmantri0902
8b0ab0ff26 refactor(filterquery): add shared list filter SQL compiler
Extracted from the dashboards list visitor: grammar walk, operator
dispatch, value extraction, LIKE builders and the Compiled output type,
behind a per-feature FieldResolver. Scope is list pages over the
relational store; telemetry queries stay on querybuilder. Also rejects
LIKE and ILIKE patterns ending in an unescaped backslash, which never
match on sqlite and abort the query on Postgres.
2026-09-09 11:33:34 +05:30
Srikanth Chekuri
d5af6f6d6b feat(telemetrymetadata): fields keys and values for the quick filters migration (#12794)
Some checks failed
build-staging / prepare (push) Has been cancelled
build-staging / js-build (push) Has been cancelled
build-staging / go-build (push) Has been cancelled
build-staging / staging (push) Has been cancelled
cacheci / tests (push) Has been cancelled
Release Drafter / update_release_draft (push) Has been cancelled
#### Description

- `fields/values` returns `boolValues` for bool fields: static span
fields such as `has_error`, stored bool attributes (decided by a
row-existence check, so they take no limit slots), and keys the caller
declares bool. `isRoot` and `isEntryPoint` return `true` only, matching
the condition builder.
- `fields/values` honors `startUnixMilli` for logs and traces against
the tag tables. The predicate is `toDate(unix_milli / 1000) >=
toDate(start)`, the partition expression, because the tag tables
deduplicate a value per day and keep whichever hour was inserted last.
For traces this means "span started on or after the start's day", for
logs "ingested on or after the start's day". `endUnixMilli` is ignored.
- `fields/keys` honors `fieldContext` and `fieldDataType` for the static
intrinsic and calculated keys, matches them case-insensitively, types
`isRoot`/`isEntryPoint` as bool, gives meter keys a data type, filters
the traces keys table by tag type for the scope context, and no longer
returns early for the log and scope contexts (logs and audit).
- The span context keeps returning attribute keys, since queries spell
span attributes as `span.<name>`.
- Default quick filters name the span and log fields the way the fields
API serves them (`has_error` as a span bool, `name`, `duration_nano`,
`response_status_code`, `http_host`, `http_url`, `trace_id` as span
fields, `severity_text` as a log field). Migration 126 rewrites the
static fields of every stored quick filter the same way, whatever
context the legacy seeds gave them. Deprecated names such as `hasError`
are not served.
- The API-backed checkbox hook merges `boolValues` into its options; the
`has_error` title is kept.
- The fields/values handler no longer drops `boolValues` from the
response.


🤖 Generated with [Claude Code](https://claude.com/claude-code)
2026-09-08 22:11:27 +00:00
18 changed files with 1445 additions and 941 deletions

View File

@@ -34,6 +34,7 @@ export function mockFieldsValuesAPI(response: {
relatedValues?: (string | null)[];
stringValues?: (string | null)[];
numberValues?: (number | null)[];
boolValues?: (boolean | null)[];
}): void {
server.use(
rest.get('http://localhost/api/v1/fields/values', (_, res, ctx) =>
@@ -46,6 +47,7 @@ export function mockFieldsValuesAPI(response: {
relatedValues: response.relatedValues ?? [],
stringValues: response.stringValues ?? [],
numberValues: response.numberValues ?? [],
boolValues: response.boolValues ?? [],
},
},
}),

View File

@@ -92,8 +92,12 @@ export function useFieldValues({
values.numberValues
?.filter((value): value is number => value !== null && value !== undefined)
.map((value) => value.toString()) || [];
const boolValues =
values.boolValues
?.filter((value): value is boolean => value !== null && value !== undefined)
.map((value) => value.toString()) || [];
return [...stringValues, ...numberValues];
return [...stringValues, ...numberValues, ...boolValues];
}, [data]);
return { relatedValues, allValues, isLoading, isFetching };

View File

@@ -8,6 +8,7 @@ import { FiltersType, IQuickFiltersConfig, SignalType } from './types';
const FILTER_TITLE_MAP: Record<string, string> = {
duration_nano: 'Duration',
hasError: 'Has Error (Status)',
has_error: 'Has Error (Status)',
};
const FILTER_TYPE_MAP: Record<string, FiltersType> = {

View File

@@ -4,41 +4,17 @@ import (
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/parser/filterquery/sqlcompiler"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
)
type Compiled struct {
SQL string
Args []any
}
func (c Compiled) IsEmpty() bool {
return c.SQL == ""
}
// Compile always returns a non-nil *Compiled. An empty query (or one that
// produces no SQL) yields a Compiled with an empty SQL — callers gate on
// SQL != "" rather than a nil check.
//
// A `key OP value` term compiles to a DSL predicate; a bare word is a
// case-insensitive substring search over the dashboard name, description, and tag
// keys/values. They compose through AND/OR/NOT, so `prod payment` matches both
// words (implicit AND) and `prod OR name = 'x'` mixes free text with a filter. A
// quoted token matches literally, e.g. `"prod payment"`.
func Compile(query string, formatter sqlstore.SQLFormatter) (*Compiled, error) {
if len(strings.TrimSpace(query)) == 0 {
return &Compiled{}, nil
}
sql, args, errs := newVisitor(formatter).compile(query)
// Compile wraps compiler errors in the dashboard list filter error code.
func Compile(query string, formatter sqlstore.SQLFormatter) (*sqlcompiler.Compiled, error) {
compiled, errs := sqlcompiler.Compile(query, formatter, dashboardFieldResolver{})
if len(errs) > 0 {
return nil, errors.NewInvalidInputf(dashboardtypes.ErrCodeDashboardListFilterInvalid,
"invalid filter query: %s", strings.Join(errs, "; "))
}
return &Compiled{
SQL: sql,
Args: args,
}, nil
return compiled, nil
}

View File

@@ -0,0 +1,125 @@
package impldashboard
import (
"strings"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
"github.com/SigNoz/signoz/pkg/parser/filterquery/sqlcompiler"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
sqlbuilder "github.com/huandu/go-sqlbuilder"
)
// dashboardFieldResolver maps dashboard list DSL keys; a non-reserved key is a tag key matched case-insensitively.
type dashboardFieldResolver struct{}
func (r dashboardFieldResolver) ResolveComparison(b *sqlcompiler.Builder, rawKey string, operation qbtypesv5.FilterOperator, ctx *grammar.ComparisonContext) string {
key := strings.ToLower(rawKey)
if allowedOperations, isReserved := dashboardtypes.ReservedOps[dashboardtypes.DSLKey(key)]; isReserved {
return r.resolveReservedKey(b, ctx, operation, dashboardtypes.DSLKey(key), allowedOperations)
}
if _, allowed := dashboardtypes.TagKeyOps[operation]; !allowed {
b.AddError("operator %s is not allowed on a tag-key filter", sqlcompiler.OperationName(operation))
return ""
}
return r.tagComparison(b, ctx, operation, key)
}
func (r dashboardFieldResolver) resolveReservedKey(b *sqlcompiler.Builder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, allowedOperations map[qbtypesv5.FilterOperator]struct{}) string {
if _, allowed := allowedOperations[operation]; !allowed {
b.AddError("operator %s is not allowed for key %q", sqlcompiler.OperationName(operation), key)
return ""
}
switch key {
case dashboardtypes.DSLKeyName:
columnExpression := string(b.Formatter().JSONExtractString("dashboard.data", "$.spec.display.name"))
return b.StringOperation(b.SelectBuilder(), ctx, operation, columnExpression, string(key))
case dashboardtypes.DSLKeyDescription:
columnExpression := string(b.Formatter().JSONExtractString("dashboard.data", "$.spec.display.description"))
return b.StringOperation(b.SelectBuilder(), ctx, operation, columnExpression, string(key))
case dashboardtypes.DSLKeyCreatedAt:
return b.TimestampComparison(ctx, operation, "dashboard.created_at")
case dashboardtypes.DSLKeyUpdatedAt:
return b.TimestampComparison(ctx, operation, "dashboard.updated_at")
case dashboardtypes.DSLKeyCreatedBy:
return b.StringOperation(b.SelectBuilder(), ctx, operation, "dashboard.created_by", string(key))
case dashboardtypes.DSLKeyLocked:
return b.BoolComparison(ctx, operation, "dashboard.locked")
}
b.AddError("no handler for reserved key %q", key)
return ""
}
func (dashboardFieldResolver) tagComparison(b *sqlcompiler.Builder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
subqueryBuilder := sqlbuilder.NewSelectBuilder()
if operation == qbtypesv5.FilterOperatorExists || operation == qbtypesv5.FilterOperatorNotExists {
buildSubqueryForTagKey(subqueryBuilder, tagKey)
} else {
// Value predicates take the positive operator; negation toggles the EXISTS wrapper.
positiveOperation := operation
if operation.IsNegativeOperator() {
positiveOperation = operation.Inverse()
}
valuePredicate := b.StringOperation(subqueryBuilder, ctx, positiveOperation, "t.value", tagKey)
if valuePredicate == "" {
return ""
}
buildSubqueryForTagKeyAndValue(subqueryBuilder, tagKey, valuePredicate)
}
if operation.IsNegativeOperator() {
return b.SelectBuilder().NotExists(subqueryBuilder)
}
return b.SelectBuilder().Exists(subqueryBuilder)
}
func buildSubqueryForTagKey(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey string) *sqlbuilder.SelectBuilder {
const dashboardTagKind = `"dashboard"`
return subqueryBuilder.
Select("1").
From("tag_relation tr").
Join("tag t", "t.id = tr.tag_id").
Where(
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
"tr.resource_id = dashboard.id",
"LOWER(t.key) = LOWER("+subqueryBuilder.Var(tagKey)+")",
)
}
func buildSubqueryForTagKeyAndValue(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey, valuePredicate string) *sqlbuilder.SelectBuilder {
return buildSubqueryForTagKey(subqueryBuilder, tagKey).Where(valuePredicate)
}
// FreeText searches name, description and tag keys/values.
func (dashboardFieldResolver) FreeText(b *sqlcompiler.Builder, value string) string {
nameColumn := string(b.Formatter().JSONExtractString("dashboard.data", "$.spec.display.name"))
descriptionColumn := string(b.Formatter().JSONExtractString("dashboard.data", "$.spec.display.description"))
namePredicate := b.FreeTextContains(b.SelectBuilder(), nameColumn, value)
descriptionPredicate := b.FreeTextContains(b.SelectBuilder(), descriptionColumn, value)
subqueryBuilder := sqlbuilder.NewSelectBuilder()
keyPredicate := b.FreeTextContains(subqueryBuilder, "t.key", value)
valuePredicate := b.FreeTextContains(subqueryBuilder, "t.value", value)
buildSubqueryForFreeTextTag(subqueryBuilder, keyPredicate, valuePredicate)
tagPredicate := b.SelectBuilder().Exists(subqueryBuilder)
return b.SelectBuilder().Or(namePredicate, descriptionPredicate, tagPredicate)
}
func buildSubqueryForFreeTextTag(subqueryBuilder *sqlbuilder.SelectBuilder, keyPredicate, valuePredicate string) *sqlbuilder.SelectBuilder {
const dashboardTagKind = `"dashboard"`
return subqueryBuilder.
Select("1").
From("tag_relation tr").
Join("tag t", "t.id = tr.tag_id").
Where(
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
"tr.resource_id = dashboard.id",
subqueryBuilder.Or(keyPredicate, valuePredicate),
)
}

View File

@@ -559,6 +559,11 @@ func TestCompile_Rejections(t *testing.T) {
dslQueryToCompile: `created_at >= 'not-a-date'`,
expectedErrShouldContain: "RFC3339",
},
{
subtestName: "rejects LIKE pattern ending in an unescaped backslash",
dslQueryToCompile: `name LIKE 'prod\\'`,
expectedErrShouldContain: "must not end with an unescaped backslash",
},
{
subtestName: "rejects REGEXP — not yet supported",
dslQueryToCompile: `name REGEXP '.*'`,
@@ -573,7 +578,7 @@ func TestCompile_Rejections(t *testing.T) {
}
// Every key in dashboardtypes.ReservedOps must have a matching case in
// visitComparisonForReservedKeys; a key that's reserved but unhandled falls
// resolveReservedKey; a key that's reserved but unhandled falls
// through to the "no handler for reserved key" error. Equal is accepted by all
// reserved keys, so `key = 'x'` always reaches the dispatch switch — a missing
// handler surfaces as that error regardless of whether the value type-checks.
@@ -583,7 +588,7 @@ func TestCompileReservedKeysAllHandled(t *testing.T) {
_, err := Compile(string(key)+` = 'x'`, formatter(t))
if err != nil {
assert.NotContains(t, err.Error(), "no handler for reserved key",
"reserved key %q has no handler in visitComparisonForReservedKeys", key)
"reserved key %q has no handler in resolveReservedKey", key)
}
})
}

View File

@@ -1,631 +0,0 @@
package impldashboard
import (
"fmt"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/parser/filterquery"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/antlr4-go/antlr/v4"
sqlbuilder "github.com/huandu/go-sqlbuilder"
)
// bunPlaceholderFlavor is any flavor that renders `?` placeholders, which bun
// re-binds to the actual backend (e.g. `$1` for Postgres) at query time.
const bunPlaceholderFlavor = sqlbuilder.SQLite
type visitor struct {
grammar.BaseFilterQueryVisitor
selectBuilder *sqlbuilder.SelectBuilder
formatter sqlstore.SQLFormatter
errors []string
}
func newVisitor(formatter sqlstore.SQLFormatter) *visitor {
return &visitor{
selectBuilder: sqlbuilder.NewSelectBuilder(),
formatter: formatter,
}
}
// compile builds `?`-placeholder WHERE SQL + args for bun. Each term is either a
// `key OP value` comparison or a bare token that becomes a free-text search; the
// two compose through the boolean grammar (AND/OR/NOT). Malformed input is
// returned as errors.
func (v *visitor) compile(query string) (string, []any, []string) {
tree, _, collector := filterquery.Parse(query)
if len(collector.Errors) > 0 {
return "", nil, collector.Errors
}
condition, _ := v.visit(tree).(string)
if len(v.errors) > 0 {
return "", nil, v.errors
}
if condition == "" {
return "", nil, nil
}
sql, arguments := v.selectBuilder.Args.CompileWithFlavor(condition, bunPlaceholderFlavor)
return sql, arguments, nil
}
func (v *visitor) visit(tree antlr.ParseTree) any {
if tree == nil {
return nil
}
return tree.Accept(v)
}
// ════════════════════════════════════════════════════════════════════════
// methods from grammar.BaseFilterQueryVisitor that are overridden
// ════════════════════════════════════════════════════════════════════════
func (v *visitor) VisitQuery(ctx *grammar.QueryContext) any {
return v.visit(ctx.Expression())
}
func (v *visitor) VisitExpression(ctx *grammar.ExpressionContext) any {
return v.visit(ctx.OrExpression())
}
func (v *visitor) VisitOrExpression(ctx *grammar.OrExpressionContext) any {
parts := ctx.AllAndExpression()
conditions := make([]string, 0, len(parts))
for _, part := range parts {
if condition, ok := v.visit(part).(string); ok && condition != "" {
conditions = append(conditions, condition)
}
}
switch len(conditions) {
case 0:
return ""
case 1:
return conditions[0]
default:
return v.selectBuilder.Or(conditions...)
}
}
func (v *visitor) VisitAndExpression(ctx *grammar.AndExpressionContext) any {
parts := ctx.AllUnaryExpression()
conditions := make([]string, 0, len(parts))
for _, part := range parts {
if condition, ok := v.visit(part).(string); ok && condition != "" {
conditions = append(conditions, condition)
}
}
switch len(conditions) {
case 0:
return ""
case 1:
return conditions[0]
default:
return v.selectBuilder.And(conditions...)
}
}
func (v *visitor) VisitUnaryExpression(ctx *grammar.UnaryExpressionContext) any {
condition, _ := v.visit(ctx.Primary()).(string)
if condition == "" {
return ""
}
if ctx.NOT() != nil {
return fmt.Sprintf("NOT (%s)", condition)
}
return condition
}
func (v *visitor) VisitPrimary(ctx *grammar.PrimaryContext) any {
if ctx.OrExpression() != nil {
return v.visit(ctx.OrExpression())
}
if ctx.Comparison() != nil {
return v.visit(ctx.Comparison())
}
// A lone key/value/full-text token is a free-text term, composed with any
// comparisons through the boolean grammar. A quoted token matches its contents
// literally — the escape hatch for a phrase or a term that looks like DSL.
return v.buildFreeTextTerm(trimQuotes(ctx.GetText()))
}
// VisitComparison dispatches a single `key OP value` term. A key that matches
// a reserved DSL key (name, description, etc.) becomes a column-level
// predicate; any other identifier is treated as a tag key — the operator
// applies to the tag's value, with a case-insensitive match on the tag's key.
func (v *visitor) VisitComparison(ctx *grammar.ComparisonContext) any {
key := strings.ToLower(strings.TrimSpace(ctx.Key().GetText()))
operation, ok := v.extractOperation(ctx)
if !ok {
return ""
}
if allowedOperations, isReserved := dashboardtypes.ReservedOps[dashboardtypes.DSLKey(key)]; isReserved {
return v.visitComparisonForReservedKeys(ctx, operation, dashboardtypes.DSLKey(key), allowedOperations)
}
return v.visitComparisonForTags(ctx, operation, key)
}
func (v *visitor) visitComparisonForReservedKeys(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, allowedOperations map[qbtypesv5.FilterOperator]struct{}) string {
if _, allowed := allowedOperations[operation]; !allowed {
v.addError("operator %s is not allowed for key %q", operationName(operation), key)
return ""
}
switch key {
case dashboardtypes.DSLKeyName:
return v.buildJSONStringComparison(ctx, operation, dashboardtypes.DSLKeyName, "$.spec.display.name")
case dashboardtypes.DSLKeyDescription:
return v.buildJSONStringComparison(ctx, operation, dashboardtypes.DSLKeyDescription, "$.spec.display.description")
case dashboardtypes.DSLKeyCreatedAt:
return v.buildTimestampComparison(ctx, operation, "dashboard.created_at")
case dashboardtypes.DSLKeyUpdatedAt:
return v.buildTimestampComparison(ctx, operation, "dashboard.updated_at")
case dashboardtypes.DSLKeyCreatedBy:
return v.buildStringComparison(ctx, operation, dashboardtypes.DSLKeyCreatedBy, "dashboard.created_by")
case dashboardtypes.DSLKeyLocked:
return v.buildBoolComparison(ctx, operation, "dashboard.locked")
}
// Unreachable for real input: every dashboardtypes.ReservedOps key has a case above, and
// TestCompileReservedKeysAllHandled guards that the two stay in sync.
v.addError("no handler for reserved key %q", key)
return ""
}
func (v *visitor) visitComparisonForTags(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
if _, allowed := dashboardtypes.TagKeyOps[operation]; !allowed {
v.addError("operator %s is not allowed on a tag-key filter", operationName(operation))
return ""
}
return v.buildTagComparison(ctx, operation, tagKey)
}
func (v *visitor) extractOperation(ctx *grammar.ComparisonContext) (qbtypesv5.FilterOperator, bool) {
// For operators that take an optional leading NOT, Inverse() maps each to
// its Not<X> counterpart.
maybeNot := func(operation qbtypesv5.FilterOperator) qbtypesv5.FilterOperator {
if ctx.NOT() != nil {
return operation.Inverse()
}
return operation
}
switch {
case ctx.EQUALS() != nil:
return qbtypesv5.FilterOperatorEqual, true
case ctx.NOT_EQUALS() != nil, ctx.NEQ() != nil:
return qbtypesv5.FilterOperatorNotEqual, true
case ctx.LT() != nil:
return qbtypesv5.FilterOperatorLessThan, true
case ctx.LE() != nil:
return qbtypesv5.FilterOperatorLessThanOrEq, true
case ctx.GT() != nil:
return qbtypesv5.FilterOperatorGreaterThan, true
case ctx.GE() != nil:
return qbtypesv5.FilterOperatorGreaterThanOrEq, true
case ctx.BETWEEN() != nil:
return maybeNot(qbtypesv5.FilterOperatorBetween), true
case ctx.LIKE() != nil:
return maybeNot(qbtypesv5.FilterOperatorLike), true
case ctx.ILIKE() != nil:
return maybeNot(qbtypesv5.FilterOperatorILike), true
case ctx.CONTAINS() != nil:
return maybeNot(qbtypesv5.FilterOperatorContains), true
case ctx.REGEXP() != nil:
return maybeNot(qbtypesv5.FilterOperatorRegexp), true
case ctx.InClause() != nil:
return qbtypesv5.FilterOperatorIn, true
case ctx.NotInClause() != nil:
return qbtypesv5.FilterOperatorNotIn, true
case ctx.EXISTS() != nil:
return maybeNot(qbtypesv5.FilterOperatorExists), true
}
v.addError("could not determine operator in expression %q", ctx.GetText())
return qbtypesv5.FilterOperatorUnknown, false
}
// ─── per-key emitters ────────────────────────────────────────────────────────
func (v *visitor) buildJSONStringComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, jsonPath string) string {
columnExpression := string(v.formatter.JSONExtractString("dashboard.data", jsonPath))
return v.buildStringOperation(v.selectBuilder, ctx, operation, columnExpression, string(key))
}
func (v *visitor) buildStringComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, columnExpression string) string {
return v.buildStringOperation(v.selectBuilder, ctx, operation, columnExpression, string(key))
}
// buildStringOperation covers all the operators the spec allows on text-shaped keys
// (name, description, created_by, and a tag's value). Placeholders are interned
// into builder — the outer builder for column predicates, the subquery builder for
// tag-value predicates — so nested EXISTS arguments thread correctly.
func (v *visitor) buildStringOperation(builder *sqlbuilder.SelectBuilder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression, keyForError string) string {
switch operation {
case qbtypesv5.FilterOperatorEqual:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
return builder.Equal(columnExpression, val)
case qbtypesv5.FilterOperatorNotEqual:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
return builder.NotEqual(columnExpression, val)
case qbtypesv5.FilterOperatorLike, qbtypesv5.FilterOperatorNotLike:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotLike {
like = "NOT LIKE"
}
// The user's % and _ stay as wildcards; ESCAPE pins backslash as the escape
// char so a literal `\` in the pattern is read the same on both dialects —
// Postgres defaults to `\`, SQLite has no default escape.
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, builder.Var(val))
case qbtypesv5.FilterOperatorILike, qbtypesv5.FilterOperatorNotILike:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
// SQLite has no ILIKE keyword and Postgres LIKE is case-sensitive — emit
// LOWER(col) LIKE LOWER(?) so behavior is identical on both dialects. ESCAPE
// pins backslash as the escape char (Postgres default; SQLite has none).
lowerColumn := string(v.formatter.LowerExpression(columnExpression))
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotILike {
like = "NOT LIKE"
}
return fmt.Sprintf("%s %s LOWER(%s) ESCAPE '\\'", lowerColumn, like, builder.Var(val))
case qbtypesv5.FilterOperatorContains, qbtypesv5.FilterOperatorNotContains:
val, ok := v.extractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotContains {
like = "NOT LIKE"
}
// Escape the user's % and _ so they match literally, then wrap in wildcards.
// ESCAPE declares the backslash the escaper injected as the escape char —
// needed on SQLite (no default) and a harmless restatement of the Postgres default.
escaped := v.formatter.EscapeLikePattern(val)
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, builder.Var("%"+escaped+"%"))
case qbtypesv5.FilterOperatorRegexp, qbtypesv5.FilterOperatorNotRegexp:
v.addError("REGEXP filtering on %q is not yet supported", keyForError)
return ""
case qbtypesv5.FilterOperatorIn, qbtypesv5.FilterOperatorNotIn:
values, ok := v.extractStringValueList(ctx, keyForError)
if !ok {
return ""
}
arguments := make([]any, len(values))
for i, s := range values {
arguments[i] = s
}
if operation == qbtypesv5.FilterOperatorNotIn {
return builder.NotIn(columnExpression, arguments...)
}
return builder.In(columnExpression, arguments...)
}
v.addError("operator %s on %q is not implemented", operationName(operation), keyForError)
return ""
}
func (v *visitor) buildTimestampComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
switch operation {
case qbtypesv5.FilterOperatorEqual, qbtypesv5.FilterOperatorNotEqual,
qbtypesv5.FilterOperatorLessThan, qbtypesv5.FilterOperatorLessThanOrEq,
qbtypesv5.FilterOperatorGreaterThan, qbtypesv5.FilterOperatorGreaterThanOrEq:
t, ok := v.extractSingleTimestampValue(ctx)
if !ok {
return ""
}
switch operation {
case qbtypesv5.FilterOperatorEqual:
return v.selectBuilder.Equal(columnExpression, t)
case qbtypesv5.FilterOperatorNotEqual:
return v.selectBuilder.NotEqual(columnExpression, t)
case qbtypesv5.FilterOperatorLessThan:
return v.selectBuilder.LessThan(columnExpression, t)
case qbtypesv5.FilterOperatorLessThanOrEq:
return v.selectBuilder.LessEqualThan(columnExpression, t)
case qbtypesv5.FilterOperatorGreaterThan:
return v.selectBuilder.GreaterThan(columnExpression, t)
case qbtypesv5.FilterOperatorGreaterThanOrEq:
return v.selectBuilder.GreaterEqualThan(columnExpression, t)
}
case qbtypesv5.FilterOperatorBetween, qbtypesv5.FilterOperatorNotBetween:
timestamps, ok := v.extractTwoTimestampValues(ctx)
if !ok {
return ""
}
if operation == qbtypesv5.FilterOperatorNotBetween {
return v.selectBuilder.NotBetween(columnExpression, timestamps[0], timestamps[1])
}
return v.selectBuilder.Between(columnExpression, timestamps[0], timestamps[1])
}
v.addError("operator %s on timestamp is not implemented", operationName(operation))
return ""
}
func (v *visitor) buildBoolComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
b, ok := v.extractSingleBoolValue(ctx)
if !ok {
return ""
}
if operation == qbtypesv5.FilterOperatorNotEqual {
return v.selectBuilder.NotEqual(columnExpression, b)
}
return v.selectBuilder.Equal(columnExpression, b)
}
func (v *visitor) buildTagComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
subqueryBuilder := sqlbuilder.NewSelectBuilder()
if operation == qbtypesv5.FilterOperatorExists || operation == qbtypesv5.FilterOperatorNotExists {
buildSubqueryForTagKey(subqueryBuilder, tagKey)
} else {
// All other tag operators take the positive form of the value predicate
// and toggle the EXISTS wrapper for negation. Inverse() flips Not<X> → <X>.
positiveOperation := operation
if operation.IsNegativeOperator() {
positiveOperation = operation.Inverse()
}
valuePredicate := v.buildStringOperation(subqueryBuilder, ctx, positiveOperation, "t.value", tagKey)
if valuePredicate == "" {
return ""
}
buildSubqueryForTagKeyAndValue(subqueryBuilder, tagKey, valuePredicate)
}
if operation.IsNegativeOperator() {
return v.selectBuilder.NotExists(subqueryBuilder)
}
return v.selectBuilder.Exists(subqueryBuilder)
}
func buildSubqueryForTagKey(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey string) *sqlbuilder.SelectBuilder {
const dashboardTagKind = `"dashboard"`
return subqueryBuilder.
Select("1").
From("tag_relation tr").
Join("tag t", "t.id = tr.tag_id").
Where(
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
"tr.resource_id = dashboard.id",
"LOWER(t.key) = LOWER("+subqueryBuilder.Var(tagKey)+")",
)
}
func buildSubqueryForTagKeyAndValue(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey, valuePredicate string) *sqlbuilder.SelectBuilder {
return buildSubqueryForTagKey(subqueryBuilder, tagKey).Where(valuePredicate)
}
// ─── free-text search ────────────────────────────────────────────────────────
// buildFreeTextTerm matches value as a case-insensitive substring of the
// dashboard name, description, or any tag key/value.
func (v *visitor) buildFreeTextTerm(value string) string {
nameColumn := string(v.formatter.JSONExtractString("dashboard.data", "$.spec.display.name"))
descriptionColumn := string(v.formatter.JSONExtractString("dashboard.data", "$.spec.display.description"))
namePredicate := v.buildFreeTextContains(v.selectBuilder, nameColumn, value)
descriptionPredicate := v.buildFreeTextContains(v.selectBuilder, descriptionColumn, value)
subqueryBuilder := sqlbuilder.NewSelectBuilder()
keyPredicate := v.buildFreeTextContains(subqueryBuilder, "t.key", value)
valuePredicate := v.buildFreeTextContains(subqueryBuilder, "t.value", value)
buildSubqueryForFreeTextTag(subqueryBuilder, keyPredicate, valuePredicate)
tagPredicate := v.selectBuilder.Exists(subqueryBuilder)
return v.selectBuilder.Or(namePredicate, descriptionPredicate, tagPredicate)
}
// buildFreeTextContains emits a case-insensitive contains as
// LOWER(COALESCE(col, '')) LIKE LOWER(?), identical on SQLite and Postgres.
// COALESCE keeps a NULL column (an absent description) false rather than NULL —
// otherwise `NOT (…)` goes NULL and drops every description-less dashboard. The
// value's % and _ are escaped, and ESCAPE pins backslash as the escape char.
func (v *visitor) buildFreeTextContains(builder *sqlbuilder.SelectBuilder, columnExpression, value string) string {
lowerColumn := string(v.formatter.LowerExpression("COALESCE(" + columnExpression + ", '')"))
pattern := "%" + v.formatter.EscapeLikePattern(value) + "%"
return fmt.Sprintf("%s LIKE LOWER(%s) ESCAPE '\\'", lowerColumn, builder.Var(pattern))
}
func buildSubqueryForFreeTextTag(subqueryBuilder *sqlbuilder.SelectBuilder, keyPredicate, valuePredicate string) *sqlbuilder.SelectBuilder {
const dashboardTagKind = `"dashboard"`
return subqueryBuilder.
Select("1").
From("tag_relation tr").
Join("tag t", "t.id = tr.tag_id").
Where(
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
"tr.resource_id = dashboard.id",
subqueryBuilder.Or(keyPredicate, valuePredicate),
)
}
// ─── value extraction helpers ───────────────────────────────────────────────
func (v *visitor) addError(format string, arguments ...any) {
v.errors = append(v.errors, fmt.Sprintf(format, arguments...))
}
func (v *visitor) extractSingleStringValue(ctx *grammar.ComparisonContext, keyForError string) (string, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.addError("expected exactly one value for %q", keyForError)
return "", false
}
return v.extractStringValue(values[0], keyForError)
}
func (v *visitor) extractSingleBoolValue(ctx *grammar.ComparisonContext) (bool, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.addError("expected a single boolean (true/false)")
return false, false
}
return v.extractBoolValue(values[0])
}
func (v *visitor) extractSingleTimestampValue(ctx *grammar.ComparisonContext) (time.Time, bool) {
values := ctx.AllValue()
if len(values) != 1 {
v.addError("expected a single RFC3339 timestamp")
return time.Time{}, false
}
return v.extractTimestampValue(values[0])
}
func (v *visitor) extractTwoTimestampValues(ctx *grammar.ComparisonContext) ([2]time.Time, bool) {
values := ctx.AllValue()
if len(values) != 2 {
v.addError("BETWEEN expects two RFC3339 timestamps")
return [2]time.Time{}, false
}
a, ok1 := v.extractTimestampValue(values[0])
b, ok2 := v.extractTimestampValue(values[1])
if !ok1 || !ok2 {
return [2]time.Time{}, false
}
return [2]time.Time{a, b}, true
}
func (v *visitor) extractStringValueList(ctx *grammar.ComparisonContext, keyForError string) ([]string, bool) {
var valuesCtx []grammar.IValueContext
switch {
case ctx.InClause() != nil:
inClause := ctx.InClause()
if inClause.ValueList() != nil {
valuesCtx = inClause.ValueList().AllValue()
} else {
valuesCtx = []grammar.IValueContext{inClause.Value()}
}
case ctx.NotInClause() != nil:
notInClause := ctx.NotInClause()
if notInClause.ValueList() != nil {
valuesCtx = notInClause.ValueList().AllValue()
} else {
valuesCtx = []grammar.IValueContext{notInClause.Value()}
}
default:
v.addError("IN clause is missing for %q", keyForError)
return nil, false
}
if len(valuesCtx) == 0 {
v.addError("IN list for %q is empty", keyForError)
return nil, false
}
out := make([]string, 0, len(valuesCtx))
for _, valueContext := range valuesCtx {
s, ok := v.extractStringValue(valueContext, keyForError)
if !ok {
return nil, false
}
out = append(out, s)
}
return out, true
}
func (v *visitor) extractStringValue(ctx grammar.IValueContext, keyForError string) (string, bool) {
if ctx.QUOTED_TEXT() != nil {
return trimQuotes(ctx.QUOTED_TEXT().GetText()), true
}
if ctx.KEY() != nil {
// Bare tokens are accepted as strings, mirroring the FilterQuery lexer's
// treatment of unquoted identifiers on the value side.
return ctx.KEY().GetText(), true
}
v.addError("expected a string value for %q, got %q", keyForError, ctx.GetText())
return "", false
}
func (v *visitor) extractBoolValue(ctx grammar.IValueContext) (bool, bool) {
if ctx.BOOL() == nil {
v.addError("expected a boolean (true/false), got %q", ctx.GetText())
return false, false
}
return strings.EqualFold(ctx.BOOL().GetText(), "true"), true
}
func (v *visitor) extractTimestampValue(ctx grammar.IValueContext) (time.Time, bool) {
if ctx.QUOTED_TEXT() == nil {
v.addError("expected an RFC3339 timestamp string, got %q", ctx.GetText())
return time.Time{}, false
}
raw := trimQuotes(ctx.QUOTED_TEXT().GetText())
t, err := time.Parse(time.RFC3339, raw)
if err != nil {
v.addError("invalid RFC3339 timestamp %q: %s", raw, err.Error())
return time.Time{}, false
}
return t, true
}
// ─── operator spelling ───────────────────────────────────────────────────────
// operationName returns the user-facing spelling of a FilterOperator, used only in
// error messages — go-sqlbuilder's Cond helpers emit the SQL keywords.
func operationName(operation qbtypesv5.FilterOperator) string {
switch operation {
case qbtypesv5.FilterOperatorEqual:
return "="
case qbtypesv5.FilterOperatorNotEqual:
return "!="
case qbtypesv5.FilterOperatorLessThan:
return "<"
case qbtypesv5.FilterOperatorLessThanOrEq:
return "<="
case qbtypesv5.FilterOperatorGreaterThan:
return ">"
case qbtypesv5.FilterOperatorGreaterThanOrEq:
return ">="
case qbtypesv5.FilterOperatorBetween:
return "BETWEEN"
case qbtypesv5.FilterOperatorNotBetween:
return "NOT BETWEEN"
case qbtypesv5.FilterOperatorLike:
return "LIKE"
case qbtypesv5.FilterOperatorNotLike:
return "NOT LIKE"
case qbtypesv5.FilterOperatorILike:
return "ILIKE"
case qbtypesv5.FilterOperatorNotILike:
return "NOT ILIKE"
case qbtypesv5.FilterOperatorContains:
return "CONTAINS"
case qbtypesv5.FilterOperatorNotContains:
return "NOT CONTAINS"
case qbtypesv5.FilterOperatorRegexp:
return "REGEXP"
case qbtypesv5.FilterOperatorNotRegexp:
return "NOT REGEXP"
case qbtypesv5.FilterOperatorIn:
return "IN"
case qbtypesv5.FilterOperatorNotIn:
return "NOT IN"
case qbtypesv5.FilterOperatorExists:
return "EXISTS"
case qbtypesv5.FilterOperatorNotExists:
return "NOT EXISTS"
}
return "?"
}
func trimQuotes(s string) string {
if len(s) >= 2 {
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
s = s[1 : len(s)-1]
}
}
s = strings.ReplaceAll(s, `\\`, `\`)
s = strings.ReplaceAll(s, `\'`, `'`)
return s
}

View File

@@ -82,6 +82,7 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques
values := &telemetrytypes.TelemetryFieldValues{
StringValues: allValues.StringValues,
BoolValues: allValues.BoolValues,
NumberValues: allValues.NumberValues,
RelatedValues: relatedValues,
}

View File

@@ -0,0 +1,521 @@
// Package sqlcompiler compiles list-page filter queries to relational-store WHERE clauses; telemetry queries stay on querybuilder's ClickHouse visitor.
package sqlcompiler
import (
"fmt"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/parser/filterquery"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
"github.com/SigNoz/signoz/pkg/sqlstore"
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/antlr4-go/antlr/v4"
sqlbuilder "github.com/huandu/go-sqlbuilder"
)
// bunPlaceholderFlavor is any flavor that renders the `?` placeholders bun expects.
const bunPlaceholderFlavor = sqlbuilder.SQLite
// FieldResolver is the per-feature policy: which keys exist and what each maps to.
type FieldResolver interface {
// ResolveComparison builds the predicate for one `key OP value` term; key keeps the user's casing.
ResolveComparison(b *Builder, key string, operation qbtypesv5.FilterOperator, ctx *grammar.ComparisonContext) string
// FreeText builds the predicate for a bare token.
FreeText(b *Builder, value string) string
}
// Compiled is a `?`-placeholder WHERE clause with its bun bind args.
type Compiled struct {
SQL string
Args []any
}
func (c Compiled) IsEmpty() bool {
return c.SQL == ""
}
// Compile on success returns a non-nil *Compiled, empty for an empty query; callers gate on IsEmpty, not nil.
func Compile(query string, formatter sqlstore.SQLFormatter, resolver FieldResolver) (*Compiled, []string) {
if len(strings.TrimSpace(query)) == 0 {
return &Compiled{}, nil
}
v := &visitor{
builder: &Builder{
selectBuilder: sqlbuilder.NewSelectBuilder(),
formatter: formatter,
},
resolver: resolver,
}
tree, _, collector := filterquery.Parse(query)
if len(collector.Errors) > 0 {
return nil, collector.Errors
}
condition, _ := v.visit(tree).(string)
if len(v.builder.errors) > 0 {
return nil, v.builder.errors
}
if condition == "" {
return &Compiled{}, nil
}
sql, arguments := v.builder.selectBuilder.Args.CompileWithFlavor(condition, bunPlaceholderFlavor)
return &Compiled{SQL: sql, Args: arguments}, nil
}
// Builder is the per-compile toolbox handed to a FieldResolver.
type Builder struct {
selectBuilder *sqlbuilder.SelectBuilder
formatter sqlstore.SQLFormatter
errors []string
}
func (b *Builder) SelectBuilder() *sqlbuilder.SelectBuilder {
return b.selectBuilder
}
func (b *Builder) Formatter() sqlstore.SQLFormatter {
return b.formatter
}
func (b *Builder) AddError(format string, arguments ...any) {
b.errors = append(b.errors, fmt.Sprintf(format, arguments...))
}
// StringOperation interns placeholders into sb so nested subquery arguments thread correctly.
func (b *Builder) StringOperation(sb *sqlbuilder.SelectBuilder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression, keyForError string) string {
switch operation {
case qbtypesv5.FilterOperatorEqual:
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
return sb.Equal(columnExpression, val)
case qbtypesv5.FilterOperatorNotEqual:
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
return sb.NotEqual(columnExpression, val)
case qbtypesv5.FilterOperatorLike, qbtypesv5.FilterOperatorNotLike:
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
if endsWithDanglingEscape(val) {
b.AddError("LIKE pattern for %q must not end with an unescaped backslash, use \\\\ to match a literal backslash", keyForError)
return ""
}
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotLike {
like = "NOT LIKE"
}
// ESCAPE pins backslash as the escape char (the Postgres default, SQLite has none).
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, sb.Var(val))
case qbtypesv5.FilterOperatorILike, qbtypesv5.FilterOperatorNotILike:
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
if endsWithDanglingEscape(val) {
b.AddError("ILIKE pattern for %q must not end with an unescaped backslash, use \\\\ to match a literal backslash", keyForError)
return ""
}
// SQLite has no ILIKE and Postgres LIKE is case-sensitive, so LOWER both sides.
lowerColumn := string(b.formatter.LowerExpression(columnExpression))
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotILike {
like = "NOT LIKE"
}
return fmt.Sprintf("%s %s LOWER(%s) ESCAPE '\\'", lowerColumn, like, sb.Var(val))
case qbtypesv5.FilterOperatorContains, qbtypesv5.FilterOperatorNotContains:
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
if !ok {
return ""
}
like := "LIKE"
if operation == qbtypesv5.FilterOperatorNotContains {
like = "NOT LIKE"
}
// Escape the user's % and _ so they match literally, then wrap in wildcards.
escaped := b.formatter.EscapeLikePattern(val)
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, sb.Var(fmt.Sprintf("%%%s%%", escaped)))
case qbtypesv5.FilterOperatorRegexp, qbtypesv5.FilterOperatorNotRegexp:
b.AddError("REGEXP filtering on %q is not supported", keyForError)
return ""
case qbtypesv5.FilterOperatorIn, qbtypesv5.FilterOperatorNotIn:
values, ok := b.ExtractStringValueList(ctx, keyForError)
if !ok {
return ""
}
arguments := make([]any, len(values))
for i, s := range values {
arguments[i] = s
}
if operation == qbtypesv5.FilterOperatorNotIn {
return sb.NotIn(columnExpression, arguments...)
}
return sb.In(columnExpression, arguments...)
}
b.AddError("operator %s on %q is not implemented", OperationName(operation), keyForError)
return ""
}
func (b *Builder) TimestampComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
switch operation {
case qbtypesv5.FilterOperatorEqual, qbtypesv5.FilterOperatorNotEqual,
qbtypesv5.FilterOperatorLessThan, qbtypesv5.FilterOperatorLessThanOrEq,
qbtypesv5.FilterOperatorGreaterThan, qbtypesv5.FilterOperatorGreaterThanOrEq:
t, ok := b.extractSingleTimestampValue(ctx)
if !ok {
return ""
}
switch operation {
case qbtypesv5.FilterOperatorEqual:
return b.selectBuilder.Equal(columnExpression, t)
case qbtypesv5.FilterOperatorNotEqual:
return b.selectBuilder.NotEqual(columnExpression, t)
case qbtypesv5.FilterOperatorLessThan:
return b.selectBuilder.LessThan(columnExpression, t)
case qbtypesv5.FilterOperatorLessThanOrEq:
return b.selectBuilder.LessEqualThan(columnExpression, t)
case qbtypesv5.FilterOperatorGreaterThan:
return b.selectBuilder.GreaterThan(columnExpression, t)
case qbtypesv5.FilterOperatorGreaterThanOrEq:
return b.selectBuilder.GreaterEqualThan(columnExpression, t)
}
case qbtypesv5.FilterOperatorBetween, qbtypesv5.FilterOperatorNotBetween:
timestamps, ok := b.extractTwoTimestampValues(ctx)
if !ok {
return ""
}
if operation == qbtypesv5.FilterOperatorNotBetween {
return b.selectBuilder.NotBetween(columnExpression, timestamps[0], timestamps[1])
}
return b.selectBuilder.Between(columnExpression, timestamps[0], timestamps[1])
}
b.AddError("operator %s on timestamp is not implemented", OperationName(operation))
return ""
}
func (b *Builder) BoolComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
value, ok := b.extractSingleBoolValue(ctx)
if !ok {
return ""
}
if operation == qbtypesv5.FilterOperatorNotEqual {
return b.selectBuilder.NotEqual(columnExpression, value)
}
return b.selectBuilder.Equal(columnExpression, value)
}
// A pattern ending in an unescaped backslash never matches on sqlite and errors on Postgres.
func endsWithDanglingEscape(value string) bool {
trailing := len(value) - len(strings.TrimRight(value, `\`))
return trailing%2 == 1
}
// FreeTextContains COALESCEs the column so NOT (...) does not go NULL and drop rows where it is absent.
func (b *Builder) FreeTextContains(sb *sqlbuilder.SelectBuilder, columnExpression, value string) string {
lowerColumn := string(b.formatter.LowerExpression(fmt.Sprintf("COALESCE(%s, '')", columnExpression)))
pattern := fmt.Sprintf("%%%s%%", b.formatter.EscapeLikePattern(value))
return fmt.Sprintf("%s LIKE LOWER(%s) ESCAPE '\\'", lowerColumn, sb.Var(pattern))
}
func (b *Builder) ExtractSingleStringValue(ctx *grammar.ComparisonContext, keyForError string) (string, bool) {
values := ctx.AllValue()
if len(values) != 1 {
b.AddError("expected exactly one value for %q", keyForError)
return "", false
}
return b.extractStringValue(values[0], keyForError)
}
func (b *Builder) ExtractStringValueList(ctx *grammar.ComparisonContext, keyForError string) ([]string, bool) {
var valuesCtx []grammar.IValueContext
switch {
case ctx.InClause() != nil:
inClause := ctx.InClause()
if inClause.ValueList() != nil {
valuesCtx = inClause.ValueList().AllValue()
} else {
valuesCtx = []grammar.IValueContext{inClause.Value()}
}
case ctx.NotInClause() != nil:
notInClause := ctx.NotInClause()
if notInClause.ValueList() != nil {
valuesCtx = notInClause.ValueList().AllValue()
} else {
valuesCtx = []grammar.IValueContext{notInClause.Value()}
}
default:
b.AddError("IN clause is missing for %q", keyForError)
return nil, false
}
if len(valuesCtx) == 0 {
b.AddError("IN list for %q is empty", keyForError)
return nil, false
}
out := make([]string, 0, len(valuesCtx))
for _, valueContext := range valuesCtx {
s, ok := b.extractStringValue(valueContext, keyForError)
if !ok {
return nil, false
}
out = append(out, s)
}
return out, true
}
func (b *Builder) extractSingleBoolValue(ctx *grammar.ComparisonContext) (bool, bool) {
values := ctx.AllValue()
if len(values) != 1 {
b.AddError("expected a single boolean (true/false)")
return false, false
}
return b.extractBoolValue(values[0])
}
func (b *Builder) extractSingleTimestampValue(ctx *grammar.ComparisonContext) (time.Time, bool) {
values := ctx.AllValue()
if len(values) != 1 {
b.AddError("expected a single RFC3339 timestamp")
return time.Time{}, false
}
return b.extractTimestampValue(values[0])
}
func (b *Builder) extractTwoTimestampValues(ctx *grammar.ComparisonContext) ([2]time.Time, bool) {
values := ctx.AllValue()
if len(values) != 2 {
b.AddError("BETWEEN expects two RFC3339 timestamps")
return [2]time.Time{}, false
}
first, ok1 := b.extractTimestampValue(values[0])
second, ok2 := b.extractTimestampValue(values[1])
if !ok1 || !ok2 {
return [2]time.Time{}, false
}
return [2]time.Time{first, second}, true
}
func (b *Builder) extractStringValue(ctx grammar.IValueContext, keyForError string) (string, bool) {
if ctx.QUOTED_TEXT() != nil {
return trimQuotes(ctx.QUOTED_TEXT().GetText()), true
}
if ctx.KEY() != nil {
return ctx.KEY().GetText(), true
}
b.AddError("expected a string value for %q, got %q", keyForError, ctx.GetText())
return "", false
}
func (b *Builder) extractBoolValue(ctx grammar.IValueContext) (bool, bool) {
if ctx.BOOL() == nil {
b.AddError("expected a boolean (true/false), got %q", ctx.GetText())
return false, false
}
return strings.EqualFold(ctx.BOOL().GetText(), "true"), true
}
func (b *Builder) extractTimestampValue(ctx grammar.IValueContext) (time.Time, bool) {
if ctx.QUOTED_TEXT() == nil {
b.AddError("expected an RFC3339 timestamp string, got %q", ctx.GetText())
return time.Time{}, false
}
raw := trimQuotes(ctx.QUOTED_TEXT().GetText())
t, err := time.Parse(time.RFC3339, raw)
if err != nil {
b.AddError("invalid RFC3339 timestamp %q: %s", raw, err.Error())
return time.Time{}, false
}
return t, true
}
type visitor struct {
grammar.BaseFilterQueryVisitor
builder *Builder
resolver FieldResolver
}
func (v *visitor) visit(tree antlr.ParseTree) any {
if tree == nil {
return nil
}
return tree.Accept(v)
}
func (v *visitor) VisitQuery(ctx *grammar.QueryContext) any {
return v.visit(ctx.Expression())
}
func (v *visitor) VisitExpression(ctx *grammar.ExpressionContext) any {
return v.visit(ctx.OrExpression())
}
func (v *visitor) VisitOrExpression(ctx *grammar.OrExpressionContext) any {
parts := ctx.AllAndExpression()
conditions := make([]string, 0, len(parts))
for _, part := range parts {
if condition, ok := v.visit(part).(string); ok && condition != "" {
conditions = append(conditions, condition)
}
}
switch len(conditions) {
case 0:
return ""
case 1:
return conditions[0]
default:
return v.builder.selectBuilder.Or(conditions...)
}
}
func (v *visitor) VisitAndExpression(ctx *grammar.AndExpressionContext) any {
parts := ctx.AllUnaryExpression()
conditions := make([]string, 0, len(parts))
for _, part := range parts {
if condition, ok := v.visit(part).(string); ok && condition != "" {
conditions = append(conditions, condition)
}
}
switch len(conditions) {
case 0:
return ""
case 1:
return conditions[0]
default:
return v.builder.selectBuilder.And(conditions...)
}
}
func (v *visitor) VisitUnaryExpression(ctx *grammar.UnaryExpressionContext) any {
condition, _ := v.visit(ctx.Primary()).(string)
if condition == "" {
return ""
}
if ctx.NOT() != nil {
return fmt.Sprintf("NOT (%s)", condition)
}
return condition
}
func (v *visitor) VisitPrimary(ctx *grammar.PrimaryContext) any {
if ctx.OrExpression() != nil {
return v.visit(ctx.OrExpression())
}
if ctx.Comparison() != nil {
return v.visit(ctx.Comparison())
}
// A quoted lone token matches its contents literally, the escape hatch for a phrase that looks like DSL.
return v.resolver.FreeText(v.builder, trimQuotes(ctx.GetText()))
}
func (v *visitor) VisitComparison(ctx *grammar.ComparisonContext) any {
key := strings.TrimSpace(ctx.Key().GetText())
operation, ok := v.extractOperation(ctx)
if !ok {
return ""
}
return v.resolver.ResolveComparison(v.builder, key, operation, ctx)
}
func (v *visitor) extractOperation(ctx *grammar.ComparisonContext) (qbtypesv5.FilterOperator, bool) {
maybeNot := func(operation qbtypesv5.FilterOperator) qbtypesv5.FilterOperator {
if ctx.NOT() != nil {
return operation.Inverse()
}
return operation
}
switch {
case ctx.EQUALS() != nil:
return qbtypesv5.FilterOperatorEqual, true
case ctx.NOT_EQUALS() != nil, ctx.NEQ() != nil:
return qbtypesv5.FilterOperatorNotEqual, true
case ctx.LT() != nil:
return qbtypesv5.FilterOperatorLessThan, true
case ctx.LE() != nil:
return qbtypesv5.FilterOperatorLessThanOrEq, true
case ctx.GT() != nil:
return qbtypesv5.FilterOperatorGreaterThan, true
case ctx.GE() != nil:
return qbtypesv5.FilterOperatorGreaterThanOrEq, true
case ctx.BETWEEN() != nil:
return maybeNot(qbtypesv5.FilterOperatorBetween), true
case ctx.LIKE() != nil:
return maybeNot(qbtypesv5.FilterOperatorLike), true
case ctx.ILIKE() != nil:
return maybeNot(qbtypesv5.FilterOperatorILike), true
case ctx.CONTAINS() != nil:
return maybeNot(qbtypesv5.FilterOperatorContains), true
case ctx.REGEXP() != nil:
return maybeNot(qbtypesv5.FilterOperatorRegexp), true
case ctx.InClause() != nil:
return qbtypesv5.FilterOperatorIn, true
case ctx.NotInClause() != nil:
return qbtypesv5.FilterOperatorNotIn, true
case ctx.EXISTS() != nil:
return maybeNot(qbtypesv5.FilterOperatorExists), true
}
v.builder.AddError("could not determine operator in expression %q", ctx.GetText())
return qbtypesv5.FilterOperatorUnknown, false
}
// OperationName is the user-facing spelling, used only in error messages.
func OperationName(operation qbtypesv5.FilterOperator) string {
switch operation {
case qbtypesv5.FilterOperatorEqual:
return "="
case qbtypesv5.FilterOperatorNotEqual:
return "!="
case qbtypesv5.FilterOperatorLessThan:
return "<"
case qbtypesv5.FilterOperatorLessThanOrEq:
return "<="
case qbtypesv5.FilterOperatorGreaterThan:
return ">"
case qbtypesv5.FilterOperatorGreaterThanOrEq:
return ">="
case qbtypesv5.FilterOperatorBetween:
return "BETWEEN"
case qbtypesv5.FilterOperatorNotBetween:
return "NOT BETWEEN"
case qbtypesv5.FilterOperatorLike:
return "LIKE"
case qbtypesv5.FilterOperatorNotLike:
return "NOT LIKE"
case qbtypesv5.FilterOperatorILike:
return "ILIKE"
case qbtypesv5.FilterOperatorNotILike:
return "NOT ILIKE"
case qbtypesv5.FilterOperatorContains:
return "CONTAINS"
case qbtypesv5.FilterOperatorNotContains:
return "NOT CONTAINS"
case qbtypesv5.FilterOperatorRegexp:
return "REGEXP"
case qbtypesv5.FilterOperatorNotRegexp:
return "NOT REGEXP"
case qbtypesv5.FilterOperatorIn:
return "IN"
case qbtypesv5.FilterOperatorNotIn:
return "NOT IN"
case qbtypesv5.FilterOperatorExists:
return "EXISTS"
case qbtypesv5.FilterOperatorNotExists:
return "NOT EXISTS"
}
return "?"
}
func trimQuotes(s string) string {
if len(s) >= 2 {
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
s = s[1 : len(s)-1]
}
}
s = strings.ReplaceAll(s, `\\`, `\`)
s = strings.ReplaceAll(s, `\'`, `'`)
return s
}

View File

@@ -253,6 +253,7 @@ func NewSQLMigrationProviderFactories(
sqlmigration.NewAddQuickFilterTuplesFactory(sqlstore),
sqlmigration.NewAddIngestionTuplesFactory(sqlstore),
sqlmigration.NewAddSubscriptionTuplesFactory(sqlstore),
sqlmigration.NewNormalizeQuickFilterFieldsFactory(sqlstore),
)
}

View File

@@ -0,0 +1,161 @@
package sqlmigration
import (
"context"
"encoding/json"
"log/slog"
"github.com/uptrace/bun"
"github.com/uptrace/bun/migrate"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/sqlstore"
)
type quickFilterSourceRow struct {
bun.BaseModel `bun:"table:quick_filter"`
ID string `bun:"id,pk"`
Source string `bun:"source"`
Filter string `bun:"filter"`
}
type quickFilterStaticField struct {
name string
fieldContext string
fieldDataType string
}
// quickFilterSpanFields are the span-level fields the fields API serves with
// the span context, keyed by every name a stored filter may carry for them.
var quickFilterSpanFields = func() map[string]quickFilterStaticField {
fields := map[string]quickFilterStaticField{}
for name, dataType := range map[string]string{
"trace_id": "string", "span_id": "string", "trace_state": "string", "parent_span_id": "string",
"flags": "number", "name": "string", "kind": "number", "kind_string": "string",
"duration_nano": "number", "status_code": "number", "status_message": "string", "status_code_string": "string",
"response_status_code": "string", "external_http_url": "string", "http_url": "string",
"external_http_method": "string", "http_method": "string", "http_host": "string",
"db_name": "string", "db_operation": "string", "has_error": "bool", "is_remote": "string",
} {
fields[name] = quickFilterStaticField{name: name, fieldContext: "span", fieldDataType: dataType}
}
for deprecated, current := range map[string]string{
"responseStatusCode": "response_status_code", "externalHttpUrl": "external_http_url", "httpUrl": "http_url",
"externalHttpMethod": "external_http_method", "httpMethod": "http_method", "httpHost": "http_host",
"dbName": "db_name", "dbOperation": "db_operation", "hasError": "has_error", "isRemote": "is_remote",
} {
fields[deprecated] = fields[current]
}
return fields
}()
// quickFilterLogFields are the log-level fields the fields API serves with
// the log context.
var quickFilterLogFields = map[string]quickFilterStaticField{
"body": {name: "body", fieldContext: "log", fieldDataType: "string"},
"severity_text": {name: "severity_text", fieldContext: "log", fieldDataType: "string"},
"severity_number": {name: "severity_number", fieldContext: "log", fieldDataType: "number"},
"trace_id": {name: "trace_id", fieldContext: "log", fieldDataType: "string"},
"span_id": {name: "span_id", fieldContext: "log", fieldDataType: "string"},
"trace_flags": {name: "trace_flags", fieldContext: "log", fieldDataType: "number"},
}
type normalizeQuickFilterFields struct {
settings factory.ProviderSettings
}
func NewNormalizeQuickFilterFieldsFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
return factory.NewProviderFactory(factory.MustNewName("normalize_quick_filter_fields"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
return &normalizeQuickFilterFields{settings: ps}, nil
})
}
func (migration *normalizeQuickFilterFields) Register(migrations *migrate.Migrations) error {
return migrations.Register(migration.Up, migration.Down)
}
func (migration *normalizeQuickFilterFields) Up(ctx context.Context, db *bun.DB) error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
var rows []*quickFilterSourceRow
if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
return err
}
var migrated, skipped int
for _, row := range rows {
normalized, changed, ok := normalizeQuickFilterEntries(row.Source, row.Filter)
if !ok {
migration.settings.Logger.WarnContext(ctx, "quick filter could not be parsed, leaving it untouched", slog.String("quick_filter_id", row.ID), slog.String("raw_filter", row.Filter))
skipped++
continue
}
if !changed {
continue
}
migrated++
if _, err := tx.NewUpdate().Model((*quickFilterSourceRow)(nil)).Set("filter = ?", normalized).Where("id = ?", row.ID).Exec(ctx); err != nil {
return err
}
}
migration.settings.Logger.InfoContext(ctx, "normalized quick filter static fields", slog.Int("total", len(rows)), slog.Int("migrated", migrated), slog.Int("skipped", skipped))
return tx.Commit()
}
func (migration *normalizeQuickFilterFields) Down(context.Context, *bun.DB) error {
return nil
}
// normalizeQuickFilterEntries rewrites the static fields of a stored filter
// list to the name, context and data type the fields API serves them with:
// span fields for the trace-based sources, log fields for logs, whatever
// context the legacy seeds gave them. Other keys are left as they are;
// ok=false means unparseable.
func normalizeQuickFilterEntries(source string, filter string) (normalized string, changed bool, ok bool) {
var staticFields map[string]quickFilterStaticField
switch source {
case "traces", "api_monitoring", "exceptions", "ai_observability":
staticFields = quickFilterSpanFields
case "logs":
staticFields = quickFilterLogFields
default:
return "", false, true
}
var entries []telemetryFieldKeyOutput
if err := json.Unmarshal([]byte(filter), &entries); err != nil {
return "", false, false
}
for i, entry := range entries {
field, static := staticFields[entry.Name]
if !static {
continue
}
if entry.Name == field.name && entry.FieldContext == field.fieldContext && entry.FieldDataType == field.fieldDataType {
continue
}
entries[i].Name = field.name
entries[i].FieldContext = field.fieldContext
entries[i].FieldDataType = field.fieldDataType
changed = true
}
if !changed {
return "", false, true
}
normalizedJSON, err := marshalUnescaped(entries)
if err != nil {
return "", false, false
}
return string(normalizedJSON), true, true
}

View File

@@ -0,0 +1,61 @@
package telemetrymetadata
import (
"strconv"
"strings"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// boolFieldValues is the suggestion set for a bool field, optionally narrowed
// by the search text.
func boolFieldValues(searchText string) *telemetrytypes.TelemetryFieldValues {
values := &telemetrytypes.TelemetryFieldValues{}
needle := strings.ToLower(searchText)
for _, v := range []bool{true, false} {
if needle == "" || strings.Contains(strconv.FormatBool(v), needle) {
values.BoolValues = append(values.BoolValues, v)
}
}
return values
}
// spanSearchScopeFieldValues is the suggestion set for a search-scope selector
// (isRoot, isEntryPoint), which only filters with true. ok is false for any
// other name.
func spanSearchScopeFieldValues(name, searchText string) (*telemetrytypes.TelemetryFieldValues, bool) {
for scopeName := range tracestelemetryschema.SpanSearchScopeFields {
if !strings.EqualFold(scopeName, name) {
continue
}
values := &telemetrytypes.TelemetryFieldValues{}
if needle := strings.ToLower(searchText); needle == "" || strings.Contains("true", needle) {
values.BoolValues = []bool{true}
}
return values, true
}
return nil, false
}
// isKnownBoolField is true when the caller asked for the bool data type, or
// when the name is one of the signal's static bool fields and the requested
// context does not rule that static field out.
func isKnownBoolField(selector *telemetrytypes.FieldValueSelector, staticFields ...map[string]telemetrytypes.TelemetryFieldKey) bool {
if selector.FieldDataType == telemetrytypes.FieldDataTypeBool {
return true
}
if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
return false
}
for _, fields := range staticFields {
field, ok := fields[selector.Name]
if !ok || field.FieldDataType != telemetrytypes.FieldDataTypeBool {
continue
}
if selector.FieldContext == telemetrytypes.FieldContextUnspecified || selector.FieldContext == field.FieldContext {
return true
}
}
return false
}

View File

@@ -187,8 +187,6 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
).From(t.tracesDBName + "." + t.spanAttributesKeysTblName)
var limit int
searchTexts := []string{}
conds := []string{}
for _, fieldKeySelector := range fieldKeySelectors {
@@ -208,14 +206,12 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
fieldKeyConds = append(fieldKeyConds, sb.ILike("tagKey", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
}
searchTexts = append(searchTexts, fieldKeySelector.Name)
// now look at the field context
// we don't write most of intrinsic fields to keys table
// for this reason we don't want to apply tagType if the field context
// is not attribute or resource attribute
if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified &&
(fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute ||
fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) {
// is not attribute, resource attribute or scope
switch fieldKeySelector.FieldContext {
case telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope:
fieldKeyConds = append(fieldKeyConds, sb.E("tagType", fieldKeySelector.FieldContext.TagType()))
}
@@ -288,41 +284,20 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
// hit the limit? (only counting DB results)
complete := rowCount <= limit
staticKeys := []string{"isRoot", "isEntryPoint"}
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.CalculatedFields)...)
// Add the matching static fields: the span scope selectors, the intrinsic
// columns and the calculated columns. These don't count towards the limit
staticFields := maps.Values(tracestelemetryschema.SpanSearchScopeFields)
staticFields = append(staticFields, maps.Values(tracestelemetryschema.IntrinsicFields)...)
staticFields = append(staticFields, maps.Values(tracestelemetryschema.CalculatedFields)...)
// Add matching intrinsic and matching calculated fields
// These don't count towards the limit
for _, key := range staticKeys {
found := false
for _, v := range searchTexts {
if v == "" || strings.Contains(key, v) {
found = true
break
}
for _, field := range staticFields {
if !staticFieldMatchesAny(field, fieldKeySelectors) {
continue
}
if found {
if field, exists := tracestelemetryschema.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
continue
}
if field, exists := tracestelemetryschema.CalculatedFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
continue
}
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
Name: key,
FieldContext: telemetrytypes.FieldContextSpan,
Signal: telemetrytypes.SignalTraces,
})
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
continue
}
keys = append(keys, &field)
}
if err = t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
@@ -542,12 +517,6 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
allArgs = append(allArgs, args...)
}
if len(queries) == 0 {
// No matching contexts, return empty result
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
}
// Combine queries with UNION ALL
var limit int
for _, fieldKeySelector := range fieldKeySelectors {
limit += fieldKeySelector.Limit
@@ -556,7 +525,15 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
limit = 1000
}
mainQuery := fmt.Sprintf(`
keys := []*telemetrytypes.TelemetryFieldKey{}
parentTypes := make(map[string][]telemetrytypes.FieldDataType)
rowCount := 0
// the log and scope contexts have no keys table; they are served by the
// static fields appended below
if len(queries) > 0 {
// Combine queries with UNION ALL
mainQuery := fmt.Sprintf(`
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
FROM (
%s
@@ -566,103 +543,75 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
LIMIT %d
`, strings.Join(queries, " UNION ALL "), limit+1)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
defer rows.Close()
keys := []*telemetrytypes.TelemetryFieldKey{}
parentTypes := make(map[string][]telemetrytypes.FieldDataType)
rowCount := 0
searchTexts := []string{}
// Collect search texts for static field matching
for _, fieldKeySelector := range fieldKeySelectors {
searchTexts = append(searchTexts, fieldKeySelector.Name)
}
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
break
}
var name string
var fieldContext telemetrytypes.FieldContext
var fieldDataType telemetrytypes.FieldDataType
var priority uint8
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
defer rows.Close()
// ArrayJSON/ArrayDynamic body rows for parent paths are needed by the JSON access plan
// builder (enrichJSONKeys). Always record them in parentTypes. Only skip adding to keys
// if the user did not also directly request this name — a field like "education" can be
// both a parent of "education[].name" and an explicitly queried field in its own right.
switch fieldDataType {
case telemetrytypes.FieldDataTypeArrayJSON, telemetrytypes.FieldDataTypeArrayDynamic:
if fieldContext == telemetrytypes.FieldContextBody && parentPaths[name] {
parentTypes[name] = append(parentTypes[name], fieldDataType)
if !mapOfRequestedSelectors[name] {
continue // skip; don't register the key.
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
break
}
var name string
var fieldContext telemetrytypes.FieldContext
var fieldDataType telemetrytypes.FieldDataType
var priority uint8
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
// ArrayJSON/ArrayDynamic body rows for parent paths are needed by the JSON access plan
// builder (enrichJSONKeys). Always record them in parentTypes. Only skip adding to keys
// if the user did not also directly request this name — a field like "education" can be
// both a parent of "education[].name" and an explicitly queried field in its own right.
switch fieldDataType {
case telemetrytypes.FieldDataTypeArrayJSON, telemetrytypes.FieldDataTypeArrayDynamic:
if fieldContext == telemetrytypes.FieldContextBody && parentPaths[name] {
parentTypes[name] = append(parentTypes[name], fieldDataType)
if !mapOfRequestedSelectors[name] {
continue // skip; don't register the key.
}
}
}
}
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
// if there is no materialised column, create a key with the field context and data type
if !ok {
key = &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
// if there is no materialised column, create a key with the field context and data type
if !ok {
key = &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
}
}
keys = append(keys, key)
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
}
keys = append(keys, key)
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
}
if rows.Err() != nil {
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
if rows.Err() != nil {
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
}
// hit the limit? (only counting DB results)
complete := rowCount <= limit
staticKeys := []string{}
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
// Add matching intrinsic and matching calculated fields
// These don't count towards the limit
for _, key := range staticKeys {
found := false
for _, v := range searchTexts {
if v == "" || strings.Contains(key, v) {
found = true
break
}
// Add the matching intrinsic columns. These don't count towards the limit
for _, field := range maps.Values(logstelemetryschema.IntrinsicFields) {
if !staticFieldMatchesAny(field, fieldKeySelectors) {
continue
}
if found {
if field, exists := logstelemetryschema.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
continue
}
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
Name: key,
FieldContext: telemetrytypes.FieldContextLog,
Signal: telemetrytypes.SignalLogs,
})
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
continue
}
keys = append(keys, &field)
}
// enrich body keys with promoted paths, indexes, and JSON access plans
@@ -806,10 +755,6 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
allArgs = append(allArgs, args...)
}
if len(queries) == 0 {
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
}
var limit int
for _, fieldKeySelector := range fieldKeySelectors {
limit += fieldKeySelector.Limit
@@ -818,7 +763,13 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
limit = 1000
}
mainQuery := fmt.Sprintf(`
keys := []*telemetrytypes.TelemetryFieldKey{}
rowCount := 0
// the log and scope contexts have no keys table; they are served by the
// static fields appended below
if len(queries) > 0 {
mainQuery := fmt.Sprintf(`
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
FROM (
%s
@@ -828,73 +779,57 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
LIMIT %d
`, strings.Join(queries, " UNION ALL "), limit+1)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
defer rows.Close()
keys := []*telemetrytypes.TelemetryFieldKey{}
rowCount := 0
searchTexts := []string{}
for _, fieldKeySelector := range fieldKeySelectors {
searchTexts = append(searchTexts, fieldKeySelector.Name)
}
for rows.Next() {
rowCount++
if rowCount > limit {
break
}
var name string
var fieldContext telemetrytypes.FieldContext
var fieldDataType telemetrytypes.FieldDataType
var priority uint8
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
defer rows.Close()
if !ok {
key = &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
for rows.Next() {
rowCount++
if rowCount > limit {
break
}
var name string
var fieldContext telemetrytypes.FieldContext
var fieldDataType telemetrytypes.FieldDataType
var priority uint8
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
if !ok {
key = &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
}
}
keys = append(keys, key)
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
}
keys = append(keys, key)
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
}
if rows.Err() != nil {
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
if rows.Err() != nil {
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
}
complete := rowCount <= limit
// Add intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
staticKeys := maps.Keys(audittelemetryschema.IntrinsicFields)
for _, key := range staticKeys {
found := false
for _, v := range searchTexts {
if v == "" || strings.Contains(key, v) {
found = true
break
}
// Add the matching intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
for _, field := range maps.Values(audittelemetryschema.IntrinsicFields) {
if !staticFieldMatchesAny(field, fieldKeySelectors) {
continue
}
if found {
if field, exists := audittelemetryschema.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
}
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
continue
}
keys = append(keys, &field)
}
return keys, complete, nil
@@ -1091,9 +1026,12 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterKeys.Error())
}
// meter labels are stored as strings in the labels JSON and have no
// attribute context, so only the data type is known
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalMetrics,
Name: name,
Signal: telemetrytypes.SignalMetrics,
FieldDataType: telemetrytypes.FieldDataTypeString,
})
}
@@ -1506,88 +1444,13 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: "getSpanFieldValues",
})
// build the query to get the keys from the spans that match the field selection criteria
limit := fieldValueSelector.Limit
if limit == 0 {
limit = 50
if values, ok := spanSearchScopeFieldValues(fieldValueSelector.Name, fieldValueSelector.Value); ok {
return values, true, nil
}
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.tracesDBName + "." + t.tracesFieldsTblName)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
}
// now look at the field context
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
}
// now look at the field data type
if fieldValueSelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
sb.Where(sb.E("tag_data_type", fieldValueSelector.FieldDataType.TagDataType()))
}
if fieldValueSelector.Value != "" {
switch fieldValueSelector.FieldDataType {
case telemetrytypes.FieldDataTypeString:
sb.Where(sb.ILike("string_value", "%"+escapeForLike(fieldValueSelector.Value)+"%"))
case telemetrytypes.FieldDataTypeNumber:
sb.Where(sb.IsNotNull("number_value"))
sb.Where(sb.ILike("toString(number_value)", "%"+escapeForLike(fieldValueSelector.Value)+"%"))
case telemetrytypes.FieldDataTypeUnspecified:
// or b/w string and number
sb.Where(sb.Or(
sb.ILike("string_value", "%"+escapeForLike(fieldValueSelector.Value)+"%"),
sb.ILike("toString(number_value)", "%"+escapeForLike(fieldValueSelector.Value)+"%"),
))
}
}
// query one extra to check if we hit the limit
sb.Limit(limit + 1)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
defer rows.Close()
values := &telemetrytypes.TelemetryFieldValues{}
seen := make(map[string]bool)
rowCount := 0
totalCount := 0 // Track total unique values
for rows.Next() {
rowCount++
var stringValue string
var numberValue float64
if err := rows.Scan(&stringValue, &numberValue); err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
// Only add values if we haven't hit the limit yet
if totalCount < limit {
if _, ok := seen[stringValue]; !ok && stringValue != "" {
values.StringValues = append(values.StringValues, stringValue)
seen[stringValue] = true
totalCount++
}
if _, ok := seen[fmt.Sprintf("%f", numberValue)]; !ok && numberValue != 0 && totalCount < limit {
values.NumberValues = append(values.NumberValues, numberValue)
seen[fmt.Sprintf("%f", numberValue)] = true
totalCount++
}
}
}
// hit the limit?
complete := rowCount <= limit
return values, complete, nil
knownBool := isKnownBoolField(fieldValueSelector, tracestelemetryschema.IntrinsicFields, tracestelemetryschema.CalculatedFields)
// unix_milli is the hour of the span start
return t.getTagTableValues(ctx, t.tracesDBName+"."+t.tracesFieldsTblName, fieldValueSelector, knownBool)
}
func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) {
@@ -1596,17 +1459,77 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: "getLogFieldValues",
})
// build the query to get the keys from the spans that match the field selection criteria
knownBool := isKnownBoolField(fieldValueSelector, logstelemetryschema.IntrinsicFields)
// unix_milli is the hour the log was ingested, not the log's own timestamp
return t.getTagTableValues(ctx, t.logsDBName+"."+t.logsFieldsTblName, fieldValueSelector, knownBool)
}
// tagTableSinceDay restricts rows to the tag table's day partitions from the
// start's day on. Partitions are toDate(unix_milli / 1000) in the server's
// timezone, and a value's surviving row within a day carries whichever hour
// was inserted last, so the day is the finest safe unit.
func tagTableSinceDay(sb *sqlbuilder.SelectBuilder, startUnixMilli int64) {
if startUnixMilli != 0 {
sb.Where(fmt.Sprintf("toDate(unix_milli / 1000) >= toDate(%d)", startUnixMilli/1000))
}
}
// tagTableHasBoolRows reports whether the tag table holds a bool row for the
// key. Bool rows carry no value, so one row is enough to know the key takes
// the values true and false.
func (t *telemetryMetaStore) tagTableHasBoolRows(ctx context.Context, table string, selector *telemetrytypes.FieldValueSelector) (bool, error) {
sb := sqlbuilder.Select("1").From(table)
sb.Where(sb.E("tag_key", selector.Name))
sb.Where(sb.E("tag_data_type", telemetrytypes.FieldDataTypeBool.TagDataType()))
if selector.FieldContext != telemetrytypes.FieldContextUnspecified {
sb.Where(sb.E("tag_type", selector.FieldContext.TagType()))
}
tagTableSinceDay(sb, selector.StartUnixMilli)
sb.Limit(1)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
defer rows.Close()
return rows.Next(), rows.Err()
}
// getTagTableValues returns the string and number values of the key from a
// tag table, and true and false when the key is a known bool field or the
// table holds bool rows for it. Bool rows do not count towards the limit.
func (t *telemetryMetaStore) getTagTableValues(ctx context.Context, table string, fieldValueSelector *telemetrytypes.FieldValueSelector, knownBool bool) (*telemetrytypes.TelemetryFieldValues, bool, error) {
limit := fieldValueSelector.Limit
if limit == 0 {
limit = 50
}
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.logsDBName + "." + t.logsFieldsTblName)
values := &telemetrytypes.TelemetryFieldValues{}
if knownBool {
values.BoolValues = boolFieldValues(fieldValueSelector.Value).BoolValues
if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeBool {
return values, true, nil
}
} else if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeUnspecified {
hasBoolRows, err := t.tagTableHasBoolRows(ctx, table, fieldValueSelector)
if err != nil {
return nil, false, err
}
if hasBoolRows {
values.BoolValues = boolFieldValues(fieldValueSelector.Value).BoolValues
}
}
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(table)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
}
sb.Where(sb.NE("tag_data_type", telemetrytypes.FieldDataTypeBool.TagDataType()))
tagTableSinceDay(sb, fieldValueSelector.StartUnixMilli)
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
@@ -1643,7 +1566,6 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
}
defer rows.Close()
values := &telemetrytypes.TelemetryFieldValues{}
seen := make(map[string]bool)
rowCount := 0
totalCount := 0 // Track total unique values
@@ -2097,6 +2019,18 @@ func populateAllUnspecifiedValues(allUnspecifiedValues *telemetrytypes.Telemetry
}
}
for _, value := range values.BoolValues {
if totalCount >= limit {
complete = false
break
}
if _, ok := mapOfValues[value]; !ok {
mapOfValues[value] = true
allUnspecifiedValues.BoolValues = append(allUnspecifiedValues.BoolValues, value)
totalCount++
}
}
for _, value := range values.RelatedValues {
if totalCount >= limit {
complete = false
@@ -2467,6 +2401,10 @@ func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Conte
// updateColumnEvolutionMetadataForKeys updates the evolution field for keys.
func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) error {
// an empty selector list would run the evolution query without a filter
if len(keysToUpdate) == 0 {
return nil
}
var metadataKeySelectors []*telemetrytypes.EvolutionSelector
for _, keySelector := range keysToUpdate {

View File

@@ -0,0 +1,53 @@
package telemetrymetadata
import (
"strings"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
func staticFieldMatchesAny(field telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) bool {
for _, selector := range selectors {
if staticFieldMatches(field, selector) {
return true
}
}
return false
}
// staticFieldMatches mirrors the keys-table lookup for a static field: the
// requested context and data type, when given, must agree with the field's,
// and the name matches case-insensitively, as a substring for fuzzy selectors
// and as the whole name for exact ones.
func staticFieldMatches(field telemetrytypes.TelemetryFieldKey, selector *telemetrytypes.FieldKeySelector) bool {
if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != field.FieldContext {
return false
}
if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && !sameDataTypeFamily(selector.FieldDataType, field.FieldDataType) {
return false
}
if selector.Name == "" {
return true
}
if selector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
return strings.EqualFold(field.Name, selector.Name)
}
return strings.Contains(strings.ToLower(field.Name), strings.ToLower(selector.Name))
}
// sameDataTypeFamily treats the numeric types as one family: static fields
// declare "number" while callers may ask for int64 or float64.
func sameDataTypeFamily(requested, actual telemetrytypes.FieldDataType) bool {
if requested == actual {
return true
}
return isNumericDataType(requested) && isNumericDataType(actual)
}
func isNumericDataType(dataType telemetrytypes.FieldDataType) bool {
switch dataType {
case telemetrytypes.FieldDataTypeNumber, telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64:
return true
}
return false
}

View File

@@ -392,6 +392,24 @@ var (
SpanSearchScopeRoot = "isroot"
SpanSearchScopeEntryPoint = "isentrypoint"
// SpanSearchScopeFields are the search-scope selectors (isRoot, isEntryPoint),
// not columns and unrelated to the instrumentation scope: they only filter
// with the value true.
SpanSearchScopeFields = map[string]telemetrytypes.TelemetryFieldKey{
"isRoot": {
Name: "isRoot",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeBool,
},
"isEntryPoint": {
Name: "isEntryPoint",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeBool,
},
}
// IntrinsicSpanFields lists the intrinsic span columns, in the order they
// should appear when a raw query expands its SelectFields.
IntrinsicSpanFields = []telemetrytypes.TelemetryFieldKey{

View File

@@ -173,18 +173,18 @@ func NewSourceFilterFromStorableQuickFilter(storableQuickFilter *StorableQuickFi
// NewDefaultQuickFilter generates default filters for all supported sources.
func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
tracesFilters := []telemetrytypes.TelemetryFieldKey{
{Name: "duration_nano", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber},
{Name: "duration_nano", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeNumber},
{Name: "deployment.environment", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "hasError", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeBool},
{Name: "has_error", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeBool},
{Name: "service.name", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "name", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "name", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "rpc.method", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "response_status_code", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "http_host", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "response_status_code", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "http_host", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "http.method", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "http.route", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "http_url", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "trace_id", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "http_url", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "trace_id", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
}
logsFilters := []telemetrytypes.TelemetryFieldKey{

View File

@@ -0,0 +1,235 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.traces import Traces
@pytest.mark.parametrize(
"signal,field_context,present,absent",
[
pytest.param("logs", "log", {"severity_text": "log", "body": "log", "trace_id": "log"}, ["code.file", "scope_name"], id="log_context_lists_log_intrinsics"),
pytest.param("logs", "scope", {"scope_name": "scope", "scope_version": "scope"}, ["severity_text", "body", "code.file"], id="scope_context_lists_scope_intrinsics_for_logs"),
pytest.param("logs", "attribute", {"code.file": "attribute"}, ["body", "scope_name"], id="attribute_context_excludes_log_intrinsics"),
pytest.param("traces", "span", {"name": "span", "has_error": "span", "isRoot": "span", "http.method": "attribute"}, ["scope.name"], id="span_context_lists_span_intrinsics_and_attributes"),
pytest.param("traces", "scope", {"scope.name": "scope", "scope.version": "scope"}, ["name", "has_error", "isRoot", "http.method", "host.name"], id="scope_context_lists_scope_intrinsics_for_traces"),
pytest.param("traces", "resource", {"host.name": "resource"}, ["name", "has_error", "isRoot", "http.method"], id="resource_context_excludes_span_intrinsics"),
pytest.param("traces", "attribute", {"http.method": "attribute"}, ["name", "has_error", "isRoot", "host.name"], id="attribute_context_excludes_span_intrinsics"),
],
)
def test_fields_keys_by_context(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
insert_traces: Callable[[list[Traces]], None],
signal: str,
field_context: str,
present: dict[str, str],
absent: list[str],
) -> None:
"""
Setup:
Insert a log with a code.file attribute and a span with an http.method attribute and a host.name resource.
Tests:
1. Keys for a context list that context's intrinsic columns and the stored keys the context maps to,
each with its context; intrinsics of other contexts are not listed. The span context also keeps
listing attributes because `span.<attribute>` resolves attributes in queries.
"""
now = datetime.now(tz=UTC)
insert_logs([Logs(timestamp=now, attributes={"code.file": "/opt/integration.go"}, body="a log line")])
insert_traces([Traces(timestamp=now, resources={"host.name": "linux-001"}, attributes={"http.method": "GET"})])
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, "fieldContext": field_context},
)
assert response.status_code == HTTPStatus.OK
keys = response.json()["data"]["keys"]
listed = {name: [key["fieldContext"] for key in keys.get(name, [])] for name in present}
assert listed == {name: [context] for name, context in present.items()}, f"keys for the {field_context} context"
assert [name for name in absent if name in keys] == [], f"keys that do not belong to the {field_context} context"
@pytest.mark.parametrize(
"signal,field_context,field_data_type,present,absent",
[
pytest.param("traces", "span", "float64", ["duration_nano", "status_code"], ["name", "has_error"], id="float64_matches_number_span_intrinsics"),
pytest.param("traces", "span", "int64", ["duration_nano", "status_code"], ["name", "has_error"], id="int64_matches_number_span_intrinsics"),
pytest.param("traces", "span", "bool", ["has_error", "isRoot", "isEntryPoint"], ["name", "duration_nano"], id="bool_matches_bool_span_intrinsics"),
pytest.param("traces", "span", "string", ["name", "http_method"], ["duration_nano", "has_error"], id="string_matches_string_span_intrinsics"),
pytest.param("logs", "log", "number", ["severity_number", "trace_flags"], ["severity_text", "body"], id="number_matches_number_log_intrinsics"),
],
)
def test_fields_keys_by_data_type(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
signal: str,
field_context: str,
field_data_type: str,
present: list[str],
absent: list[str],
) -> None:
"""
Tests:
1. A data type filter keeps the intrinsic columns of that type; number, int64 and float64 are one family.
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, "fieldContext": field_context, "fieldDataType": field_data_type},
)
assert response.status_code == HTTPStatus.OK
keys = response.json()["data"]["keys"]
assert [name for name in present if name not in keys] == [], f"intrinsics of type {field_data_type}"
assert [name for name in absent if name in keys] == [], f"intrinsics not of type {field_data_type}"
@pytest.mark.parametrize(
"signal,search_text,present",
[
pytest.param("logs", "SEVERITY", ["severity_text", "severity_number"], id="upper_case_search_logs"),
pytest.param("traces", "HTTP_", ["http_method", "http_host", "http_url"], id="upper_case_search_traces"),
pytest.param("traces", "Duration", ["duration_nano"], id="mixed_case_search_traces"),
pytest.param("traces", "span.HAS_ERR", ["has_error"], id="context_prefix_with_upper_case_search"),
],
)
def test_fields_keys_search_matches_intrinsics_case_insensitively(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
signal: str,
search_text: str,
present: list[str],
) -> None:
"""
Tests:
1. The search text matches intrinsic columns case-insensitively, as it does for stored keys,
with or without a context prefix.
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, "searchText": search_text},
)
assert response.status_code == HTTPStatus.OK
keys = response.json()["data"]["keys"]
assert [name for name in present if name not in keys] == [], f"intrinsics matching {search_text!r}"
@pytest.mark.parametrize(
"signal,params,expected",
[
pytest.param("traces", {"name": "has_error"}, [True, False], id="calculated_bool_span_field"),
pytest.param("traces", {"name": "has_error", "fieldContext": "span"}, [True, False], id="calculated_bool_span_field_with_context"),
pytest.param("traces", {"name": "has_error", "searchText": "tr"}, [True], id="search_text_narrows_bool_values"),
pytest.param("traces", {"name": "isRoot"}, [True], id="span_scope_field_is_true_only"),
pytest.param("logs", {"name": "retry"}, [True, False], id="bool_attribute_from_tag_rows"),
pytest.param("logs", {"name": "retry", "fieldContext": "attribute"}, [True, False], id="bool_attribute_with_context"),
pytest.param("logs", {"name": "retry", "searchText": "tr"}, [True], id="search_text_narrows_stored_bool_values"),
pytest.param("logs", {"name": "never_seen", "fieldDataType": "bool"}, [True, False], id="declared_bool_type_needs_no_rows"),
],
)
def test_fields_values_bool_fields(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
signal: str,
params: dict[str, str],
expected: list[bool],
) -> None:
"""
Setup:
Insert a log with a bool attribute.
Tests:
1. Values for a bool field are true and false (narrowed by the search text): for the calculated span
field, for a stored bool attribute whose tag rows carry no value, and for a key the caller
declares bool.
2. A span scope selector (isRoot) only takes true.
"""
insert_logs([Logs(timestamp=datetime.now(tz=UTC), attributes={"retry": True}, body="retrying")])
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, **params},
)
assert response.status_code == HTTPStatus.OK
assert response.json()["data"]["values"]["boolValues"] == expected
assert response.json()["data"]["complete"] is True
def test_fields_values_start_excludes_span_values_not_seen_since_the_day(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert a span three days old and a span now, with different service names.
Tests:
1. Values with startUnixMilli an hour ago contain only the service seen today: the start is
floored to the day, the tag table's deduplication unit.
2. Values without a start contain both services.
Logs are not covered: the logs collector stamps tag rows with the ingestion hour, not the
log's timestamp, and the fixture writes the log's timestamp.
"""
signal = "traces"
now = datetime.now(tz=UTC)
insert_traces(
[
Traces(timestamp=now - timedelta(days=3), resources={"service.name": "archived-service"}),
Traces(timestamp=now, resources={"service.name": "live-service"}),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={
"signal": signal,
"name": "service.name",
"startUnixMilli": int((now - timedelta(hours=1)).timestamp() * 1000),
},
)
assert response.status_code == HTTPStatus.OK
assert response.json()["data"]["values"]["stringValues"] == ["live-service"], "values last seen before the start must be dropped"
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, "name": "service.name"},
)
assert response.status_code == HTTPStatus.OK
assert set(response.json()["data"]["values"]["stringValues"]) == {"archived-service", "live-service"}

View File

@@ -71,7 +71,7 @@ def test_v1_get_serves_legacy_shape(
assert response.status_code == HTTPStatus.OK, response.text
filters = response.json()["data"]["filters"]
assert filters[0]["key"] == "duration_nano"
assert filters[0]["type"] == "tag"
assert filters[0]["type"] == "", "span fields have no v3 attribute type"
assert filters[0]["dataType"] == "float64"
assert all("name" not in legacy_filter for legacy_filter in filters)
@@ -274,3 +274,36 @@ def test_update_quick_filters_rejects_invalid_input(
timeout=2,
)
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
def test_default_traces_filters_are_served_as_the_fields_api_serves_them(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
):
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v2/quick_filters/traces"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=2,
)
assert response.status_code == HTTPStatus.OK, response.text
filters = {field_key["name"]: field_key for field_key in response.json()["data"]["filters"]}
assert "hasError" not in filters
assert (filters["has_error"]["fieldContext"], filters["has_error"]["fieldDataType"]) == ("span", "bool")
assert (filters["name"]["fieldContext"], filters["name"]["fieldDataType"]) == ("span", "string")
assert (filters["duration_nano"]["fieldContext"], filters["duration_nano"]["fieldDataType"]) == ("span", "number")
assert (filters["http.route"]["fieldContext"], filters["http.route"]["fieldDataType"]) == ("attribute", "string")
for name in ("has_error", "name"):
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=2,
params={"signal": "traces", "searchText": name, "fieldContext": filters[name]["fieldContext"]},
)
assert response.status_code == HTTPStatus.OK, response.text
served = response.json()["data"]["keys"][name]
assert (filters[name]["fieldContext"], filters[name]["fieldDataType"]) in [(key["fieldContext"], key["fieldDataType"]) for key in served]