mirror of
https://github.com/SigNoz/signoz.git
synced 2026-09-09 13:00:41 +01:00
Compare commits
3 Commits
feat/story
...
feat/commo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f945b5f513 | ||
|
|
8b0ab0ff26 | ||
|
|
d5af6f6d6b |
@@ -34,6 +34,7 @@ export function mockFieldsValuesAPI(response: {
|
||||
relatedValues?: (string | null)[];
|
||||
stringValues?: (string | null)[];
|
||||
numberValues?: (number | null)[];
|
||||
boolValues?: (boolean | null)[];
|
||||
}): void {
|
||||
server.use(
|
||||
rest.get('http://localhost/api/v1/fields/values', (_, res, ctx) =>
|
||||
@@ -46,6 +47,7 @@ export function mockFieldsValuesAPI(response: {
|
||||
relatedValues: response.relatedValues ?? [],
|
||||
stringValues: response.stringValues ?? [],
|
||||
numberValues: response.numberValues ?? [],
|
||||
boolValues: response.boolValues ?? [],
|
||||
},
|
||||
},
|
||||
}),
|
||||
|
||||
@@ -92,8 +92,12 @@ export function useFieldValues({
|
||||
values.numberValues
|
||||
?.filter((value): value is number => value !== null && value !== undefined)
|
||||
.map((value) => value.toString()) || [];
|
||||
const boolValues =
|
||||
values.boolValues
|
||||
?.filter((value): value is boolean => value !== null && value !== undefined)
|
||||
.map((value) => value.toString()) || [];
|
||||
|
||||
return [...stringValues, ...numberValues];
|
||||
return [...stringValues, ...numberValues, ...boolValues];
|
||||
}, [data]);
|
||||
|
||||
return { relatedValues, allValues, isLoading, isFetching };
|
||||
|
||||
@@ -8,6 +8,7 @@ import { FiltersType, IQuickFiltersConfig, SignalType } from './types';
|
||||
const FILTER_TITLE_MAP: Record<string, string> = {
|
||||
duration_nano: 'Duration',
|
||||
hasError: 'Has Error (Status)',
|
||||
has_error: 'Has Error (Status)',
|
||||
};
|
||||
|
||||
const FILTER_TYPE_MAP: Record<string, FiltersType> = {
|
||||
|
||||
@@ -4,41 +4,17 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/parser/filterquery/sqlcompiler"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
)
|
||||
|
||||
type Compiled struct {
|
||||
SQL string
|
||||
Args []any
|
||||
}
|
||||
|
||||
func (c Compiled) IsEmpty() bool {
|
||||
return c.SQL == ""
|
||||
}
|
||||
|
||||
// Compile always returns a non-nil *Compiled. An empty query (or one that
|
||||
// produces no SQL) yields a Compiled with an empty SQL — callers gate on
|
||||
// SQL != "" rather than a nil check.
|
||||
//
|
||||
// A `key OP value` term compiles to a DSL predicate; a bare word is a
|
||||
// case-insensitive substring search over the dashboard name, description, and tag
|
||||
// keys/values. They compose through AND/OR/NOT, so `prod payment` matches both
|
||||
// words (implicit AND) and `prod OR name = 'x'` mixes free text with a filter. A
|
||||
// quoted token matches literally, e.g. `"prod payment"`.
|
||||
func Compile(query string, formatter sqlstore.SQLFormatter) (*Compiled, error) {
|
||||
if len(strings.TrimSpace(query)) == 0 {
|
||||
return &Compiled{}, nil
|
||||
}
|
||||
|
||||
sql, args, errs := newVisitor(formatter).compile(query)
|
||||
// Compile wraps compiler errors in the dashboard list filter error code.
|
||||
func Compile(query string, formatter sqlstore.SQLFormatter) (*sqlcompiler.Compiled, error) {
|
||||
compiled, errs := sqlcompiler.Compile(query, formatter, dashboardFieldResolver{})
|
||||
if len(errs) > 0 {
|
||||
return nil, errors.NewInvalidInputf(dashboardtypes.ErrCodeDashboardListFilterInvalid,
|
||||
"invalid filter query: %s", strings.Join(errs, "; "))
|
||||
}
|
||||
|
||||
return &Compiled{
|
||||
SQL: sql,
|
||||
Args: args,
|
||||
}, nil
|
||||
return compiled, nil
|
||||
}
|
||||
|
||||
125
pkg/modules/dashboard/impldashboard/listfilter_resolver.go
Normal file
125
pkg/modules/dashboard/impldashboard/listfilter_resolver.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package impldashboard
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
"github.com/SigNoz/signoz/pkg/parser/filterquery/sqlcompiler"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
// dashboardFieldResolver maps dashboard list DSL keys; a non-reserved key is a tag key matched case-insensitively.
|
||||
type dashboardFieldResolver struct{}
|
||||
|
||||
func (r dashboardFieldResolver) ResolveComparison(b *sqlcompiler.Builder, rawKey string, operation qbtypesv5.FilterOperator, ctx *grammar.ComparisonContext) string {
|
||||
key := strings.ToLower(rawKey)
|
||||
|
||||
if allowedOperations, isReserved := dashboardtypes.ReservedOps[dashboardtypes.DSLKey(key)]; isReserved {
|
||||
return r.resolveReservedKey(b, ctx, operation, dashboardtypes.DSLKey(key), allowedOperations)
|
||||
}
|
||||
|
||||
if _, allowed := dashboardtypes.TagKeyOps[operation]; !allowed {
|
||||
b.AddError("operator %s is not allowed on a tag-key filter", sqlcompiler.OperationName(operation))
|
||||
return ""
|
||||
}
|
||||
return r.tagComparison(b, ctx, operation, key)
|
||||
}
|
||||
|
||||
func (r dashboardFieldResolver) resolveReservedKey(b *sqlcompiler.Builder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, allowedOperations map[qbtypesv5.FilterOperator]struct{}) string {
|
||||
if _, allowed := allowedOperations[operation]; !allowed {
|
||||
b.AddError("operator %s is not allowed for key %q", sqlcompiler.OperationName(operation), key)
|
||||
return ""
|
||||
}
|
||||
switch key {
|
||||
case dashboardtypes.DSLKeyName:
|
||||
columnExpression := string(b.Formatter().JSONExtractString("dashboard.data", "$.spec.display.name"))
|
||||
return b.StringOperation(b.SelectBuilder(), ctx, operation, columnExpression, string(key))
|
||||
case dashboardtypes.DSLKeyDescription:
|
||||
columnExpression := string(b.Formatter().JSONExtractString("dashboard.data", "$.spec.display.description"))
|
||||
return b.StringOperation(b.SelectBuilder(), ctx, operation, columnExpression, string(key))
|
||||
case dashboardtypes.DSLKeyCreatedAt:
|
||||
return b.TimestampComparison(ctx, operation, "dashboard.created_at")
|
||||
case dashboardtypes.DSLKeyUpdatedAt:
|
||||
return b.TimestampComparison(ctx, operation, "dashboard.updated_at")
|
||||
case dashboardtypes.DSLKeyCreatedBy:
|
||||
return b.StringOperation(b.SelectBuilder(), ctx, operation, "dashboard.created_by", string(key))
|
||||
case dashboardtypes.DSLKeyLocked:
|
||||
return b.BoolComparison(ctx, operation, "dashboard.locked")
|
||||
}
|
||||
b.AddError("no handler for reserved key %q", key)
|
||||
return ""
|
||||
}
|
||||
|
||||
func (dashboardFieldResolver) tagComparison(b *sqlcompiler.Builder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
|
||||
subqueryBuilder := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
if operation == qbtypesv5.FilterOperatorExists || operation == qbtypesv5.FilterOperatorNotExists {
|
||||
buildSubqueryForTagKey(subqueryBuilder, tagKey)
|
||||
} else {
|
||||
// Value predicates take the positive operator; negation toggles the EXISTS wrapper.
|
||||
positiveOperation := operation
|
||||
if operation.IsNegativeOperator() {
|
||||
positiveOperation = operation.Inverse()
|
||||
}
|
||||
valuePredicate := b.StringOperation(subqueryBuilder, ctx, positiveOperation, "t.value", tagKey)
|
||||
if valuePredicate == "" {
|
||||
return ""
|
||||
}
|
||||
buildSubqueryForTagKeyAndValue(subqueryBuilder, tagKey, valuePredicate)
|
||||
}
|
||||
|
||||
if operation.IsNegativeOperator() {
|
||||
return b.SelectBuilder().NotExists(subqueryBuilder)
|
||||
}
|
||||
return b.SelectBuilder().Exists(subqueryBuilder)
|
||||
}
|
||||
|
||||
func buildSubqueryForTagKey(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey string) *sqlbuilder.SelectBuilder {
|
||||
const dashboardTagKind = `"dashboard"`
|
||||
|
||||
return subqueryBuilder.
|
||||
Select("1").
|
||||
From("tag_relation tr").
|
||||
Join("tag t", "t.id = tr.tag_id").
|
||||
Where(
|
||||
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
|
||||
"tr.resource_id = dashboard.id",
|
||||
"LOWER(t.key) = LOWER("+subqueryBuilder.Var(tagKey)+")",
|
||||
)
|
||||
}
|
||||
|
||||
func buildSubqueryForTagKeyAndValue(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey, valuePredicate string) *sqlbuilder.SelectBuilder {
|
||||
return buildSubqueryForTagKey(subqueryBuilder, tagKey).Where(valuePredicate)
|
||||
}
|
||||
|
||||
// FreeText searches name, description and tag keys/values.
|
||||
func (dashboardFieldResolver) FreeText(b *sqlcompiler.Builder, value string) string {
|
||||
nameColumn := string(b.Formatter().JSONExtractString("dashboard.data", "$.spec.display.name"))
|
||||
descriptionColumn := string(b.Formatter().JSONExtractString("dashboard.data", "$.spec.display.description"))
|
||||
namePredicate := b.FreeTextContains(b.SelectBuilder(), nameColumn, value)
|
||||
descriptionPredicate := b.FreeTextContains(b.SelectBuilder(), descriptionColumn, value)
|
||||
|
||||
subqueryBuilder := sqlbuilder.NewSelectBuilder()
|
||||
keyPredicate := b.FreeTextContains(subqueryBuilder, "t.key", value)
|
||||
valuePredicate := b.FreeTextContains(subqueryBuilder, "t.value", value)
|
||||
buildSubqueryForFreeTextTag(subqueryBuilder, keyPredicate, valuePredicate)
|
||||
tagPredicate := b.SelectBuilder().Exists(subqueryBuilder)
|
||||
|
||||
return b.SelectBuilder().Or(namePredicate, descriptionPredicate, tagPredicate)
|
||||
}
|
||||
|
||||
func buildSubqueryForFreeTextTag(subqueryBuilder *sqlbuilder.SelectBuilder, keyPredicate, valuePredicate string) *sqlbuilder.SelectBuilder {
|
||||
const dashboardTagKind = `"dashboard"`
|
||||
|
||||
return subqueryBuilder.
|
||||
Select("1").
|
||||
From("tag_relation tr").
|
||||
Join("tag t", "t.id = tr.tag_id").
|
||||
Where(
|
||||
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
|
||||
"tr.resource_id = dashboard.id",
|
||||
subqueryBuilder.Or(keyPredicate, valuePredicate),
|
||||
)
|
||||
}
|
||||
@@ -559,6 +559,11 @@ func TestCompile_Rejections(t *testing.T) {
|
||||
dslQueryToCompile: `created_at >= 'not-a-date'`,
|
||||
expectedErrShouldContain: "RFC3339",
|
||||
},
|
||||
{
|
||||
subtestName: "rejects LIKE pattern ending in an unescaped backslash",
|
||||
dslQueryToCompile: `name LIKE 'prod\\'`,
|
||||
expectedErrShouldContain: "must not end with an unescaped backslash",
|
||||
},
|
||||
{
|
||||
subtestName: "rejects REGEXP — not yet supported",
|
||||
dslQueryToCompile: `name REGEXP '.*'`,
|
||||
@@ -573,7 +578,7 @@ func TestCompile_Rejections(t *testing.T) {
|
||||
}
|
||||
|
||||
// Every key in dashboardtypes.ReservedOps must have a matching case in
|
||||
// visitComparisonForReservedKeys; a key that's reserved but unhandled falls
|
||||
// resolveReservedKey; a key that's reserved but unhandled falls
|
||||
// through to the "no handler for reserved key" error. Equal is accepted by all
|
||||
// reserved keys, so `key = 'x'` always reaches the dispatch switch — a missing
|
||||
// handler surfaces as that error regardless of whether the value type-checks.
|
||||
@@ -583,7 +588,7 @@ func TestCompileReservedKeysAllHandled(t *testing.T) {
|
||||
_, err := Compile(string(key)+` = 'x'`, formatter(t))
|
||||
if err != nil {
|
||||
assert.NotContains(t, err.Error(), "no handler for reserved key",
|
||||
"reserved key %q has no handler in visitComparisonForReservedKeys", key)
|
||||
"reserved key %q has no handler in resolveReservedKey", key)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,631 +0,0 @@
|
||||
package impldashboard
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/parser/filterquery"
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
// bunPlaceholderFlavor is any flavor that renders `?` placeholders, which bun
|
||||
// re-binds to the actual backend (e.g. `$1` for Postgres) at query time.
|
||||
const bunPlaceholderFlavor = sqlbuilder.SQLite
|
||||
|
||||
type visitor struct {
|
||||
grammar.BaseFilterQueryVisitor
|
||||
selectBuilder *sqlbuilder.SelectBuilder
|
||||
formatter sqlstore.SQLFormatter
|
||||
errors []string
|
||||
}
|
||||
|
||||
func newVisitor(formatter sqlstore.SQLFormatter) *visitor {
|
||||
return &visitor{
|
||||
selectBuilder: sqlbuilder.NewSelectBuilder(),
|
||||
formatter: formatter,
|
||||
}
|
||||
}
|
||||
|
||||
// compile builds `?`-placeholder WHERE SQL + args for bun. Each term is either a
|
||||
// `key OP value` comparison or a bare token that becomes a free-text search; the
|
||||
// two compose through the boolean grammar (AND/OR/NOT). Malformed input is
|
||||
// returned as errors.
|
||||
func (v *visitor) compile(query string) (string, []any, []string) {
|
||||
tree, _, collector := filterquery.Parse(query)
|
||||
if len(collector.Errors) > 0 {
|
||||
return "", nil, collector.Errors
|
||||
}
|
||||
condition, _ := v.visit(tree).(string)
|
||||
if len(v.errors) > 0 {
|
||||
return "", nil, v.errors
|
||||
}
|
||||
if condition == "" {
|
||||
return "", nil, nil
|
||||
}
|
||||
sql, arguments := v.selectBuilder.Args.CompileWithFlavor(condition, bunPlaceholderFlavor)
|
||||
return sql, arguments, nil
|
||||
}
|
||||
|
||||
func (v *visitor) visit(tree antlr.ParseTree) any {
|
||||
if tree == nil {
|
||||
return nil
|
||||
}
|
||||
return tree.Accept(v)
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════════════════
|
||||
// methods from grammar.BaseFilterQueryVisitor that are overridden
|
||||
// ════════════════════════════════════════════════════════════════════════
|
||||
|
||||
func (v *visitor) VisitQuery(ctx *grammar.QueryContext) any {
|
||||
return v.visit(ctx.Expression())
|
||||
}
|
||||
|
||||
func (v *visitor) VisitExpression(ctx *grammar.ExpressionContext) any {
|
||||
return v.visit(ctx.OrExpression())
|
||||
}
|
||||
|
||||
func (v *visitor) VisitOrExpression(ctx *grammar.OrExpressionContext) any {
|
||||
parts := ctx.AllAndExpression()
|
||||
conditions := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if condition, ok := v.visit(part).(string); ok && condition != "" {
|
||||
conditions = append(conditions, condition)
|
||||
}
|
||||
}
|
||||
switch len(conditions) {
|
||||
case 0:
|
||||
return ""
|
||||
case 1:
|
||||
return conditions[0]
|
||||
default:
|
||||
return v.selectBuilder.Or(conditions...)
|
||||
}
|
||||
}
|
||||
|
||||
func (v *visitor) VisitAndExpression(ctx *grammar.AndExpressionContext) any {
|
||||
parts := ctx.AllUnaryExpression()
|
||||
conditions := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if condition, ok := v.visit(part).(string); ok && condition != "" {
|
||||
conditions = append(conditions, condition)
|
||||
}
|
||||
}
|
||||
switch len(conditions) {
|
||||
case 0:
|
||||
return ""
|
||||
case 1:
|
||||
return conditions[0]
|
||||
default:
|
||||
return v.selectBuilder.And(conditions...)
|
||||
}
|
||||
}
|
||||
|
||||
func (v *visitor) VisitUnaryExpression(ctx *grammar.UnaryExpressionContext) any {
|
||||
condition, _ := v.visit(ctx.Primary()).(string)
|
||||
if condition == "" {
|
||||
return ""
|
||||
}
|
||||
if ctx.NOT() != nil {
|
||||
return fmt.Sprintf("NOT (%s)", condition)
|
||||
}
|
||||
return condition
|
||||
}
|
||||
|
||||
func (v *visitor) VisitPrimary(ctx *grammar.PrimaryContext) any {
|
||||
if ctx.OrExpression() != nil {
|
||||
return v.visit(ctx.OrExpression())
|
||||
}
|
||||
if ctx.Comparison() != nil {
|
||||
return v.visit(ctx.Comparison())
|
||||
}
|
||||
// A lone key/value/full-text token is a free-text term, composed with any
|
||||
// comparisons through the boolean grammar. A quoted token matches its contents
|
||||
// literally — the escape hatch for a phrase or a term that looks like DSL.
|
||||
return v.buildFreeTextTerm(trimQuotes(ctx.GetText()))
|
||||
}
|
||||
|
||||
// VisitComparison dispatches a single `key OP value` term. A key that matches
|
||||
// a reserved DSL key (name, description, etc.) becomes a column-level
|
||||
// predicate; any other identifier is treated as a tag key — the operator
|
||||
// applies to the tag's value, with a case-insensitive match on the tag's key.
|
||||
func (v *visitor) VisitComparison(ctx *grammar.ComparisonContext) any {
|
||||
key := strings.ToLower(strings.TrimSpace(ctx.Key().GetText()))
|
||||
|
||||
operation, ok := v.extractOperation(ctx)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
|
||||
if allowedOperations, isReserved := dashboardtypes.ReservedOps[dashboardtypes.DSLKey(key)]; isReserved {
|
||||
return v.visitComparisonForReservedKeys(ctx, operation, dashboardtypes.DSLKey(key), allowedOperations)
|
||||
}
|
||||
return v.visitComparisonForTags(ctx, operation, key)
|
||||
}
|
||||
|
||||
func (v *visitor) visitComparisonForReservedKeys(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, allowedOperations map[qbtypesv5.FilterOperator]struct{}) string {
|
||||
if _, allowed := allowedOperations[operation]; !allowed {
|
||||
v.addError("operator %s is not allowed for key %q", operationName(operation), key)
|
||||
return ""
|
||||
}
|
||||
switch key {
|
||||
case dashboardtypes.DSLKeyName:
|
||||
return v.buildJSONStringComparison(ctx, operation, dashboardtypes.DSLKeyName, "$.spec.display.name")
|
||||
case dashboardtypes.DSLKeyDescription:
|
||||
return v.buildJSONStringComparison(ctx, operation, dashboardtypes.DSLKeyDescription, "$.spec.display.description")
|
||||
case dashboardtypes.DSLKeyCreatedAt:
|
||||
return v.buildTimestampComparison(ctx, operation, "dashboard.created_at")
|
||||
case dashboardtypes.DSLKeyUpdatedAt:
|
||||
return v.buildTimestampComparison(ctx, operation, "dashboard.updated_at")
|
||||
case dashboardtypes.DSLKeyCreatedBy:
|
||||
return v.buildStringComparison(ctx, operation, dashboardtypes.DSLKeyCreatedBy, "dashboard.created_by")
|
||||
case dashboardtypes.DSLKeyLocked:
|
||||
return v.buildBoolComparison(ctx, operation, "dashboard.locked")
|
||||
}
|
||||
// Unreachable for real input: every dashboardtypes.ReservedOps key has a case above, and
|
||||
// TestCompileReservedKeysAllHandled guards that the two stay in sync.
|
||||
v.addError("no handler for reserved key %q", key)
|
||||
return ""
|
||||
}
|
||||
|
||||
func (v *visitor) visitComparisonForTags(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
|
||||
if _, allowed := dashboardtypes.TagKeyOps[operation]; !allowed {
|
||||
v.addError("operator %s is not allowed on a tag-key filter", operationName(operation))
|
||||
return ""
|
||||
}
|
||||
return v.buildTagComparison(ctx, operation, tagKey)
|
||||
}
|
||||
|
||||
func (v *visitor) extractOperation(ctx *grammar.ComparisonContext) (qbtypesv5.FilterOperator, bool) {
|
||||
// For operators that take an optional leading NOT, Inverse() maps each to
|
||||
// its Not<X> counterpart.
|
||||
maybeNot := func(operation qbtypesv5.FilterOperator) qbtypesv5.FilterOperator {
|
||||
if ctx.NOT() != nil {
|
||||
return operation.Inverse()
|
||||
}
|
||||
return operation
|
||||
}
|
||||
switch {
|
||||
case ctx.EQUALS() != nil:
|
||||
return qbtypesv5.FilterOperatorEqual, true
|
||||
case ctx.NOT_EQUALS() != nil, ctx.NEQ() != nil:
|
||||
return qbtypesv5.FilterOperatorNotEqual, true
|
||||
case ctx.LT() != nil:
|
||||
return qbtypesv5.FilterOperatorLessThan, true
|
||||
case ctx.LE() != nil:
|
||||
return qbtypesv5.FilterOperatorLessThanOrEq, true
|
||||
case ctx.GT() != nil:
|
||||
return qbtypesv5.FilterOperatorGreaterThan, true
|
||||
case ctx.GE() != nil:
|
||||
return qbtypesv5.FilterOperatorGreaterThanOrEq, true
|
||||
case ctx.BETWEEN() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorBetween), true
|
||||
case ctx.LIKE() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorLike), true
|
||||
case ctx.ILIKE() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorILike), true
|
||||
case ctx.CONTAINS() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorContains), true
|
||||
case ctx.REGEXP() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorRegexp), true
|
||||
case ctx.InClause() != nil:
|
||||
return qbtypesv5.FilterOperatorIn, true
|
||||
case ctx.NotInClause() != nil:
|
||||
return qbtypesv5.FilterOperatorNotIn, true
|
||||
case ctx.EXISTS() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorExists), true
|
||||
}
|
||||
v.addError("could not determine operator in expression %q", ctx.GetText())
|
||||
return qbtypesv5.FilterOperatorUnknown, false
|
||||
}
|
||||
|
||||
// ─── per-key emitters ────────────────────────────────────────────────────────
|
||||
|
||||
func (v *visitor) buildJSONStringComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, jsonPath string) string {
|
||||
columnExpression := string(v.formatter.JSONExtractString("dashboard.data", jsonPath))
|
||||
return v.buildStringOperation(v.selectBuilder, ctx, operation, columnExpression, string(key))
|
||||
}
|
||||
|
||||
func (v *visitor) buildStringComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, key dashboardtypes.DSLKey, columnExpression string) string {
|
||||
return v.buildStringOperation(v.selectBuilder, ctx, operation, columnExpression, string(key))
|
||||
}
|
||||
|
||||
// buildStringOperation covers all the operators the spec allows on text-shaped keys
|
||||
// (name, description, created_by, and a tag's value). Placeholders are interned
|
||||
// into builder — the outer builder for column predicates, the subquery builder for
|
||||
// tag-value predicates — so nested EXISTS arguments thread correctly.
|
||||
func (v *visitor) buildStringOperation(builder *sqlbuilder.SelectBuilder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression, keyForError string) string {
|
||||
switch operation {
|
||||
case qbtypesv5.FilterOperatorEqual:
|
||||
val, ok := v.extractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return builder.Equal(columnExpression, val)
|
||||
case qbtypesv5.FilterOperatorNotEqual:
|
||||
val, ok := v.extractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return builder.NotEqual(columnExpression, val)
|
||||
case qbtypesv5.FilterOperatorLike, qbtypesv5.FilterOperatorNotLike:
|
||||
val, ok := v.extractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
like := "LIKE"
|
||||
if operation == qbtypesv5.FilterOperatorNotLike {
|
||||
like = "NOT LIKE"
|
||||
}
|
||||
// The user's % and _ stay as wildcards; ESCAPE pins backslash as the escape
|
||||
// char so a literal `\` in the pattern is read the same on both dialects —
|
||||
// Postgres defaults to `\`, SQLite has no default escape.
|
||||
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, builder.Var(val))
|
||||
case qbtypesv5.FilterOperatorILike, qbtypesv5.FilterOperatorNotILike:
|
||||
val, ok := v.extractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
// SQLite has no ILIKE keyword and Postgres LIKE is case-sensitive — emit
|
||||
// LOWER(col) LIKE LOWER(?) so behavior is identical on both dialects. ESCAPE
|
||||
// pins backslash as the escape char (Postgres default; SQLite has none).
|
||||
lowerColumn := string(v.formatter.LowerExpression(columnExpression))
|
||||
like := "LIKE"
|
||||
if operation == qbtypesv5.FilterOperatorNotILike {
|
||||
like = "NOT LIKE"
|
||||
}
|
||||
return fmt.Sprintf("%s %s LOWER(%s) ESCAPE '\\'", lowerColumn, like, builder.Var(val))
|
||||
case qbtypesv5.FilterOperatorContains, qbtypesv5.FilterOperatorNotContains:
|
||||
val, ok := v.extractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
like := "LIKE"
|
||||
if operation == qbtypesv5.FilterOperatorNotContains {
|
||||
like = "NOT LIKE"
|
||||
}
|
||||
// Escape the user's % and _ so they match literally, then wrap in wildcards.
|
||||
// ESCAPE declares the backslash the escaper injected as the escape char —
|
||||
// needed on SQLite (no default) and a harmless restatement of the Postgres default.
|
||||
escaped := v.formatter.EscapeLikePattern(val)
|
||||
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, builder.Var("%"+escaped+"%"))
|
||||
case qbtypesv5.FilterOperatorRegexp, qbtypesv5.FilterOperatorNotRegexp:
|
||||
v.addError("REGEXP filtering on %q is not yet supported", keyForError)
|
||||
return ""
|
||||
case qbtypesv5.FilterOperatorIn, qbtypesv5.FilterOperatorNotIn:
|
||||
values, ok := v.extractStringValueList(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
arguments := make([]any, len(values))
|
||||
for i, s := range values {
|
||||
arguments[i] = s
|
||||
}
|
||||
if operation == qbtypesv5.FilterOperatorNotIn {
|
||||
return builder.NotIn(columnExpression, arguments...)
|
||||
}
|
||||
return builder.In(columnExpression, arguments...)
|
||||
}
|
||||
v.addError("operator %s on %q is not implemented", operationName(operation), keyForError)
|
||||
return ""
|
||||
}
|
||||
|
||||
func (v *visitor) buildTimestampComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
|
||||
switch operation {
|
||||
case qbtypesv5.FilterOperatorEqual, qbtypesv5.FilterOperatorNotEqual,
|
||||
qbtypesv5.FilterOperatorLessThan, qbtypesv5.FilterOperatorLessThanOrEq,
|
||||
qbtypesv5.FilterOperatorGreaterThan, qbtypesv5.FilterOperatorGreaterThanOrEq:
|
||||
t, ok := v.extractSingleTimestampValue(ctx)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
switch operation {
|
||||
case qbtypesv5.FilterOperatorEqual:
|
||||
return v.selectBuilder.Equal(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorNotEqual:
|
||||
return v.selectBuilder.NotEqual(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorLessThan:
|
||||
return v.selectBuilder.LessThan(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorLessThanOrEq:
|
||||
return v.selectBuilder.LessEqualThan(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorGreaterThan:
|
||||
return v.selectBuilder.GreaterThan(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorGreaterThanOrEq:
|
||||
return v.selectBuilder.GreaterEqualThan(columnExpression, t)
|
||||
}
|
||||
case qbtypesv5.FilterOperatorBetween, qbtypesv5.FilterOperatorNotBetween:
|
||||
timestamps, ok := v.extractTwoTimestampValues(ctx)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if operation == qbtypesv5.FilterOperatorNotBetween {
|
||||
return v.selectBuilder.NotBetween(columnExpression, timestamps[0], timestamps[1])
|
||||
}
|
||||
return v.selectBuilder.Between(columnExpression, timestamps[0], timestamps[1])
|
||||
}
|
||||
v.addError("operator %s on timestamp is not implemented", operationName(operation))
|
||||
return ""
|
||||
}
|
||||
|
||||
func (v *visitor) buildBoolComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
|
||||
b, ok := v.extractSingleBoolValue(ctx)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if operation == qbtypesv5.FilterOperatorNotEqual {
|
||||
return v.selectBuilder.NotEqual(columnExpression, b)
|
||||
}
|
||||
return v.selectBuilder.Equal(columnExpression, b)
|
||||
}
|
||||
|
||||
func (v *visitor) buildTagComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, tagKey string) string {
|
||||
subqueryBuilder := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
if operation == qbtypesv5.FilterOperatorExists || operation == qbtypesv5.FilterOperatorNotExists {
|
||||
buildSubqueryForTagKey(subqueryBuilder, tagKey)
|
||||
} else {
|
||||
// All other tag operators take the positive form of the value predicate
|
||||
// and toggle the EXISTS wrapper for negation. Inverse() flips Not<X> → <X>.
|
||||
positiveOperation := operation
|
||||
if operation.IsNegativeOperator() {
|
||||
positiveOperation = operation.Inverse()
|
||||
}
|
||||
valuePredicate := v.buildStringOperation(subqueryBuilder, ctx, positiveOperation, "t.value", tagKey)
|
||||
if valuePredicate == "" {
|
||||
return ""
|
||||
}
|
||||
buildSubqueryForTagKeyAndValue(subqueryBuilder, tagKey, valuePredicate)
|
||||
}
|
||||
|
||||
if operation.IsNegativeOperator() {
|
||||
return v.selectBuilder.NotExists(subqueryBuilder)
|
||||
}
|
||||
return v.selectBuilder.Exists(subqueryBuilder)
|
||||
}
|
||||
|
||||
func buildSubqueryForTagKey(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey string) *sqlbuilder.SelectBuilder {
|
||||
const dashboardTagKind = `"dashboard"`
|
||||
|
||||
return subqueryBuilder.
|
||||
Select("1").
|
||||
From("tag_relation tr").
|
||||
Join("tag t", "t.id = tr.tag_id").
|
||||
Where(
|
||||
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
|
||||
"tr.resource_id = dashboard.id",
|
||||
"LOWER(t.key) = LOWER("+subqueryBuilder.Var(tagKey)+")",
|
||||
)
|
||||
}
|
||||
|
||||
func buildSubqueryForTagKeyAndValue(subqueryBuilder *sqlbuilder.SelectBuilder, tagKey, valuePredicate string) *sqlbuilder.SelectBuilder {
|
||||
return buildSubqueryForTagKey(subqueryBuilder, tagKey).Where(valuePredicate)
|
||||
}
|
||||
|
||||
// ─── free-text search ────────────────────────────────────────────────────────
|
||||
|
||||
// buildFreeTextTerm matches value as a case-insensitive substring of the
|
||||
// dashboard name, description, or any tag key/value.
|
||||
func (v *visitor) buildFreeTextTerm(value string) string {
|
||||
nameColumn := string(v.formatter.JSONExtractString("dashboard.data", "$.spec.display.name"))
|
||||
descriptionColumn := string(v.formatter.JSONExtractString("dashboard.data", "$.spec.display.description"))
|
||||
namePredicate := v.buildFreeTextContains(v.selectBuilder, nameColumn, value)
|
||||
descriptionPredicate := v.buildFreeTextContains(v.selectBuilder, descriptionColumn, value)
|
||||
|
||||
subqueryBuilder := sqlbuilder.NewSelectBuilder()
|
||||
keyPredicate := v.buildFreeTextContains(subqueryBuilder, "t.key", value)
|
||||
valuePredicate := v.buildFreeTextContains(subqueryBuilder, "t.value", value)
|
||||
buildSubqueryForFreeTextTag(subqueryBuilder, keyPredicate, valuePredicate)
|
||||
tagPredicate := v.selectBuilder.Exists(subqueryBuilder)
|
||||
|
||||
return v.selectBuilder.Or(namePredicate, descriptionPredicate, tagPredicate)
|
||||
}
|
||||
|
||||
// buildFreeTextContains emits a case-insensitive contains as
|
||||
// LOWER(COALESCE(col, '')) LIKE LOWER(?), identical on SQLite and Postgres.
|
||||
// COALESCE keeps a NULL column (an absent description) false rather than NULL —
|
||||
// otherwise `NOT (…)` goes NULL and drops every description-less dashboard. The
|
||||
// value's % and _ are escaped, and ESCAPE pins backslash as the escape char.
|
||||
func (v *visitor) buildFreeTextContains(builder *sqlbuilder.SelectBuilder, columnExpression, value string) string {
|
||||
lowerColumn := string(v.formatter.LowerExpression("COALESCE(" + columnExpression + ", '')"))
|
||||
pattern := "%" + v.formatter.EscapeLikePattern(value) + "%"
|
||||
return fmt.Sprintf("%s LIKE LOWER(%s) ESCAPE '\\'", lowerColumn, builder.Var(pattern))
|
||||
}
|
||||
|
||||
func buildSubqueryForFreeTextTag(subqueryBuilder *sqlbuilder.SelectBuilder, keyPredicate, valuePredicate string) *sqlbuilder.SelectBuilder {
|
||||
const dashboardTagKind = `"dashboard"`
|
||||
|
||||
return subqueryBuilder.
|
||||
Select("1").
|
||||
From("tag_relation tr").
|
||||
Join("tag t", "t.id = tr.tag_id").
|
||||
Where(
|
||||
subqueryBuilder.Equal("tr.kind", dashboardTagKind),
|
||||
"tr.resource_id = dashboard.id",
|
||||
subqueryBuilder.Or(keyPredicate, valuePredicate),
|
||||
)
|
||||
}
|
||||
|
||||
// ─── value extraction helpers ───────────────────────────────────────────────
|
||||
|
||||
func (v *visitor) addError(format string, arguments ...any) {
|
||||
v.errors = append(v.errors, fmt.Sprintf(format, arguments...))
|
||||
}
|
||||
|
||||
func (v *visitor) extractSingleStringValue(ctx *grammar.ComparisonContext, keyForError string) (string, bool) {
|
||||
values := ctx.AllValue()
|
||||
if len(values) != 1 {
|
||||
v.addError("expected exactly one value for %q", keyForError)
|
||||
return "", false
|
||||
}
|
||||
return v.extractStringValue(values[0], keyForError)
|
||||
}
|
||||
|
||||
func (v *visitor) extractSingleBoolValue(ctx *grammar.ComparisonContext) (bool, bool) {
|
||||
values := ctx.AllValue()
|
||||
if len(values) != 1 {
|
||||
v.addError("expected a single boolean (true/false)")
|
||||
return false, false
|
||||
}
|
||||
return v.extractBoolValue(values[0])
|
||||
}
|
||||
|
||||
func (v *visitor) extractSingleTimestampValue(ctx *grammar.ComparisonContext) (time.Time, bool) {
|
||||
values := ctx.AllValue()
|
||||
if len(values) != 1 {
|
||||
v.addError("expected a single RFC3339 timestamp")
|
||||
return time.Time{}, false
|
||||
}
|
||||
return v.extractTimestampValue(values[0])
|
||||
}
|
||||
|
||||
func (v *visitor) extractTwoTimestampValues(ctx *grammar.ComparisonContext) ([2]time.Time, bool) {
|
||||
values := ctx.AllValue()
|
||||
if len(values) != 2 {
|
||||
v.addError("BETWEEN expects two RFC3339 timestamps")
|
||||
return [2]time.Time{}, false
|
||||
}
|
||||
a, ok1 := v.extractTimestampValue(values[0])
|
||||
b, ok2 := v.extractTimestampValue(values[1])
|
||||
if !ok1 || !ok2 {
|
||||
return [2]time.Time{}, false
|
||||
}
|
||||
return [2]time.Time{a, b}, true
|
||||
}
|
||||
|
||||
func (v *visitor) extractStringValueList(ctx *grammar.ComparisonContext, keyForError string) ([]string, bool) {
|
||||
var valuesCtx []grammar.IValueContext
|
||||
switch {
|
||||
case ctx.InClause() != nil:
|
||||
inClause := ctx.InClause()
|
||||
if inClause.ValueList() != nil {
|
||||
valuesCtx = inClause.ValueList().AllValue()
|
||||
} else {
|
||||
valuesCtx = []grammar.IValueContext{inClause.Value()}
|
||||
}
|
||||
case ctx.NotInClause() != nil:
|
||||
notInClause := ctx.NotInClause()
|
||||
if notInClause.ValueList() != nil {
|
||||
valuesCtx = notInClause.ValueList().AllValue()
|
||||
} else {
|
||||
valuesCtx = []grammar.IValueContext{notInClause.Value()}
|
||||
}
|
||||
default:
|
||||
v.addError("IN clause is missing for %q", keyForError)
|
||||
return nil, false
|
||||
}
|
||||
if len(valuesCtx) == 0 {
|
||||
v.addError("IN list for %q is empty", keyForError)
|
||||
return nil, false
|
||||
}
|
||||
out := make([]string, 0, len(valuesCtx))
|
||||
for _, valueContext := range valuesCtx {
|
||||
s, ok := v.extractStringValue(valueContext, keyForError)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
func (v *visitor) extractStringValue(ctx grammar.IValueContext, keyForError string) (string, bool) {
|
||||
if ctx.QUOTED_TEXT() != nil {
|
||||
return trimQuotes(ctx.QUOTED_TEXT().GetText()), true
|
||||
}
|
||||
if ctx.KEY() != nil {
|
||||
// Bare tokens are accepted as strings, mirroring the FilterQuery lexer's
|
||||
// treatment of unquoted identifiers on the value side.
|
||||
return ctx.KEY().GetText(), true
|
||||
}
|
||||
v.addError("expected a string value for %q, got %q", keyForError, ctx.GetText())
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (v *visitor) extractBoolValue(ctx grammar.IValueContext) (bool, bool) {
|
||||
if ctx.BOOL() == nil {
|
||||
v.addError("expected a boolean (true/false), got %q", ctx.GetText())
|
||||
return false, false
|
||||
}
|
||||
return strings.EqualFold(ctx.BOOL().GetText(), "true"), true
|
||||
}
|
||||
|
||||
func (v *visitor) extractTimestampValue(ctx grammar.IValueContext) (time.Time, bool) {
|
||||
if ctx.QUOTED_TEXT() == nil {
|
||||
v.addError("expected an RFC3339 timestamp string, got %q", ctx.GetText())
|
||||
return time.Time{}, false
|
||||
}
|
||||
raw := trimQuotes(ctx.QUOTED_TEXT().GetText())
|
||||
t, err := time.Parse(time.RFC3339, raw)
|
||||
if err != nil {
|
||||
v.addError("invalid RFC3339 timestamp %q: %s", raw, err.Error())
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
// ─── operator spelling ───────────────────────────────────────────────────────
|
||||
|
||||
// operationName returns the user-facing spelling of a FilterOperator, used only in
|
||||
// error messages — go-sqlbuilder's Cond helpers emit the SQL keywords.
|
||||
func operationName(operation qbtypesv5.FilterOperator) string {
|
||||
switch operation {
|
||||
case qbtypesv5.FilterOperatorEqual:
|
||||
return "="
|
||||
case qbtypesv5.FilterOperatorNotEqual:
|
||||
return "!="
|
||||
case qbtypesv5.FilterOperatorLessThan:
|
||||
return "<"
|
||||
case qbtypesv5.FilterOperatorLessThanOrEq:
|
||||
return "<="
|
||||
case qbtypesv5.FilterOperatorGreaterThan:
|
||||
return ">"
|
||||
case qbtypesv5.FilterOperatorGreaterThanOrEq:
|
||||
return ">="
|
||||
case qbtypesv5.FilterOperatorBetween:
|
||||
return "BETWEEN"
|
||||
case qbtypesv5.FilterOperatorNotBetween:
|
||||
return "NOT BETWEEN"
|
||||
case qbtypesv5.FilterOperatorLike:
|
||||
return "LIKE"
|
||||
case qbtypesv5.FilterOperatorNotLike:
|
||||
return "NOT LIKE"
|
||||
case qbtypesv5.FilterOperatorILike:
|
||||
return "ILIKE"
|
||||
case qbtypesv5.FilterOperatorNotILike:
|
||||
return "NOT ILIKE"
|
||||
case qbtypesv5.FilterOperatorContains:
|
||||
return "CONTAINS"
|
||||
case qbtypesv5.FilterOperatorNotContains:
|
||||
return "NOT CONTAINS"
|
||||
case qbtypesv5.FilterOperatorRegexp:
|
||||
return "REGEXP"
|
||||
case qbtypesv5.FilterOperatorNotRegexp:
|
||||
return "NOT REGEXP"
|
||||
case qbtypesv5.FilterOperatorIn:
|
||||
return "IN"
|
||||
case qbtypesv5.FilterOperatorNotIn:
|
||||
return "NOT IN"
|
||||
case qbtypesv5.FilterOperatorExists:
|
||||
return "EXISTS"
|
||||
case qbtypesv5.FilterOperatorNotExists:
|
||||
return "NOT EXISTS"
|
||||
}
|
||||
return "?"
|
||||
}
|
||||
|
||||
func trimQuotes(s string) string {
|
||||
if len(s) >= 2 {
|
||||
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
|
||||
s = s[1 : len(s)-1]
|
||||
}
|
||||
}
|
||||
s = strings.ReplaceAll(s, `\\`, `\`)
|
||||
s = strings.ReplaceAll(s, `\'`, `'`)
|
||||
return s
|
||||
}
|
||||
@@ -82,6 +82,7 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques
|
||||
|
||||
values := &telemetrytypes.TelemetryFieldValues{
|
||||
StringValues: allValues.StringValues,
|
||||
BoolValues: allValues.BoolValues,
|
||||
NumberValues: allValues.NumberValues,
|
||||
RelatedValues: relatedValues,
|
||||
}
|
||||
|
||||
521
pkg/parser/filterquery/sqlcompiler/compiler.go
Normal file
521
pkg/parser/filterquery/sqlcompiler/compiler.go
Normal file
@@ -0,0 +1,521 @@
|
||||
// Package sqlcompiler compiles list-page filter queries to relational-store WHERE clauses; telemetry queries stay on querybuilder's ClickHouse visitor.
|
||||
package sqlcompiler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/parser/filterquery"
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
qbtypesv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
// bunPlaceholderFlavor is any flavor that renders the `?` placeholders bun expects.
|
||||
const bunPlaceholderFlavor = sqlbuilder.SQLite
|
||||
|
||||
// FieldResolver is the per-feature policy: which keys exist and what each maps to.
|
||||
type FieldResolver interface {
|
||||
// ResolveComparison builds the predicate for one `key OP value` term; key keeps the user's casing.
|
||||
ResolveComparison(b *Builder, key string, operation qbtypesv5.FilterOperator, ctx *grammar.ComparisonContext) string
|
||||
// FreeText builds the predicate for a bare token.
|
||||
FreeText(b *Builder, value string) string
|
||||
}
|
||||
|
||||
// Compiled is a `?`-placeholder WHERE clause with its bun bind args.
|
||||
type Compiled struct {
|
||||
SQL string
|
||||
Args []any
|
||||
}
|
||||
|
||||
func (c Compiled) IsEmpty() bool {
|
||||
return c.SQL == ""
|
||||
}
|
||||
|
||||
// Compile on success returns a non-nil *Compiled, empty for an empty query; callers gate on IsEmpty, not nil.
|
||||
func Compile(query string, formatter sqlstore.SQLFormatter, resolver FieldResolver) (*Compiled, []string) {
|
||||
if len(strings.TrimSpace(query)) == 0 {
|
||||
return &Compiled{}, nil
|
||||
}
|
||||
|
||||
v := &visitor{
|
||||
builder: &Builder{
|
||||
selectBuilder: sqlbuilder.NewSelectBuilder(),
|
||||
formatter: formatter,
|
||||
},
|
||||
resolver: resolver,
|
||||
}
|
||||
|
||||
tree, _, collector := filterquery.Parse(query)
|
||||
if len(collector.Errors) > 0 {
|
||||
return nil, collector.Errors
|
||||
}
|
||||
condition, _ := v.visit(tree).(string)
|
||||
if len(v.builder.errors) > 0 {
|
||||
return nil, v.builder.errors
|
||||
}
|
||||
if condition == "" {
|
||||
return &Compiled{}, nil
|
||||
}
|
||||
sql, arguments := v.builder.selectBuilder.Args.CompileWithFlavor(condition, bunPlaceholderFlavor)
|
||||
return &Compiled{SQL: sql, Args: arguments}, nil
|
||||
}
|
||||
|
||||
// Builder is the per-compile toolbox handed to a FieldResolver.
|
||||
type Builder struct {
|
||||
selectBuilder *sqlbuilder.SelectBuilder
|
||||
formatter sqlstore.SQLFormatter
|
||||
errors []string
|
||||
}
|
||||
|
||||
func (b *Builder) SelectBuilder() *sqlbuilder.SelectBuilder {
|
||||
return b.selectBuilder
|
||||
}
|
||||
|
||||
func (b *Builder) Formatter() sqlstore.SQLFormatter {
|
||||
return b.formatter
|
||||
}
|
||||
|
||||
func (b *Builder) AddError(format string, arguments ...any) {
|
||||
b.errors = append(b.errors, fmt.Sprintf(format, arguments...))
|
||||
}
|
||||
|
||||
// StringOperation interns placeholders into sb so nested subquery arguments thread correctly.
|
||||
func (b *Builder) StringOperation(sb *sqlbuilder.SelectBuilder, ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression, keyForError string) string {
|
||||
switch operation {
|
||||
case qbtypesv5.FilterOperatorEqual:
|
||||
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return sb.Equal(columnExpression, val)
|
||||
case qbtypesv5.FilterOperatorNotEqual:
|
||||
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return sb.NotEqual(columnExpression, val)
|
||||
case qbtypesv5.FilterOperatorLike, qbtypesv5.FilterOperatorNotLike:
|
||||
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if endsWithDanglingEscape(val) {
|
||||
b.AddError("LIKE pattern for %q must not end with an unescaped backslash, use \\\\ to match a literal backslash", keyForError)
|
||||
return ""
|
||||
}
|
||||
like := "LIKE"
|
||||
if operation == qbtypesv5.FilterOperatorNotLike {
|
||||
like = "NOT LIKE"
|
||||
}
|
||||
// ESCAPE pins backslash as the escape char (the Postgres default, SQLite has none).
|
||||
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, sb.Var(val))
|
||||
case qbtypesv5.FilterOperatorILike, qbtypesv5.FilterOperatorNotILike:
|
||||
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if endsWithDanglingEscape(val) {
|
||||
b.AddError("ILIKE pattern for %q must not end with an unescaped backslash, use \\\\ to match a literal backslash", keyForError)
|
||||
return ""
|
||||
}
|
||||
// SQLite has no ILIKE and Postgres LIKE is case-sensitive, so LOWER both sides.
|
||||
lowerColumn := string(b.formatter.LowerExpression(columnExpression))
|
||||
like := "LIKE"
|
||||
if operation == qbtypesv5.FilterOperatorNotILike {
|
||||
like = "NOT LIKE"
|
||||
}
|
||||
return fmt.Sprintf("%s %s LOWER(%s) ESCAPE '\\'", lowerColumn, like, sb.Var(val))
|
||||
case qbtypesv5.FilterOperatorContains, qbtypesv5.FilterOperatorNotContains:
|
||||
val, ok := b.ExtractSingleStringValue(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
like := "LIKE"
|
||||
if operation == qbtypesv5.FilterOperatorNotContains {
|
||||
like = "NOT LIKE"
|
||||
}
|
||||
// Escape the user's % and _ so they match literally, then wrap in wildcards.
|
||||
escaped := b.formatter.EscapeLikePattern(val)
|
||||
return fmt.Sprintf("%s %s %s ESCAPE '\\'", columnExpression, like, sb.Var(fmt.Sprintf("%%%s%%", escaped)))
|
||||
case qbtypesv5.FilterOperatorRegexp, qbtypesv5.FilterOperatorNotRegexp:
|
||||
b.AddError("REGEXP filtering on %q is not supported", keyForError)
|
||||
return ""
|
||||
case qbtypesv5.FilterOperatorIn, qbtypesv5.FilterOperatorNotIn:
|
||||
values, ok := b.ExtractStringValueList(ctx, keyForError)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
arguments := make([]any, len(values))
|
||||
for i, s := range values {
|
||||
arguments[i] = s
|
||||
}
|
||||
if operation == qbtypesv5.FilterOperatorNotIn {
|
||||
return sb.NotIn(columnExpression, arguments...)
|
||||
}
|
||||
return sb.In(columnExpression, arguments...)
|
||||
}
|
||||
b.AddError("operator %s on %q is not implemented", OperationName(operation), keyForError)
|
||||
return ""
|
||||
}
|
||||
|
||||
func (b *Builder) TimestampComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
|
||||
switch operation {
|
||||
case qbtypesv5.FilterOperatorEqual, qbtypesv5.FilterOperatorNotEqual,
|
||||
qbtypesv5.FilterOperatorLessThan, qbtypesv5.FilterOperatorLessThanOrEq,
|
||||
qbtypesv5.FilterOperatorGreaterThan, qbtypesv5.FilterOperatorGreaterThanOrEq:
|
||||
t, ok := b.extractSingleTimestampValue(ctx)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
switch operation {
|
||||
case qbtypesv5.FilterOperatorEqual:
|
||||
return b.selectBuilder.Equal(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorNotEqual:
|
||||
return b.selectBuilder.NotEqual(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorLessThan:
|
||||
return b.selectBuilder.LessThan(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorLessThanOrEq:
|
||||
return b.selectBuilder.LessEqualThan(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorGreaterThan:
|
||||
return b.selectBuilder.GreaterThan(columnExpression, t)
|
||||
case qbtypesv5.FilterOperatorGreaterThanOrEq:
|
||||
return b.selectBuilder.GreaterEqualThan(columnExpression, t)
|
||||
}
|
||||
case qbtypesv5.FilterOperatorBetween, qbtypesv5.FilterOperatorNotBetween:
|
||||
timestamps, ok := b.extractTwoTimestampValues(ctx)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if operation == qbtypesv5.FilterOperatorNotBetween {
|
||||
return b.selectBuilder.NotBetween(columnExpression, timestamps[0], timestamps[1])
|
||||
}
|
||||
return b.selectBuilder.Between(columnExpression, timestamps[0], timestamps[1])
|
||||
}
|
||||
b.AddError("operator %s on timestamp is not implemented", OperationName(operation))
|
||||
return ""
|
||||
}
|
||||
|
||||
func (b *Builder) BoolComparison(ctx *grammar.ComparisonContext, operation qbtypesv5.FilterOperator, columnExpression string) string {
|
||||
value, ok := b.extractSingleBoolValue(ctx)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if operation == qbtypesv5.FilterOperatorNotEqual {
|
||||
return b.selectBuilder.NotEqual(columnExpression, value)
|
||||
}
|
||||
return b.selectBuilder.Equal(columnExpression, value)
|
||||
}
|
||||
|
||||
// A pattern ending in an unescaped backslash never matches on sqlite and errors on Postgres.
|
||||
func endsWithDanglingEscape(value string) bool {
|
||||
trailing := len(value) - len(strings.TrimRight(value, `\`))
|
||||
return trailing%2 == 1
|
||||
}
|
||||
|
||||
// FreeTextContains COALESCEs the column so NOT (...) does not go NULL and drop rows where it is absent.
|
||||
func (b *Builder) FreeTextContains(sb *sqlbuilder.SelectBuilder, columnExpression, value string) string {
|
||||
lowerColumn := string(b.formatter.LowerExpression(fmt.Sprintf("COALESCE(%s, '')", columnExpression)))
|
||||
pattern := fmt.Sprintf("%%%s%%", b.formatter.EscapeLikePattern(value))
|
||||
return fmt.Sprintf("%s LIKE LOWER(%s) ESCAPE '\\'", lowerColumn, sb.Var(pattern))
|
||||
}
|
||||
|
||||
func (b *Builder) ExtractSingleStringValue(ctx *grammar.ComparisonContext, keyForError string) (string, bool) {
|
||||
values := ctx.AllValue()
|
||||
if len(values) != 1 {
|
||||
b.AddError("expected exactly one value for %q", keyForError)
|
||||
return "", false
|
||||
}
|
||||
return b.extractStringValue(values[0], keyForError)
|
||||
}
|
||||
|
||||
func (b *Builder) ExtractStringValueList(ctx *grammar.ComparisonContext, keyForError string) ([]string, bool) {
|
||||
var valuesCtx []grammar.IValueContext
|
||||
switch {
|
||||
case ctx.InClause() != nil:
|
||||
inClause := ctx.InClause()
|
||||
if inClause.ValueList() != nil {
|
||||
valuesCtx = inClause.ValueList().AllValue()
|
||||
} else {
|
||||
valuesCtx = []grammar.IValueContext{inClause.Value()}
|
||||
}
|
||||
case ctx.NotInClause() != nil:
|
||||
notInClause := ctx.NotInClause()
|
||||
if notInClause.ValueList() != nil {
|
||||
valuesCtx = notInClause.ValueList().AllValue()
|
||||
} else {
|
||||
valuesCtx = []grammar.IValueContext{notInClause.Value()}
|
||||
}
|
||||
default:
|
||||
b.AddError("IN clause is missing for %q", keyForError)
|
||||
return nil, false
|
||||
}
|
||||
if len(valuesCtx) == 0 {
|
||||
b.AddError("IN list for %q is empty", keyForError)
|
||||
return nil, false
|
||||
}
|
||||
out := make([]string, 0, len(valuesCtx))
|
||||
for _, valueContext := range valuesCtx {
|
||||
s, ok := b.extractStringValue(valueContext, keyForError)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
func (b *Builder) extractSingleBoolValue(ctx *grammar.ComparisonContext) (bool, bool) {
|
||||
values := ctx.AllValue()
|
||||
if len(values) != 1 {
|
||||
b.AddError("expected a single boolean (true/false)")
|
||||
return false, false
|
||||
}
|
||||
return b.extractBoolValue(values[0])
|
||||
}
|
||||
|
||||
func (b *Builder) extractSingleTimestampValue(ctx *grammar.ComparisonContext) (time.Time, bool) {
|
||||
values := ctx.AllValue()
|
||||
if len(values) != 1 {
|
||||
b.AddError("expected a single RFC3339 timestamp")
|
||||
return time.Time{}, false
|
||||
}
|
||||
return b.extractTimestampValue(values[0])
|
||||
}
|
||||
|
||||
func (b *Builder) extractTwoTimestampValues(ctx *grammar.ComparisonContext) ([2]time.Time, bool) {
|
||||
values := ctx.AllValue()
|
||||
if len(values) != 2 {
|
||||
b.AddError("BETWEEN expects two RFC3339 timestamps")
|
||||
return [2]time.Time{}, false
|
||||
}
|
||||
first, ok1 := b.extractTimestampValue(values[0])
|
||||
second, ok2 := b.extractTimestampValue(values[1])
|
||||
if !ok1 || !ok2 {
|
||||
return [2]time.Time{}, false
|
||||
}
|
||||
return [2]time.Time{first, second}, true
|
||||
}
|
||||
|
||||
func (b *Builder) extractStringValue(ctx grammar.IValueContext, keyForError string) (string, bool) {
|
||||
if ctx.QUOTED_TEXT() != nil {
|
||||
return trimQuotes(ctx.QUOTED_TEXT().GetText()), true
|
||||
}
|
||||
if ctx.KEY() != nil {
|
||||
return ctx.KEY().GetText(), true
|
||||
}
|
||||
b.AddError("expected a string value for %q, got %q", keyForError, ctx.GetText())
|
||||
return "", false
|
||||
}
|
||||
|
||||
func (b *Builder) extractBoolValue(ctx grammar.IValueContext) (bool, bool) {
|
||||
if ctx.BOOL() == nil {
|
||||
b.AddError("expected a boolean (true/false), got %q", ctx.GetText())
|
||||
return false, false
|
||||
}
|
||||
return strings.EqualFold(ctx.BOOL().GetText(), "true"), true
|
||||
}
|
||||
|
||||
func (b *Builder) extractTimestampValue(ctx grammar.IValueContext) (time.Time, bool) {
|
||||
if ctx.QUOTED_TEXT() == nil {
|
||||
b.AddError("expected an RFC3339 timestamp string, got %q", ctx.GetText())
|
||||
return time.Time{}, false
|
||||
}
|
||||
raw := trimQuotes(ctx.QUOTED_TEXT().GetText())
|
||||
t, err := time.Parse(time.RFC3339, raw)
|
||||
if err != nil {
|
||||
b.AddError("invalid RFC3339 timestamp %q: %s", raw, err.Error())
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
type visitor struct {
|
||||
grammar.BaseFilterQueryVisitor
|
||||
builder *Builder
|
||||
resolver FieldResolver
|
||||
}
|
||||
|
||||
func (v *visitor) visit(tree antlr.ParseTree) any {
|
||||
if tree == nil {
|
||||
return nil
|
||||
}
|
||||
return tree.Accept(v)
|
||||
}
|
||||
|
||||
func (v *visitor) VisitQuery(ctx *grammar.QueryContext) any {
|
||||
return v.visit(ctx.Expression())
|
||||
}
|
||||
|
||||
func (v *visitor) VisitExpression(ctx *grammar.ExpressionContext) any {
|
||||
return v.visit(ctx.OrExpression())
|
||||
}
|
||||
|
||||
func (v *visitor) VisitOrExpression(ctx *grammar.OrExpressionContext) any {
|
||||
parts := ctx.AllAndExpression()
|
||||
conditions := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if condition, ok := v.visit(part).(string); ok && condition != "" {
|
||||
conditions = append(conditions, condition)
|
||||
}
|
||||
}
|
||||
switch len(conditions) {
|
||||
case 0:
|
||||
return ""
|
||||
case 1:
|
||||
return conditions[0]
|
||||
default:
|
||||
return v.builder.selectBuilder.Or(conditions...)
|
||||
}
|
||||
}
|
||||
|
||||
func (v *visitor) VisitAndExpression(ctx *grammar.AndExpressionContext) any {
|
||||
parts := ctx.AllUnaryExpression()
|
||||
conditions := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if condition, ok := v.visit(part).(string); ok && condition != "" {
|
||||
conditions = append(conditions, condition)
|
||||
}
|
||||
}
|
||||
switch len(conditions) {
|
||||
case 0:
|
||||
return ""
|
||||
case 1:
|
||||
return conditions[0]
|
||||
default:
|
||||
return v.builder.selectBuilder.And(conditions...)
|
||||
}
|
||||
}
|
||||
|
||||
func (v *visitor) VisitUnaryExpression(ctx *grammar.UnaryExpressionContext) any {
|
||||
condition, _ := v.visit(ctx.Primary()).(string)
|
||||
if condition == "" {
|
||||
return ""
|
||||
}
|
||||
if ctx.NOT() != nil {
|
||||
return fmt.Sprintf("NOT (%s)", condition)
|
||||
}
|
||||
return condition
|
||||
}
|
||||
|
||||
func (v *visitor) VisitPrimary(ctx *grammar.PrimaryContext) any {
|
||||
if ctx.OrExpression() != nil {
|
||||
return v.visit(ctx.OrExpression())
|
||||
}
|
||||
if ctx.Comparison() != nil {
|
||||
return v.visit(ctx.Comparison())
|
||||
}
|
||||
// A quoted lone token matches its contents literally, the escape hatch for a phrase that looks like DSL.
|
||||
return v.resolver.FreeText(v.builder, trimQuotes(ctx.GetText()))
|
||||
}
|
||||
|
||||
func (v *visitor) VisitComparison(ctx *grammar.ComparisonContext) any {
|
||||
key := strings.TrimSpace(ctx.Key().GetText())
|
||||
operation, ok := v.extractOperation(ctx)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return v.resolver.ResolveComparison(v.builder, key, operation, ctx)
|
||||
}
|
||||
|
||||
func (v *visitor) extractOperation(ctx *grammar.ComparisonContext) (qbtypesv5.FilterOperator, bool) {
|
||||
maybeNot := func(operation qbtypesv5.FilterOperator) qbtypesv5.FilterOperator {
|
||||
if ctx.NOT() != nil {
|
||||
return operation.Inverse()
|
||||
}
|
||||
return operation
|
||||
}
|
||||
switch {
|
||||
case ctx.EQUALS() != nil:
|
||||
return qbtypesv5.FilterOperatorEqual, true
|
||||
case ctx.NOT_EQUALS() != nil, ctx.NEQ() != nil:
|
||||
return qbtypesv5.FilterOperatorNotEqual, true
|
||||
case ctx.LT() != nil:
|
||||
return qbtypesv5.FilterOperatorLessThan, true
|
||||
case ctx.LE() != nil:
|
||||
return qbtypesv5.FilterOperatorLessThanOrEq, true
|
||||
case ctx.GT() != nil:
|
||||
return qbtypesv5.FilterOperatorGreaterThan, true
|
||||
case ctx.GE() != nil:
|
||||
return qbtypesv5.FilterOperatorGreaterThanOrEq, true
|
||||
case ctx.BETWEEN() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorBetween), true
|
||||
case ctx.LIKE() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorLike), true
|
||||
case ctx.ILIKE() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorILike), true
|
||||
case ctx.CONTAINS() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorContains), true
|
||||
case ctx.REGEXP() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorRegexp), true
|
||||
case ctx.InClause() != nil:
|
||||
return qbtypesv5.FilterOperatorIn, true
|
||||
case ctx.NotInClause() != nil:
|
||||
return qbtypesv5.FilterOperatorNotIn, true
|
||||
case ctx.EXISTS() != nil:
|
||||
return maybeNot(qbtypesv5.FilterOperatorExists), true
|
||||
}
|
||||
v.builder.AddError("could not determine operator in expression %q", ctx.GetText())
|
||||
return qbtypesv5.FilterOperatorUnknown, false
|
||||
}
|
||||
|
||||
// OperationName is the user-facing spelling, used only in error messages.
|
||||
func OperationName(operation qbtypesv5.FilterOperator) string {
|
||||
switch operation {
|
||||
case qbtypesv5.FilterOperatorEqual:
|
||||
return "="
|
||||
case qbtypesv5.FilterOperatorNotEqual:
|
||||
return "!="
|
||||
case qbtypesv5.FilterOperatorLessThan:
|
||||
return "<"
|
||||
case qbtypesv5.FilterOperatorLessThanOrEq:
|
||||
return "<="
|
||||
case qbtypesv5.FilterOperatorGreaterThan:
|
||||
return ">"
|
||||
case qbtypesv5.FilterOperatorGreaterThanOrEq:
|
||||
return ">="
|
||||
case qbtypesv5.FilterOperatorBetween:
|
||||
return "BETWEEN"
|
||||
case qbtypesv5.FilterOperatorNotBetween:
|
||||
return "NOT BETWEEN"
|
||||
case qbtypesv5.FilterOperatorLike:
|
||||
return "LIKE"
|
||||
case qbtypesv5.FilterOperatorNotLike:
|
||||
return "NOT LIKE"
|
||||
case qbtypesv5.FilterOperatorILike:
|
||||
return "ILIKE"
|
||||
case qbtypesv5.FilterOperatorNotILike:
|
||||
return "NOT ILIKE"
|
||||
case qbtypesv5.FilterOperatorContains:
|
||||
return "CONTAINS"
|
||||
case qbtypesv5.FilterOperatorNotContains:
|
||||
return "NOT CONTAINS"
|
||||
case qbtypesv5.FilterOperatorRegexp:
|
||||
return "REGEXP"
|
||||
case qbtypesv5.FilterOperatorNotRegexp:
|
||||
return "NOT REGEXP"
|
||||
case qbtypesv5.FilterOperatorIn:
|
||||
return "IN"
|
||||
case qbtypesv5.FilterOperatorNotIn:
|
||||
return "NOT IN"
|
||||
case qbtypesv5.FilterOperatorExists:
|
||||
return "EXISTS"
|
||||
case qbtypesv5.FilterOperatorNotExists:
|
||||
return "NOT EXISTS"
|
||||
}
|
||||
return "?"
|
||||
}
|
||||
|
||||
func trimQuotes(s string) string {
|
||||
if len(s) >= 2 {
|
||||
if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '\'' && s[len(s)-1] == '\'') {
|
||||
s = s[1 : len(s)-1]
|
||||
}
|
||||
}
|
||||
s = strings.ReplaceAll(s, `\\`, `\`)
|
||||
s = strings.ReplaceAll(s, `\'`, `'`)
|
||||
return s
|
||||
}
|
||||
@@ -253,6 +253,7 @@ func NewSQLMigrationProviderFactories(
|
||||
sqlmigration.NewAddQuickFilterTuplesFactory(sqlstore),
|
||||
sqlmigration.NewAddIngestionTuplesFactory(sqlstore),
|
||||
sqlmigration.NewAddSubscriptionTuplesFactory(sqlstore),
|
||||
sqlmigration.NewNormalizeQuickFilterFieldsFactory(sqlstore),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
161
pkg/sqlmigration/126_normalize_quick_filter_fields.go
Normal file
161
pkg/sqlmigration/126_normalize_quick_filter_fields.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
)
|
||||
|
||||
type quickFilterSourceRow struct {
|
||||
bun.BaseModel `bun:"table:quick_filter"`
|
||||
|
||||
ID string `bun:"id,pk"`
|
||||
Source string `bun:"source"`
|
||||
Filter string `bun:"filter"`
|
||||
}
|
||||
|
||||
type quickFilterStaticField struct {
|
||||
name string
|
||||
fieldContext string
|
||||
fieldDataType string
|
||||
}
|
||||
|
||||
// quickFilterSpanFields are the span-level fields the fields API serves with
|
||||
// the span context, keyed by every name a stored filter may carry for them.
|
||||
var quickFilterSpanFields = func() map[string]quickFilterStaticField {
|
||||
fields := map[string]quickFilterStaticField{}
|
||||
for name, dataType := range map[string]string{
|
||||
"trace_id": "string", "span_id": "string", "trace_state": "string", "parent_span_id": "string",
|
||||
"flags": "number", "name": "string", "kind": "number", "kind_string": "string",
|
||||
"duration_nano": "number", "status_code": "number", "status_message": "string", "status_code_string": "string",
|
||||
"response_status_code": "string", "external_http_url": "string", "http_url": "string",
|
||||
"external_http_method": "string", "http_method": "string", "http_host": "string",
|
||||
"db_name": "string", "db_operation": "string", "has_error": "bool", "is_remote": "string",
|
||||
} {
|
||||
fields[name] = quickFilterStaticField{name: name, fieldContext: "span", fieldDataType: dataType}
|
||||
}
|
||||
for deprecated, current := range map[string]string{
|
||||
"responseStatusCode": "response_status_code", "externalHttpUrl": "external_http_url", "httpUrl": "http_url",
|
||||
"externalHttpMethod": "external_http_method", "httpMethod": "http_method", "httpHost": "http_host",
|
||||
"dbName": "db_name", "dbOperation": "db_operation", "hasError": "has_error", "isRemote": "is_remote",
|
||||
} {
|
||||
fields[deprecated] = fields[current]
|
||||
}
|
||||
return fields
|
||||
}()
|
||||
|
||||
// quickFilterLogFields are the log-level fields the fields API serves with
|
||||
// the log context.
|
||||
var quickFilterLogFields = map[string]quickFilterStaticField{
|
||||
"body": {name: "body", fieldContext: "log", fieldDataType: "string"},
|
||||
"severity_text": {name: "severity_text", fieldContext: "log", fieldDataType: "string"},
|
||||
"severity_number": {name: "severity_number", fieldContext: "log", fieldDataType: "number"},
|
||||
"trace_id": {name: "trace_id", fieldContext: "log", fieldDataType: "string"},
|
||||
"span_id": {name: "span_id", fieldContext: "log", fieldDataType: "string"},
|
||||
"trace_flags": {name: "trace_flags", fieldContext: "log", fieldDataType: "number"},
|
||||
}
|
||||
|
||||
type normalizeQuickFilterFields struct {
|
||||
settings factory.ProviderSettings
|
||||
}
|
||||
|
||||
func NewNormalizeQuickFilterFieldsFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("normalize_quick_filter_fields"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
|
||||
return &normalizeQuickFilterFields{settings: ps}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (migration *normalizeQuickFilterFields) Register(migrations *migrate.Migrations) error {
|
||||
return migrations.Register(migration.Up, migration.Down)
|
||||
}
|
||||
|
||||
func (migration *normalizeQuickFilterFields) Up(ctx context.Context, db *bun.DB) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
var rows []*quickFilterSourceRow
|
||||
if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var migrated, skipped int
|
||||
for _, row := range rows {
|
||||
normalized, changed, ok := normalizeQuickFilterEntries(row.Source, row.Filter)
|
||||
if !ok {
|
||||
migration.settings.Logger.WarnContext(ctx, "quick filter could not be parsed, leaving it untouched", slog.String("quick_filter_id", row.ID), slog.String("raw_filter", row.Filter))
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
|
||||
migrated++
|
||||
if _, err := tx.NewUpdate().Model((*quickFilterSourceRow)(nil)).Set("filter = ?", normalized).Where("id = ?", row.ID).Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
migration.settings.Logger.InfoContext(ctx, "normalized quick filter static fields", slog.Int("total", len(rows)), slog.Int("migrated", migrated), slog.Int("skipped", skipped))
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (migration *normalizeQuickFilterFields) Down(context.Context, *bun.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeQuickFilterEntries rewrites the static fields of a stored filter
|
||||
// list to the name, context and data type the fields API serves them with:
|
||||
// span fields for the trace-based sources, log fields for logs, whatever
|
||||
// context the legacy seeds gave them. Other keys are left as they are;
|
||||
// ok=false means unparseable.
|
||||
func normalizeQuickFilterEntries(source string, filter string) (normalized string, changed bool, ok bool) {
|
||||
var staticFields map[string]quickFilterStaticField
|
||||
switch source {
|
||||
case "traces", "api_monitoring", "exceptions", "ai_observability":
|
||||
staticFields = quickFilterSpanFields
|
||||
case "logs":
|
||||
staticFields = quickFilterLogFields
|
||||
default:
|
||||
return "", false, true
|
||||
}
|
||||
|
||||
var entries []telemetryFieldKeyOutput
|
||||
if err := json.Unmarshal([]byte(filter), &entries); err != nil {
|
||||
return "", false, false
|
||||
}
|
||||
|
||||
for i, entry := range entries {
|
||||
field, static := staticFields[entry.Name]
|
||||
if !static {
|
||||
continue
|
||||
}
|
||||
if entry.Name == field.name && entry.FieldContext == field.fieldContext && entry.FieldDataType == field.fieldDataType {
|
||||
continue
|
||||
}
|
||||
entries[i].Name = field.name
|
||||
entries[i].FieldContext = field.fieldContext
|
||||
entries[i].FieldDataType = field.fieldDataType
|
||||
changed = true
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return "", false, true
|
||||
}
|
||||
|
||||
normalizedJSON, err := marshalUnescaped(entries)
|
||||
if err != nil {
|
||||
return "", false, false
|
||||
}
|
||||
return string(normalizedJSON), true, true
|
||||
}
|
||||
61
pkg/telemetrymetadata/bool_values.go
Normal file
61
pkg/telemetrymetadata/bool_values.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package telemetrymetadata
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// boolFieldValues is the suggestion set for a bool field, optionally narrowed
|
||||
// by the search text.
|
||||
func boolFieldValues(searchText string) *telemetrytypes.TelemetryFieldValues {
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
needle := strings.ToLower(searchText)
|
||||
for _, v := range []bool{true, false} {
|
||||
if needle == "" || strings.Contains(strconv.FormatBool(v), needle) {
|
||||
values.BoolValues = append(values.BoolValues, v)
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// spanSearchScopeFieldValues is the suggestion set for a search-scope selector
|
||||
// (isRoot, isEntryPoint), which only filters with true. ok is false for any
|
||||
// other name.
|
||||
func spanSearchScopeFieldValues(name, searchText string) (*telemetrytypes.TelemetryFieldValues, bool) {
|
||||
for scopeName := range tracestelemetryschema.SpanSearchScopeFields {
|
||||
if !strings.EqualFold(scopeName, name) {
|
||||
continue
|
||||
}
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
if needle := strings.ToLower(searchText); needle == "" || strings.Contains("true", needle) {
|
||||
values.BoolValues = []bool{true}
|
||||
}
|
||||
return values, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// isKnownBoolField is true when the caller asked for the bool data type, or
|
||||
// when the name is one of the signal's static bool fields and the requested
|
||||
// context does not rule that static field out.
|
||||
func isKnownBoolField(selector *telemetrytypes.FieldValueSelector, staticFields ...map[string]telemetrytypes.TelemetryFieldKey) bool {
|
||||
if selector.FieldDataType == telemetrytypes.FieldDataTypeBool {
|
||||
return true
|
||||
}
|
||||
if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
|
||||
return false
|
||||
}
|
||||
for _, fields := range staticFields {
|
||||
field, ok := fields[selector.Name]
|
||||
if !ok || field.FieldDataType != telemetrytypes.FieldDataTypeBool {
|
||||
continue
|
||||
}
|
||||
if selector.FieldContext == telemetrytypes.FieldContextUnspecified || selector.FieldContext == field.FieldContext {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -187,8 +187,6 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
).From(t.tracesDBName + "." + t.spanAttributesKeysTblName)
|
||||
var limit int
|
||||
|
||||
searchTexts := []string{}
|
||||
|
||||
conds := []string{}
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
|
||||
@@ -208,14 +206,12 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
fieldKeyConds = append(fieldKeyConds, sb.ILike("tagKey", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
|
||||
}
|
||||
|
||||
searchTexts = append(searchTexts, fieldKeySelector.Name)
|
||||
// now look at the field context
|
||||
// we don't write most of intrinsic fields to keys table
|
||||
// for this reason we don't want to apply tagType if the field context
|
||||
// is not attribute or resource attribute
|
||||
if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified &&
|
||||
(fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute ||
|
||||
fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) {
|
||||
// is not attribute, resource attribute or scope
|
||||
switch fieldKeySelector.FieldContext {
|
||||
case telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope:
|
||||
fieldKeyConds = append(fieldKeyConds, sb.E("tagType", fieldKeySelector.FieldContext.TagType()))
|
||||
}
|
||||
|
||||
@@ -288,41 +284,20 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
// hit the limit? (only counting DB results)
|
||||
complete := rowCount <= limit
|
||||
|
||||
staticKeys := []string{"isRoot", "isEntryPoint"}
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.CalculatedFields)...)
|
||||
// Add the matching static fields: the span scope selectors, the intrinsic
|
||||
// columns and the calculated columns. These don't count towards the limit
|
||||
staticFields := maps.Values(tracestelemetryschema.SpanSearchScopeFields)
|
||||
staticFields = append(staticFields, maps.Values(tracestelemetryschema.IntrinsicFields)...)
|
||||
staticFields = append(staticFields, maps.Values(tracestelemetryschema.CalculatedFields)...)
|
||||
|
||||
// Add matching intrinsic and matching calculated fields
|
||||
// These don't count towards the limit
|
||||
for _, key := range staticKeys {
|
||||
found := false
|
||||
for _, v := range searchTexts {
|
||||
if v == "" || strings.Contains(key, v) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
for _, field := range staticFields {
|
||||
if !staticFieldMatchesAny(field, fieldKeySelectors) {
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := tracestelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if field, exists := tracestelemetryschema.CalculatedFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
continue
|
||||
}
|
||||
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: key,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
})
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
|
||||
if err = t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
|
||||
@@ -542,12 +517,6 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
allArgs = append(allArgs, args...)
|
||||
}
|
||||
|
||||
if len(queries) == 0 {
|
||||
// No matching contexts, return empty result
|
||||
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
|
||||
}
|
||||
|
||||
// Combine queries with UNION ALL
|
||||
var limit int
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
limit += fieldKeySelector.Limit
|
||||
@@ -556,7 +525,15 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
mainQuery := fmt.Sprintf(`
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
parentTypes := make(map[string][]telemetrytypes.FieldDataType)
|
||||
rowCount := 0
|
||||
|
||||
// the log and scope contexts have no keys table; they are served by the
|
||||
// static fields appended below
|
||||
if len(queries) > 0 {
|
||||
// Combine queries with UNION ALL
|
||||
mainQuery := fmt.Sprintf(`
|
||||
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
|
||||
FROM (
|
||||
%s
|
||||
@@ -566,103 +543,75 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
LIMIT %d
|
||||
`, strings.Join(queries, " UNION ALL "), limit+1)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
parentTypes := make(map[string][]telemetrytypes.FieldDataType)
|
||||
rowCount := 0
|
||||
searchTexts := []string{}
|
||||
|
||||
// Collect search texts for static field matching
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
searchTexts = append(searchTexts, fieldKeySelector.Name)
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
var priority uint8
|
||||
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// ArrayJSON/ArrayDynamic body rows for parent paths are needed by the JSON access plan
|
||||
// builder (enrichJSONKeys). Always record them in parentTypes. Only skip adding to keys
|
||||
// if the user did not also directly request this name — a field like "education" can be
|
||||
// both a parent of "education[].name" and an explicitly queried field in its own right.
|
||||
switch fieldDataType {
|
||||
case telemetrytypes.FieldDataTypeArrayJSON, telemetrytypes.FieldDataTypeArrayDynamic:
|
||||
if fieldContext == telemetrytypes.FieldContextBody && parentPaths[name] {
|
||||
parentTypes[name] = append(parentTypes[name], fieldDataType)
|
||||
if !mapOfRequestedSelectors[name] {
|
||||
continue // skip; don't register the key.
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
var priority uint8
|
||||
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
|
||||
// ArrayJSON/ArrayDynamic body rows for parent paths are needed by the JSON access plan
|
||||
// builder (enrichJSONKeys). Always record them in parentTypes. Only skip adding to keys
|
||||
// if the user did not also directly request this name — a field like "education" can be
|
||||
// both a parent of "education[].name" and an explicitly queried field in its own right.
|
||||
switch fieldDataType {
|
||||
case telemetrytypes.FieldDataTypeArrayJSON, telemetrytypes.FieldDataTypeArrayDynamic:
|
||||
if fieldContext == telemetrytypes.FieldContextBody && parentPaths[name] {
|
||||
parentTypes[name] = append(parentTypes[name], fieldDataType)
|
||||
if !mapOfRequestedSelectors[name] {
|
||||
continue // skip; don't register the key.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
|
||||
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
|
||||
|
||||
// if there is no materialised column, create a key with the field context and data type
|
||||
if !ok {
|
||||
key = &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
// if there is no materialised column, create a key with the field context and data type
|
||||
if !ok {
|
||||
key = &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
}
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
if rows.Err() != nil {
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
if rows.Err() != nil {
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// hit the limit? (only counting DB results)
|
||||
complete := rowCount <= limit
|
||||
|
||||
staticKeys := []string{}
|
||||
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
|
||||
|
||||
// Add matching intrinsic and matching calculated fields
|
||||
// These don't count towards the limit
|
||||
for _, key := range staticKeys {
|
||||
found := false
|
||||
for _, v := range searchTexts {
|
||||
if v == "" || strings.Contains(key, v) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
// Add the matching intrinsic columns. These don't count towards the limit
|
||||
for _, field := range maps.Values(logstelemetryschema.IntrinsicFields) {
|
||||
if !staticFieldMatchesAny(field, fieldKeySelectors) {
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := logstelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: key,
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
})
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
|
||||
// enrich body keys with promoted paths, indexes, and JSON access plans
|
||||
@@ -806,10 +755,6 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
allArgs = append(allArgs, args...)
|
||||
}
|
||||
|
||||
if len(queries) == 0 {
|
||||
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
|
||||
}
|
||||
|
||||
var limit int
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
limit += fieldKeySelector.Limit
|
||||
@@ -818,7 +763,13 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
mainQuery := fmt.Sprintf(`
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
rowCount := 0
|
||||
|
||||
// the log and scope contexts have no keys table; they are served by the
|
||||
// static fields appended below
|
||||
if len(queries) > 0 {
|
||||
mainQuery := fmt.Sprintf(`
|
||||
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
|
||||
FROM (
|
||||
%s
|
||||
@@ -828,73 +779,57 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
LIMIT %d
|
||||
`, strings.Join(queries, " UNION ALL "), limit+1)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
rowCount := 0
|
||||
searchTexts := []string{}
|
||||
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
searchTexts = append(searchTexts, fieldKeySelector.Name)
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
var priority uint8
|
||||
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
}
|
||||
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
|
||||
defer rows.Close()
|
||||
|
||||
if !ok {
|
||||
key = &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
var priority uint8
|
||||
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
}
|
||||
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
|
||||
|
||||
if !ok {
|
||||
key = &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
}
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
if rows.Err() != nil {
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
if rows.Err() != nil {
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
}
|
||||
}
|
||||
|
||||
complete := rowCount <= limit
|
||||
|
||||
// Add intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
|
||||
staticKeys := maps.Keys(audittelemetryschema.IntrinsicFields)
|
||||
for _, key := range staticKeys {
|
||||
found := false
|
||||
for _, v := range searchTexts {
|
||||
if v == "" || strings.Contains(key, v) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
// Add the matching intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
|
||||
for _, field := range maps.Values(audittelemetryschema.IntrinsicFields) {
|
||||
if !staticFieldMatchesAny(field, fieldKeySelectors) {
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := audittelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
}
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
|
||||
return keys, complete, nil
|
||||
@@ -1091,9 +1026,12 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterKeys.Error())
|
||||
}
|
||||
// meter labels are stored as strings in the labels JSON and have no
|
||||
// attribute context, so only the data type is known
|
||||
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1506,88 +1444,13 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS
|
||||
instrumentationtypes.CodeNamespace: "metadata",
|
||||
instrumentationtypes.CodeFunctionName: "getSpanFieldValues",
|
||||
})
|
||||
// build the query to get the keys from the spans that match the field selection criteria
|
||||
limit := fieldValueSelector.Limit
|
||||
if limit == 0 {
|
||||
limit = 50
|
||||
|
||||
if values, ok := spanSearchScopeFieldValues(fieldValueSelector.Name, fieldValueSelector.Value); ok {
|
||||
return values, true, nil
|
||||
}
|
||||
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.tracesDBName + "." + t.tracesFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
|
||||
}
|
||||
|
||||
// now look at the field context
|
||||
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
|
||||
}
|
||||
|
||||
// now look at the field data type
|
||||
if fieldValueSelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
|
||||
sb.Where(sb.E("tag_data_type", fieldValueSelector.FieldDataType.TagDataType()))
|
||||
}
|
||||
|
||||
if fieldValueSelector.Value != "" {
|
||||
switch fieldValueSelector.FieldDataType {
|
||||
case telemetrytypes.FieldDataTypeString:
|
||||
sb.Where(sb.ILike("string_value", "%"+escapeForLike(fieldValueSelector.Value)+"%"))
|
||||
case telemetrytypes.FieldDataTypeNumber:
|
||||
sb.Where(sb.IsNotNull("number_value"))
|
||||
sb.Where(sb.ILike("toString(number_value)", "%"+escapeForLike(fieldValueSelector.Value)+"%"))
|
||||
case telemetrytypes.FieldDataTypeUnspecified:
|
||||
// or b/w string and number
|
||||
sb.Where(sb.Or(
|
||||
sb.ILike("string_value", "%"+escapeForLike(fieldValueSelector.Value)+"%"),
|
||||
sb.ILike("toString(number_value)", "%"+escapeForLike(fieldValueSelector.Value)+"%"),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// query one extra to check if we hit the limit
|
||||
sb.Limit(limit + 1)
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
seen := make(map[string]bool)
|
||||
rowCount := 0
|
||||
totalCount := 0 // Track total unique values
|
||||
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
|
||||
var stringValue string
|
||||
var numberValue float64
|
||||
if err := rows.Scan(&stringValue, &numberValue); err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
|
||||
// Only add values if we haven't hit the limit yet
|
||||
if totalCount < limit {
|
||||
if _, ok := seen[stringValue]; !ok && stringValue != "" {
|
||||
values.StringValues = append(values.StringValues, stringValue)
|
||||
seen[stringValue] = true
|
||||
totalCount++
|
||||
}
|
||||
if _, ok := seen[fmt.Sprintf("%f", numberValue)]; !ok && numberValue != 0 && totalCount < limit {
|
||||
values.NumberValues = append(values.NumberValues, numberValue)
|
||||
seen[fmt.Sprintf("%f", numberValue)] = true
|
||||
totalCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hit the limit?
|
||||
complete := rowCount <= limit
|
||||
|
||||
return values, complete, nil
|
||||
knownBool := isKnownBoolField(fieldValueSelector, tracestelemetryschema.IntrinsicFields, tracestelemetryschema.CalculatedFields)
|
||||
// unix_milli is the hour of the span start
|
||||
return t.getTagTableValues(ctx, t.tracesDBName+"."+t.tracesFieldsTblName, fieldValueSelector, knownBool)
|
||||
}
|
||||
|
||||
func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) {
|
||||
@@ -1596,17 +1459,77 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
|
||||
instrumentationtypes.CodeNamespace: "metadata",
|
||||
instrumentationtypes.CodeFunctionName: "getLogFieldValues",
|
||||
})
|
||||
// build the query to get the keys from the spans that match the field selection criteria
|
||||
|
||||
knownBool := isKnownBoolField(fieldValueSelector, logstelemetryschema.IntrinsicFields)
|
||||
// unix_milli is the hour the log was ingested, not the log's own timestamp
|
||||
return t.getTagTableValues(ctx, t.logsDBName+"."+t.logsFieldsTblName, fieldValueSelector, knownBool)
|
||||
}
|
||||
|
||||
// tagTableSinceDay restricts rows to the tag table's day partitions from the
|
||||
// start's day on. Partitions are toDate(unix_milli / 1000) in the server's
|
||||
// timezone, and a value's surviving row within a day carries whichever hour
|
||||
// was inserted last, so the day is the finest safe unit.
|
||||
func tagTableSinceDay(sb *sqlbuilder.SelectBuilder, startUnixMilli int64) {
|
||||
if startUnixMilli != 0 {
|
||||
sb.Where(fmt.Sprintf("toDate(unix_milli / 1000) >= toDate(%d)", startUnixMilli/1000))
|
||||
}
|
||||
}
|
||||
|
||||
// tagTableHasBoolRows reports whether the tag table holds a bool row for the
|
||||
// key. Bool rows carry no value, so one row is enough to know the key takes
|
||||
// the values true and false.
|
||||
func (t *telemetryMetaStore) tagTableHasBoolRows(ctx context.Context, table string, selector *telemetrytypes.FieldValueSelector) (bool, error) {
|
||||
sb := sqlbuilder.Select("1").From(table)
|
||||
sb.Where(sb.E("tag_key", selector.Name))
|
||||
sb.Where(sb.E("tag_data_type", telemetrytypes.FieldDataTypeBool.TagDataType()))
|
||||
if selector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
sb.Where(sb.E("tag_type", selector.FieldContext.TagType()))
|
||||
}
|
||||
tagTableSinceDay(sb, selector.StartUnixMilli)
|
||||
sb.Limit(1)
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
return rows.Next(), rows.Err()
|
||||
}
|
||||
|
||||
// getTagTableValues returns the string and number values of the key from a
|
||||
// tag table, and true and false when the key is a known bool field or the
|
||||
// table holds bool rows for it. Bool rows do not count towards the limit.
|
||||
func (t *telemetryMetaStore) getTagTableValues(ctx context.Context, table string, fieldValueSelector *telemetrytypes.FieldValueSelector, knownBool bool) (*telemetrytypes.TelemetryFieldValues, bool, error) {
|
||||
limit := fieldValueSelector.Limit
|
||||
if limit == 0 {
|
||||
limit = 50
|
||||
}
|
||||
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.logsDBName + "." + t.logsFieldsTblName)
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
if knownBool {
|
||||
values.BoolValues = boolFieldValues(fieldValueSelector.Value).BoolValues
|
||||
if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeBool {
|
||||
return values, true, nil
|
||||
}
|
||||
} else if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeUnspecified {
|
||||
hasBoolRows, err := t.tagTableHasBoolRows(ctx, table, fieldValueSelector)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if hasBoolRows {
|
||||
values.BoolValues = boolFieldValues(fieldValueSelector.Value).BoolValues
|
||||
}
|
||||
}
|
||||
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(table)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
|
||||
}
|
||||
sb.Where(sb.NE("tag_data_type", telemetrytypes.FieldDataTypeBool.TagDataType()))
|
||||
|
||||
tagTableSinceDay(sb, fieldValueSelector.StartUnixMilli)
|
||||
|
||||
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
|
||||
@@ -1643,7 +1566,6 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
seen := make(map[string]bool)
|
||||
rowCount := 0
|
||||
totalCount := 0 // Track total unique values
|
||||
@@ -2097,6 +2019,18 @@ func populateAllUnspecifiedValues(allUnspecifiedValues *telemetrytypes.Telemetry
|
||||
}
|
||||
}
|
||||
|
||||
for _, value := range values.BoolValues {
|
||||
if totalCount >= limit {
|
||||
complete = false
|
||||
break
|
||||
}
|
||||
if _, ok := mapOfValues[value]; !ok {
|
||||
mapOfValues[value] = true
|
||||
allUnspecifiedValues.BoolValues = append(allUnspecifiedValues.BoolValues, value)
|
||||
totalCount++
|
||||
}
|
||||
}
|
||||
|
||||
for _, value := range values.RelatedValues {
|
||||
if totalCount >= limit {
|
||||
complete = false
|
||||
@@ -2467,6 +2401,10 @@ func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Conte
|
||||
|
||||
// updateColumnEvolutionMetadataForKeys updates the evolution field for keys.
|
||||
func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) error {
|
||||
// an empty selector list would run the evolution query without a filter
|
||||
if len(keysToUpdate) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var metadataKeySelectors []*telemetrytypes.EvolutionSelector
|
||||
for _, keySelector := range keysToUpdate {
|
||||
|
||||
53
pkg/telemetrymetadata/static_fields.go
Normal file
53
pkg/telemetrymetadata/static_fields.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package telemetrymetadata
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
func staticFieldMatchesAny(field telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) bool {
|
||||
for _, selector := range selectors {
|
||||
if staticFieldMatches(field, selector) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// staticFieldMatches mirrors the keys-table lookup for a static field: the
|
||||
// requested context and data type, when given, must agree with the field's,
|
||||
// and the name matches case-insensitively, as a substring for fuzzy selectors
|
||||
// and as the whole name for exact ones.
|
||||
func staticFieldMatches(field telemetrytypes.TelemetryFieldKey, selector *telemetrytypes.FieldKeySelector) bool {
|
||||
if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != field.FieldContext {
|
||||
return false
|
||||
}
|
||||
if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && !sameDataTypeFamily(selector.FieldDataType, field.FieldDataType) {
|
||||
return false
|
||||
}
|
||||
if selector.Name == "" {
|
||||
return true
|
||||
}
|
||||
if selector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
return strings.EqualFold(field.Name, selector.Name)
|
||||
}
|
||||
return strings.Contains(strings.ToLower(field.Name), strings.ToLower(selector.Name))
|
||||
}
|
||||
|
||||
// sameDataTypeFamily treats the numeric types as one family: static fields
|
||||
// declare "number" while callers may ask for int64 or float64.
|
||||
func sameDataTypeFamily(requested, actual telemetrytypes.FieldDataType) bool {
|
||||
if requested == actual {
|
||||
return true
|
||||
}
|
||||
return isNumericDataType(requested) && isNumericDataType(actual)
|
||||
}
|
||||
|
||||
func isNumericDataType(dataType telemetrytypes.FieldDataType) bool {
|
||||
switch dataType {
|
||||
case telemetrytypes.FieldDataTypeNumber, telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -392,6 +392,24 @@ var (
|
||||
SpanSearchScopeRoot = "isroot"
|
||||
SpanSearchScopeEntryPoint = "isentrypoint"
|
||||
|
||||
// SpanSearchScopeFields are the search-scope selectors (isRoot, isEntryPoint),
|
||||
// not columns and unrelated to the instrumentation scope: they only filter
|
||||
// with the value true.
|
||||
SpanSearchScopeFields = map[string]telemetrytypes.TelemetryFieldKey{
|
||||
"isRoot": {
|
||||
Name: "isRoot",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
"isEntryPoint": {
|
||||
Name: "isEntryPoint",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
}
|
||||
|
||||
// IntrinsicSpanFields lists the intrinsic span columns, in the order they
|
||||
// should appear when a raw query expands its SelectFields.
|
||||
IntrinsicSpanFields = []telemetrytypes.TelemetryFieldKey{
|
||||
|
||||
@@ -173,18 +173,18 @@ func NewSourceFilterFromStorableQuickFilter(storableQuickFilter *StorableQuickFi
|
||||
// NewDefaultQuickFilter generates default filters for all supported sources.
|
||||
func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
|
||||
tracesFilters := []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "duration_nano", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber},
|
||||
{Name: "duration_nano", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeNumber},
|
||||
{Name: "deployment.environment", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "hasError", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeBool},
|
||||
{Name: "has_error", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeBool},
|
||||
{Name: "service.name", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "name", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "name", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "rpc.method", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "response_status_code", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http_host", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "response_status_code", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http_host", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http.method", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http.route", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http_url", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "trace_id", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http_url", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "trace_id", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
}
|
||||
|
||||
logsFilters := []telemetrytypes.TelemetryFieldKey{
|
||||
|
||||
235
tests/integration/tests/queriercommon/07_fields_keys_values.py
Normal file
235
tests/integration/tests/queriercommon/07_fields_keys_values.py
Normal file
@@ -0,0 +1,235 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.traces import Traces
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal,field_context,present,absent",
|
||||
[
|
||||
pytest.param("logs", "log", {"severity_text": "log", "body": "log", "trace_id": "log"}, ["code.file", "scope_name"], id="log_context_lists_log_intrinsics"),
|
||||
pytest.param("logs", "scope", {"scope_name": "scope", "scope_version": "scope"}, ["severity_text", "body", "code.file"], id="scope_context_lists_scope_intrinsics_for_logs"),
|
||||
pytest.param("logs", "attribute", {"code.file": "attribute"}, ["body", "scope_name"], id="attribute_context_excludes_log_intrinsics"),
|
||||
pytest.param("traces", "span", {"name": "span", "has_error": "span", "isRoot": "span", "http.method": "attribute"}, ["scope.name"], id="span_context_lists_span_intrinsics_and_attributes"),
|
||||
pytest.param("traces", "scope", {"scope.name": "scope", "scope.version": "scope"}, ["name", "has_error", "isRoot", "http.method", "host.name"], id="scope_context_lists_scope_intrinsics_for_traces"),
|
||||
pytest.param("traces", "resource", {"host.name": "resource"}, ["name", "has_error", "isRoot", "http.method"], id="resource_context_excludes_span_intrinsics"),
|
||||
pytest.param("traces", "attribute", {"http.method": "attribute"}, ["name", "has_error", "isRoot", "host.name"], id="attribute_context_excludes_span_intrinsics"),
|
||||
],
|
||||
)
|
||||
def test_fields_keys_by_context(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
signal: str,
|
||||
field_context: str,
|
||||
present: dict[str, str],
|
||||
absent: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert a log with a code.file attribute and a span with an http.method attribute and a host.name resource.
|
||||
|
||||
Tests:
|
||||
1. Keys for a context list that context's intrinsic columns and the stored keys the context maps to,
|
||||
each with its context; intrinsics of other contexts are not listed. The span context also keeps
|
||||
listing attributes because `span.<attribute>` resolves attributes in queries.
|
||||
"""
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs([Logs(timestamp=now, attributes={"code.file": "/opt/integration.go"}, body="a log line")])
|
||||
insert_traces([Traces(timestamp=now, resources={"host.name": "linux-001"}, attributes={"http.method": "GET"})])
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, "fieldContext": field_context},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
keys = response.json()["data"]["keys"]
|
||||
listed = {name: [key["fieldContext"] for key in keys.get(name, [])] for name in present}
|
||||
assert listed == {name: [context] for name, context in present.items()}, f"keys for the {field_context} context"
|
||||
assert [name for name in absent if name in keys] == [], f"keys that do not belong to the {field_context} context"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal,field_context,field_data_type,present,absent",
|
||||
[
|
||||
pytest.param("traces", "span", "float64", ["duration_nano", "status_code"], ["name", "has_error"], id="float64_matches_number_span_intrinsics"),
|
||||
pytest.param("traces", "span", "int64", ["duration_nano", "status_code"], ["name", "has_error"], id="int64_matches_number_span_intrinsics"),
|
||||
pytest.param("traces", "span", "bool", ["has_error", "isRoot", "isEntryPoint"], ["name", "duration_nano"], id="bool_matches_bool_span_intrinsics"),
|
||||
pytest.param("traces", "span", "string", ["name", "http_method"], ["duration_nano", "has_error"], id="string_matches_string_span_intrinsics"),
|
||||
pytest.param("logs", "log", "number", ["severity_number", "trace_flags"], ["severity_text", "body"], id="number_matches_number_log_intrinsics"),
|
||||
],
|
||||
)
|
||||
def test_fields_keys_by_data_type(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
signal: str,
|
||||
field_context: str,
|
||||
field_data_type: str,
|
||||
present: list[str],
|
||||
absent: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
Tests:
|
||||
1. A data type filter keeps the intrinsic columns of that type; number, int64 and float64 are one family.
|
||||
"""
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, "fieldContext": field_context, "fieldDataType": field_data_type},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert [name for name in present if name not in keys] == [], f"intrinsics of type {field_data_type}"
|
||||
assert [name for name in absent if name in keys] == [], f"intrinsics not of type {field_data_type}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal,search_text,present",
|
||||
[
|
||||
pytest.param("logs", "SEVERITY", ["severity_text", "severity_number"], id="upper_case_search_logs"),
|
||||
pytest.param("traces", "HTTP_", ["http_method", "http_host", "http_url"], id="upper_case_search_traces"),
|
||||
pytest.param("traces", "Duration", ["duration_nano"], id="mixed_case_search_traces"),
|
||||
pytest.param("traces", "span.HAS_ERR", ["has_error"], id="context_prefix_with_upper_case_search"),
|
||||
],
|
||||
)
|
||||
def test_fields_keys_search_matches_intrinsics_case_insensitively(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
signal: str,
|
||||
search_text: str,
|
||||
present: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
Tests:
|
||||
1. The search text matches intrinsic columns case-insensitively, as it does for stored keys,
|
||||
with or without a context prefix.
|
||||
"""
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, "searchText": search_text},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert [name for name in present if name not in keys] == [], f"intrinsics matching {search_text!r}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal,params,expected",
|
||||
[
|
||||
pytest.param("traces", {"name": "has_error"}, [True, False], id="calculated_bool_span_field"),
|
||||
pytest.param("traces", {"name": "has_error", "fieldContext": "span"}, [True, False], id="calculated_bool_span_field_with_context"),
|
||||
pytest.param("traces", {"name": "has_error", "searchText": "tr"}, [True], id="search_text_narrows_bool_values"),
|
||||
pytest.param("traces", {"name": "isRoot"}, [True], id="span_scope_field_is_true_only"),
|
||||
pytest.param("logs", {"name": "retry"}, [True, False], id="bool_attribute_from_tag_rows"),
|
||||
pytest.param("logs", {"name": "retry", "fieldContext": "attribute"}, [True, False], id="bool_attribute_with_context"),
|
||||
pytest.param("logs", {"name": "retry", "searchText": "tr"}, [True], id="search_text_narrows_stored_bool_values"),
|
||||
pytest.param("logs", {"name": "never_seen", "fieldDataType": "bool"}, [True, False], id="declared_bool_type_needs_no_rows"),
|
||||
],
|
||||
)
|
||||
def test_fields_values_bool_fields(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
signal: str,
|
||||
params: dict[str, str],
|
||||
expected: list[bool],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert a log with a bool attribute.
|
||||
|
||||
Tests:
|
||||
1. Values for a bool field are true and false (narrowed by the search text): for the calculated span
|
||||
field, for a stored bool attribute whose tag rows carry no value, and for a key the caller
|
||||
declares bool.
|
||||
2. A span scope selector (isRoot) only takes true.
|
||||
"""
|
||||
insert_logs([Logs(timestamp=datetime.now(tz=UTC), attributes={"retry": True}, body="retrying")])
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, **params},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["data"]["values"]["boolValues"] == expected
|
||||
assert response.json()["data"]["complete"] is True
|
||||
|
||||
|
||||
def test_fields_values_start_excludes_span_values_not_seen_since_the_day(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert a span three days old and a span now, with different service names.
|
||||
|
||||
Tests:
|
||||
1. Values with startUnixMilli an hour ago contain only the service seen today: the start is
|
||||
floored to the day, the tag table's deduplication unit.
|
||||
2. Values without a start contain both services.
|
||||
|
||||
Logs are not covered: the logs collector stamps tag rows with the ingestion hour, not the
|
||||
log's timestamp, and the fixture writes the log's timestamp.
|
||||
"""
|
||||
signal = "traces"
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_traces(
|
||||
[
|
||||
Traces(timestamp=now - timedelta(days=3), resources={"service.name": "archived-service"}),
|
||||
Traces(timestamp=now, resources={"service.name": "live-service"}),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": signal,
|
||||
"name": "service.name",
|
||||
"startUnixMilli": int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["data"]["values"]["stringValues"] == ["live-service"], "values last seen before the start must be dropped"
|
||||
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, "name": "service.name"},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert set(response.json()["data"]["values"]["stringValues"]) == {"archived-service", "live-service"}
|
||||
@@ -71,7 +71,7 @@ def test_v1_get_serves_legacy_shape(
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
filters = response.json()["data"]["filters"]
|
||||
assert filters[0]["key"] == "duration_nano"
|
||||
assert filters[0]["type"] == "tag"
|
||||
assert filters[0]["type"] == "", "span fields have no v3 attribute type"
|
||||
assert filters[0]["dataType"] == "float64"
|
||||
assert all("name" not in legacy_filter for legacy_filter in filters)
|
||||
|
||||
@@ -274,3 +274,36 @@ def test_update_quick_filters_rejects_invalid_input(
|
||||
timeout=2,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
|
||||
|
||||
def test_default_traces_filters_are_served_as_the_fields_api_serves_them(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
):
|
||||
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v2/quick_filters/traces"),
|
||||
headers={"Authorization": f"Bearer {admin_token}"},
|
||||
timeout=2,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
filters = {field_key["name"]: field_key for field_key in response.json()["data"]["filters"]}
|
||||
|
||||
assert "hasError" not in filters
|
||||
assert (filters["has_error"]["fieldContext"], filters["has_error"]["fieldDataType"]) == ("span", "bool")
|
||||
assert (filters["name"]["fieldContext"], filters["name"]["fieldDataType"]) == ("span", "string")
|
||||
assert (filters["duration_nano"]["fieldContext"], filters["duration_nano"]["fieldDataType"]) == ("span", "number")
|
||||
assert (filters["http.route"]["fieldContext"], filters["http.route"]["fieldDataType"]) == ("attribute", "string")
|
||||
|
||||
for name in ("has_error", "name"):
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
headers={"Authorization": f"Bearer {admin_token}"},
|
||||
timeout=2,
|
||||
params={"signal": "traces", "searchText": name, "fieldContext": filters[name]["fieldContext"]},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
served = response.json()["data"]["keys"][name]
|
||||
assert (filters[name]["fieldContext"], filters[name]["fieldDataType"]) in [(key["fieldContext"], key["fieldDataType"]) for key in served]
|
||||
|
||||
Reference in New Issue
Block a user