mirror of
https://github.com/SigNoz/signoz.git
synced 2026-07-14 18:40:26 +01:00
Compare commits
12 Commits
feat/rules
...
issue-1000
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
349a70156e | ||
|
|
4decb41df8 | ||
|
|
0aa17540a7 | ||
|
|
585caa84fe | ||
|
|
814fd40a1d | ||
|
|
0978bdfa7f | ||
|
|
4f99261743 | ||
|
|
836988273f | ||
|
|
a622d65226 | ||
|
|
2410e3d411 | ||
|
|
654e2e4b7e | ||
|
|
03ad7a85fa |
1
.github/workflows/integrationci.yaml
vendored
1
.github/workflows/integrationci.yaml
vendored
@@ -53,6 +53,7 @@ jobs:
|
||||
- queriermetrics
|
||||
- querierscalar
|
||||
- queriercommon
|
||||
- queriervariables
|
||||
- rawexportdata
|
||||
- role
|
||||
- rootuser
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
@@ -246,28 +247,37 @@ func (handler *handler) ReplaceVariables(rw http.ResponseWriter, req *http.Reque
|
||||
switch spec := item.Spec.(type) {
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
|
||||
if spec.Filter != nil && spec.Filter.Expression != "" {
|
||||
replaced, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
|
||||
replaced, warnings, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
handler.set.Logger.WarnContext(req.Context(), "variable replace warnings", slog.Any("warnings", warnings))
|
||||
}
|
||||
spec.Filter.Expression = replaced
|
||||
}
|
||||
queryRangeRequest.CompositeQuery.Queries[idx].Spec = spec
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
|
||||
if spec.Filter != nil && spec.Filter.Expression != "" {
|
||||
replaced, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
|
||||
replaced, warnings, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
handler.set.Logger.WarnContext(req.Context(), "variable replace warnings", slog.Any("warnings", warnings))
|
||||
}
|
||||
spec.Filter.Expression = replaced
|
||||
}
|
||||
queryRangeRequest.CompositeQuery.Queries[idx].Spec = spec
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
|
||||
if spec.Filter != nil && spec.Filter.Expression != "" {
|
||||
replaced, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
|
||||
replaced, warnings, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
handler.set.Logger.WarnContext(req.Context(), "variable replace warnings", slog.Any("warnings", warnings))
|
||||
}
|
||||
spec.Filter.Expression = replaced
|
||||
}
|
||||
queryRangeRequest.CompositeQuery.Queries[idx].Spec = spec
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/variables"
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
@@ -428,6 +429,11 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
values = []any{ret}
|
||||
}
|
||||
|
||||
// a value with an embedded variable resolved to __all__ makes the whole condition moot
|
||||
if slices.ContainsFunc(values, isSkipConditionValue) {
|
||||
return SkipConditionLiteral
|
||||
}
|
||||
|
||||
if len(values) == 1 {
|
||||
if var_, ok := values[0].(string); ok {
|
||||
// check if this is a variables
|
||||
@@ -443,7 +449,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
// we have a variable, now check for dynamic variable
|
||||
if varItem.Type == qbtypes.DynamicVariableType {
|
||||
// check if it is special value to skip entire filter, if so skip it
|
||||
if all_, ok := varItem.Value.(string); ok && all_ == "__all__" {
|
||||
if all_, ok := varItem.Value.(string); ok && all_ == qbtypes.AllVariableValue {
|
||||
return SkipConditionLiteral
|
||||
}
|
||||
}
|
||||
@@ -498,6 +504,11 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
value1 := v.Visit(values[0])
|
||||
value2 := v.Visit(values[1])
|
||||
|
||||
// a bound with an embedded variable resolved to __all__ makes the whole condition moot
|
||||
if isSkipConditionValue(value1) || isSkipConditionValue(value2) {
|
||||
return SkipConditionLiteral
|
||||
}
|
||||
|
||||
switch value1.(type) {
|
||||
case float64:
|
||||
if _, ok := value2.(float64); !ok {
|
||||
@@ -535,6 +546,11 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
if len(values) > 0 {
|
||||
value := v.Visit(values[0])
|
||||
|
||||
// the value had an embedded variable resolved to __all__; drop the condition
|
||||
if isSkipConditionValue(value) {
|
||||
return SkipConditionLiteral
|
||||
}
|
||||
|
||||
if var_, ok := value.(string); ok {
|
||||
// check if this is a variables
|
||||
var ok bool
|
||||
@@ -546,6 +562,13 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
}
|
||||
|
||||
if ok {
|
||||
if varItem.Type == qbtypes.DynamicVariableType {
|
||||
// __all__ used with a single-value operator: there is nothing to
|
||||
// filter on, drop the condition like the IN clause handling does
|
||||
if all_, ok := varItem.Value.(string); ok && all_ == qbtypes.AllVariableValue {
|
||||
return SkipConditionLiteral
|
||||
}
|
||||
}
|
||||
switch varValues := varItem.Value.(type) {
|
||||
case []any:
|
||||
if len(varValues) == 0 {
|
||||
@@ -731,6 +754,11 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
|
||||
|
||||
value := params[1:]
|
||||
|
||||
// a param with an embedded variable resolved to __all__ makes the whole condition moot
|
||||
if slices.ContainsFunc(value, isSkipConditionValue) {
|
||||
return SkipConditionLiteral
|
||||
}
|
||||
|
||||
conds, ok := v.buildConditions(key, matchingFieldKeys(key, v.fieldKeys), operator, value)
|
||||
if !ok {
|
||||
return ErrorConditionLiteral
|
||||
@@ -780,7 +808,16 @@ func (v *filterExpressionVisitor) VisitValue(ctx *grammar.ValueContext) any {
|
||||
if ctx.QUOTED_TEXT() != nil {
|
||||
txt := ctx.QUOTED_TEXT().GetText()
|
||||
// trim quotes and return the value
|
||||
return trimQuotes(txt)
|
||||
value := trimQuotes(txt)
|
||||
// the string may have variables embedded in it (e.g. '$env-suffix')
|
||||
if strings.Contains(value, "$") {
|
||||
interpolated, hasAll := v.interpolateVariablesInString(value)
|
||||
if hasAll {
|
||||
return skipConditionValue{}
|
||||
}
|
||||
return interpolated
|
||||
}
|
||||
return value
|
||||
} else if ctx.NUMBER() != nil {
|
||||
number, err := strconv.ParseFloat(ctx.NUMBER().GetText(), 64)
|
||||
if err != nil {
|
||||
@@ -797,7 +834,16 @@ func (v *filterExpressionVisitor) VisitValue(ctx *grammar.ValueContext) any {
|
||||
// When the user writes an expression like `service.name=redis`
|
||||
// The `redis` part is a VALUE context but parsed as a KEY token
|
||||
// so we return the text as is
|
||||
return ctx.KEY().GetText()
|
||||
keyText := ctx.KEY().GetText()
|
||||
// an unquoted value may compose variables too (e.g. $environment-xyz)
|
||||
if strings.Contains(keyText, "$") {
|
||||
interpolated, hasAll := v.interpolateVariablesInString(keyText)
|
||||
if hasAll {
|
||||
return skipConditionValue{}
|
||||
}
|
||||
return interpolated
|
||||
}
|
||||
return keyText
|
||||
}
|
||||
|
||||
return ErrorConditionLiteral // Should not happen with valid input
|
||||
@@ -898,3 +944,26 @@ func matchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[st
|
||||
|
||||
return fieldKeysForName
|
||||
}
|
||||
|
||||
// skipConditionValue is the value-level counterpart of SkipConditionLiteral: VisitValue
|
||||
// returns it when a variable embedded in the value resolved to __all__, and the
|
||||
// comparison handling turns it into SkipConditionLiteral for the whole condition. A
|
||||
// dedicated type (rather than a sentinel string) so no user-supplied value can ever
|
||||
// collide with it.
|
||||
type skipConditionValue struct{}
|
||||
|
||||
// isSkipConditionValue reports whether a visited value says "drop the enclosing
|
||||
// condition" (an embedded variable resolved to __all__).
|
||||
func isSkipConditionValue(value any) bool {
|
||||
_, ok := value.(skipConditionValue)
|
||||
return ok
|
||||
}
|
||||
|
||||
// interpolateVariablesInString delegates to variables.Interpolate (embedded $variable
|
||||
// references, boundary-aware, __all__ detection), folding its warnings into the
|
||||
// visitor. See that function for the full contract.
|
||||
func (v *filterExpressionVisitor) interpolateVariablesInString(s string) (string, bool) {
|
||||
interpolated, hasAll, warnings := variables.Interpolate(s, v.variables)
|
||||
v.addWarnings(warnings, false)
|
||||
return interpolated, hasAll
|
||||
}
|
||||
|
||||
@@ -1552,11 +1552,34 @@ func TestVisitComparison_AllVariable(t *testing.T) {
|
||||
wantSB: "WHERE body_cond",
|
||||
},
|
||||
{
|
||||
// Equality does not trigger __all__ short-circuit; ConditionFor called normally.
|
||||
name: "equality with __all__ variable no shortcircuit",
|
||||
// __all__ with a single-value operator means "no filter", same as IN:
|
||||
// the user picked ALL, so the condition is dropped instead of matching
|
||||
// the literal string "__all__".
|
||||
name: "equality with __all__ variable skips condition",
|
||||
expr: "a = $service",
|
||||
wantRSB: "",
|
||||
wantSB: "WHERE a_cond",
|
||||
wantSB: "",
|
||||
},
|
||||
{
|
||||
// __all__ referenced from inside a composed value drops the condition too.
|
||||
name: "embedded __all__ variable skips condition",
|
||||
expr: "a = '$service-suffix'",
|
||||
wantRSB: "",
|
||||
wantSB: "",
|
||||
},
|
||||
{
|
||||
// Embedded __all__ inside an IN list member drops the condition.
|
||||
name: "embedded __all__ variable in IN list skips condition",
|
||||
expr: "a IN ('$service-suffix', 'other')",
|
||||
wantRSB: "",
|
||||
wantSB: "",
|
||||
},
|
||||
{
|
||||
// Embedded __all__ in one bound of BETWEEN drops the condition.
|
||||
name: "embedded __all__ variable in BETWEEN skips condition",
|
||||
expr: "a BETWEEN '$service-lo' AND 'hi'",
|
||||
wantRSB: "",
|
||||
wantSB: "",
|
||||
},
|
||||
{
|
||||
// __all__→TrueConditionLiteral stripped in AND; y_cond wrapped in paren; NOT wraps.
|
||||
@@ -1829,3 +1852,281 @@ func TestVisitComparison_SkippableLiteralValues(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestInterpolateVariablesInString tests the embedded variable interpolation
|
||||
// feature (GitHub issue #10008).
|
||||
func TestInterpolateVariablesInString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
variables map[string]qbtypes.VariableItem
|
||||
expected string
|
||||
expectAll bool
|
||||
wantWarnings int
|
||||
}{
|
||||
{
|
||||
name: "pure variable reference - not interpolated",
|
||||
input: "$service",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"service": {Value: "auth-service"},
|
||||
},
|
||||
expected: "$service", // handled by the standalone variable flow
|
||||
},
|
||||
{
|
||||
name: "variable composed with suffix",
|
||||
input: "$environment-xyz",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"environment": {Value: "prod"},
|
||||
},
|
||||
expected: "prod-xyz",
|
||||
},
|
||||
{
|
||||
name: "variable with prefix and suffix",
|
||||
input: "prefix-$var-suffix",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"var": {Value: "middle"},
|
||||
},
|
||||
expected: "prefix-middle-suffix",
|
||||
},
|
||||
{
|
||||
name: "multiple variables in one string",
|
||||
input: "$region-$env-cluster",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"region": {Value: "us-west"},
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
expected: "us-west-prod-cluster",
|
||||
},
|
||||
{
|
||||
name: "adjacent variables",
|
||||
input: "$env$region",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod-"},
|
||||
"region": {Value: "us"},
|
||||
},
|
||||
expected: "prod-us",
|
||||
},
|
||||
{
|
||||
name: "similar variable names - longer matches first",
|
||||
input: "$env-$environment",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "dev"},
|
||||
"environment": {Value: "production"},
|
||||
},
|
||||
expected: "dev-production",
|
||||
},
|
||||
{
|
||||
name: "unknown variable - preserved as-is",
|
||||
input: "$unknown-suffix",
|
||||
variables: map[string]qbtypes.VariableItem{},
|
||||
expected: "$unknown-suffix",
|
||||
},
|
||||
{
|
||||
name: "shorter variable does not match inside longer unknown reference",
|
||||
input: "$environment",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
expected: "$environment", // NOT "prodironment"
|
||||
},
|
||||
{
|
||||
name: "underscore extends the variable name",
|
||||
input: "$env_name",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
expected: "$env_name",
|
||||
},
|
||||
{
|
||||
name: "digit extends the variable name",
|
||||
input: "$env2",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
expected: "$env2",
|
||||
},
|
||||
{
|
||||
name: "dot is a boundary",
|
||||
input: "$env.internal",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
expected: "prod.internal",
|
||||
},
|
||||
{
|
||||
name: "LIKE pattern wildcards are boundaries",
|
||||
input: "%$env%",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
expected: "%prod%",
|
||||
},
|
||||
{
|
||||
name: "variable with underscore in name",
|
||||
input: "$my_var-test",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"my_var": {Value: "hello"},
|
||||
},
|
||||
expected: "hello-test",
|
||||
},
|
||||
{
|
||||
name: "map key with dollar prefix",
|
||||
input: "$env-xyz",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"$env": {Value: "prod"},
|
||||
},
|
||||
expected: "prod-xyz",
|
||||
},
|
||||
{
|
||||
name: "dollar with no variable after it",
|
||||
input: "cost is 100$",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
expected: "cost is 100$",
|
||||
},
|
||||
{
|
||||
name: "__all__ value reports hasAll",
|
||||
input: "$env-suffix",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "__all__",
|
||||
},
|
||||
},
|
||||
expected: "",
|
||||
expectAll: true,
|
||||
},
|
||||
{
|
||||
name: "multi-select takes first value with warning",
|
||||
input: "$env-suffix",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: []any{"prod", "staging", "dev"}},
|
||||
},
|
||||
expected: "prod-suffix",
|
||||
wantWarnings: 1,
|
||||
},
|
||||
{
|
||||
name: "numeric variable value",
|
||||
input: "code-$status",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"status": {Value: float64(200)},
|
||||
},
|
||||
expected: "code-200",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
visitor := &filterExpressionVisitor{
|
||||
variables: tt.variables,
|
||||
keysWithWarnings: make(map[string]bool),
|
||||
}
|
||||
result, hasAll := visitor.interpolateVariablesInString(tt.input)
|
||||
assert.Equal(t, tt.expectAll, hasAll)
|
||||
if !tt.expectAll {
|
||||
assert.Equal(t, tt.expected, result)
|
||||
}
|
||||
assert.Len(t, visitor.warnings, tt.wantWarnings)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// valueRecordingConditionBuilder records the value each ConditionFor call receives,
|
||||
// so tests can assert what the visitor actually hands to the condition builder.
|
||||
type valueRecordingConditionBuilder struct {
|
||||
values []any
|
||||
}
|
||||
|
||||
func (b *valueRecordingConditionBuilder) ConditionFor(
|
||||
_ context.Context,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
_ []*telemetrytypes.TelemetryFieldKey,
|
||||
_ qbtypes.FilterOperator,
|
||||
value any,
|
||||
_ *sqlbuilder.SelectBuilder,
|
||||
) ([]string, []string, error) {
|
||||
b.values = append(b.values, value)
|
||||
return []string{fmt.Sprintf("%s_cond", key.Name)}, nil, nil
|
||||
}
|
||||
|
||||
// TestInterpolatedValueReachesConditionBuilder pins the visitor → condition builder
|
||||
// handoff for embedded variables: the interpolated string (not the raw reference) is
|
||||
// the value passed to ConditionFor. The condition builder is signal-specific but this
|
||||
// seam is not, so it is covered here once instead of per telemetry package.
|
||||
func TestInterpolatedValueReachesConditionBuilder(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
expr string
|
||||
variables map[string]qbtypes.VariableItem
|
||||
wantValues []any
|
||||
}{
|
||||
{
|
||||
name: "composed value in quoted string",
|
||||
expr: "a = '$env-xyz'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
wantValues: []any{"prod-xyz"},
|
||||
},
|
||||
{
|
||||
name: "composed value in LIKE pattern",
|
||||
expr: "a LIKE '$env%'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
wantValues: []any{"prod%"},
|
||||
},
|
||||
{
|
||||
name: "pure reference goes through standalone substitution",
|
||||
expr: "a = $env",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
wantValues: []any{"prod"},
|
||||
},
|
||||
{
|
||||
name: "multi-select collapses to first value",
|
||||
expr: "a = '$env-xyz'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: []any{"prod", "staging"}},
|
||||
},
|
||||
wantValues: []any{"prod-xyz"},
|
||||
},
|
||||
{
|
||||
name: "composed members of an IN list",
|
||||
expr: "a IN ('$env-1', '$env-2')",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
wantValues: []any{[]any{"prod-1", "prod-2"}},
|
||||
},
|
||||
{
|
||||
name: "unknown variable passes through untouched",
|
||||
expr: "a = '$unknown-xyz'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {Value: "prod"},
|
||||
},
|
||||
wantValues: []any{"$unknown-xyz"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
recorder := &valueRecordingConditionBuilder{}
|
||||
opts := FilterExprVisitorOpts{
|
||||
Context: t.Context(),
|
||||
FieldKeys: visitTestKeys,
|
||||
ConditionBuilder: recorder,
|
||||
Variables: tt.variables,
|
||||
}
|
||||
|
||||
result, err := PrepareWhereClause(tt.expr, opts)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, result.IsEmpty())
|
||||
assert.Equal(t, tt.wantValues, recorder.values)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,6 +314,10 @@ func (VariableType) Enum() []any {
|
||||
}
|
||||
}
|
||||
|
||||
// AllVariableValue is the sentinel value a dynamic variable carries when "ALL"
|
||||
// is selected; conditions referencing such a variable are dropped entirely.
|
||||
const AllVariableValue = "__all__"
|
||||
|
||||
type VariableItem struct {
|
||||
Type VariableType `json:"type"`
|
||||
Value any `json:"value"`
|
||||
|
||||
134
pkg/variables/interpolate.go
Normal file
134
pkg/variables/interpolate.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package variables
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
)
|
||||
|
||||
// isVariableNameChar reports whether c can be part of a variable name.
|
||||
func isVariableNameChar(c byte) bool {
|
||||
return c == '_' || ('0' <= c && c <= '9') || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')
|
||||
}
|
||||
|
||||
// lookupVariable resolves a variable name against the variables map, tolerating a
|
||||
// `$` prefix on either the name or the map keys.
|
||||
func lookupVariable(name string, variables map[string]qbtypes.VariableItem) (qbtypes.VariableItem, bool) {
|
||||
if item, ok := variables[name]; ok {
|
||||
return item, true
|
||||
}
|
||||
if strings.HasPrefix(name, "$") {
|
||||
item, ok := variables[name[1:]]
|
||||
return item, ok
|
||||
}
|
||||
item, ok := variables["$"+name]
|
||||
return item, ok
|
||||
}
|
||||
|
||||
// Interpolate replaces $variable references embedded in s with their values, matching
|
||||
// against the names in the variables map (longest name first). A reference is only
|
||||
// replaced when the character following it cannot extend a variable name: with only
|
||||
// `env` defined, "$env-suffix" becomes "prod-suffix" while "$environment" stays
|
||||
// untouched instead of turning into "prodironment".
|
||||
// A string that is a single variable reference (e.g. "$env") is returned unchanged so
|
||||
// callers can process standalone references with their type intact (lists for IN,
|
||||
// numbers, __all__, etc.).
|
||||
// The returned bool is true when a referenced dynamic variable has the __all__ value,
|
||||
// meaning the enclosing condition must be dropped. Warnings (e.g. a multi-value
|
||||
// variable collapsing to its first value) are returned for the caller to surface.
|
||||
func Interpolate(s string, variables map[string]qbtypes.VariableItem) (string, bool, []string) {
|
||||
if len(variables) == 0 {
|
||||
return s, false, nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(s, "$") {
|
||||
if _, ok := lookupVariable(s, variables); ok {
|
||||
return s, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
names := make([]string, 0, len(variables))
|
||||
seen := make(map[string]bool, len(variables))
|
||||
for name := range variables {
|
||||
name = strings.TrimPrefix(name, "$")
|
||||
if name == "" || seen[name] {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
names = append(names, name)
|
||||
}
|
||||
// longest first so that $environment is not mistaken for $env plus a suffix
|
||||
sort.Slice(names, func(i, j int) bool { return len(names[i]) > len(names[j]) })
|
||||
|
||||
var warnings []string
|
||||
var sb strings.Builder
|
||||
i := 0
|
||||
for i < len(s) {
|
||||
if s[i] != '$' {
|
||||
sb.WriteByte(s[i])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
rest := s[i+1:]
|
||||
matched := false
|
||||
for _, name := range names {
|
||||
if !strings.HasPrefix(rest, name) {
|
||||
continue
|
||||
}
|
||||
// a name character right after the match means this occurrence references
|
||||
// a longer, unknown variable; leave it for a shorter-name check or as-is
|
||||
if len(rest) > len(name) && isVariableNameChar(rest[len(name)]) {
|
||||
continue
|
||||
}
|
||||
varItem, _ := lookupVariable(name, variables)
|
||||
if varItem.Type == qbtypes.DynamicVariableType {
|
||||
if allVal, ok := varItem.Value.(string); ok && allVal == qbtypes.AllVariableValue {
|
||||
return "", true, warnings
|
||||
}
|
||||
}
|
||||
sb.WriteString(formatValueForInterpolation(varItem.Value, name, &warnings))
|
||||
i += 1 + len(name)
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
if !matched {
|
||||
sb.WriteByte('$')
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String(), false, warnings
|
||||
}
|
||||
|
||||
// formatValueForInterpolation renders a variable value as a plain string for embedding
|
||||
// inside a larger value. Multi-value variables collapse to their first value with a
|
||||
// warning, since a pattern like "%$var%" can only hold one.
|
||||
func formatValueForInterpolation(value any, varName string, warnings *[]string) string {
|
||||
switch val := value.(type) {
|
||||
case string:
|
||||
return val
|
||||
case []string:
|
||||
if len(val) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(val) > 1 {
|
||||
*warnings = append(*warnings, fmt.Sprintf("variable `%s` has multiple values, using first value `%s` for string interpolation", varName, val[0]))
|
||||
}
|
||||
return val[0]
|
||||
case []any:
|
||||
if len(val) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(val) > 1 {
|
||||
*warnings = append(*warnings, fmt.Sprintf("variable `%s` has multiple values, using first value for string interpolation", varName))
|
||||
}
|
||||
return formatValueForInterpolation(val[0], varName, warnings)
|
||||
case bool:
|
||||
return strconv.FormatBool(val)
|
||||
default:
|
||||
return fmt.Sprintf("%v", val)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package variables
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -35,19 +36,22 @@ func (e *ErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol
|
||||
type variableReplacementVisitor struct {
|
||||
variables map[string]qbtypes.VariableItem
|
||||
errors []string
|
||||
warnings []string
|
||||
}
|
||||
|
||||
// specialSkipMarker is used to indicate that a condition should be removed.
|
||||
const specialSkipMarker = "__SKIP_CONDITION__"
|
||||
|
||||
// ReplaceVariablesInExpression takes a filter expression and returns it with variables replaced.
|
||||
func ReplaceVariablesInExpression(expression string, variables map[string]qbtypes.VariableItem) (string, error) {
|
||||
// ReplaceVariablesInExpression takes a filter expression and returns it with variables
|
||||
// replaced, along with any warnings generated during the replacement.
|
||||
func ReplaceVariablesInExpression(expression string, variables map[string]qbtypes.VariableItem) (string, []string, error) {
|
||||
input := antlr.NewInputStream(expression)
|
||||
lexer := grammar.NewFilterQueryLexer(input)
|
||||
|
||||
visitor := &variableReplacementVisitor{
|
||||
variables: variables,
|
||||
errors: []string{},
|
||||
warnings: []string{},
|
||||
}
|
||||
|
||||
lexerErrorListener := NewErrorListener()
|
||||
@@ -63,21 +67,21 @@ func ReplaceVariablesInExpression(expression string, variables map[string]qbtype
|
||||
tree := parser.Query()
|
||||
|
||||
if len(parserErrorListener.SyntaxErrors) > 0 {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "syntax errors in expression: %v", parserErrorListener.SyntaxErrors)
|
||||
return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "syntax errors in expression: %v", parserErrorListener.SyntaxErrors)
|
||||
}
|
||||
|
||||
result := visitor.Visit(tree).(string)
|
||||
|
||||
if len(visitor.errors) > 0 {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "errors processing expression: %v", visitor.errors)
|
||||
return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "errors processing expression: %v", visitor.errors)
|
||||
}
|
||||
|
||||
// If the entire expression should be skipped, return empty string
|
||||
if result == specialSkipMarker {
|
||||
return "", nil
|
||||
return "", visitor.warnings, nil
|
||||
}
|
||||
|
||||
return result, nil
|
||||
return result, visitor.warnings, nil
|
||||
}
|
||||
|
||||
// Visit dispatches to the specific visit method based on node type.
|
||||
@@ -469,7 +473,7 @@ func (v *variableReplacementVisitor) VisitValue(ctx *grammar.ValueContext) any {
|
||||
if ok {
|
||||
// Handle dynamic variable with __all__ value
|
||||
if varItem.Type == qbtypes.DynamicVariableType {
|
||||
if allVal, ok := varItem.Value.(string); ok && allVal == "__all__" {
|
||||
if allVal, ok := varItem.Value.(string); ok && allVal == qbtypes.AllVariableValue {
|
||||
// Return special marker to indicate this condition should be removed
|
||||
return specialSkipMarker
|
||||
}
|
||||
@@ -480,6 +484,18 @@ func (v *variableReplacementVisitor) VisitValue(ctx *grammar.ValueContext) any {
|
||||
}
|
||||
}
|
||||
|
||||
// Not a standalone variable reference; the value may still have variables
|
||||
// embedded in it (e.g. '$env-suffix' or 'prefix-$var-suffix')
|
||||
if strings.Contains(originalValue, "$") {
|
||||
interpolated, hasAll := v.interpolateVariablesInString(originalValue)
|
||||
if hasAll {
|
||||
return specialSkipMarker
|
||||
}
|
||||
if interpolated != originalValue {
|
||||
return v.formatVariableValue(interpolated)
|
||||
}
|
||||
}
|
||||
|
||||
// Return original value if not a variable or variable not found
|
||||
// If it was quoted text and not a variable, return with quotes
|
||||
if ctx.QUOTED_TEXT() != nil && !strings.HasPrefix(originalValue, "$") {
|
||||
@@ -505,7 +521,7 @@ func (v *variableReplacementVisitor) VisitKey(ctx *grammar.KeyContext) any {
|
||||
if ok {
|
||||
// Handle dynamic variable with __all__ value
|
||||
if varItem.Type == qbtypes.DynamicVariableType {
|
||||
if allVal, ok := varItem.Value.(string); ok && allVal == "__all__" {
|
||||
if allVal, ok := varItem.Value.(string); ok && allVal == qbtypes.AllVariableValue {
|
||||
return specialSkipMarker
|
||||
}
|
||||
}
|
||||
@@ -517,6 +533,25 @@ func (v *variableReplacementVisitor) VisitKey(ctx *grammar.KeyContext) any {
|
||||
return keyText
|
||||
}
|
||||
|
||||
// addWarning appends w unless it is already recorded; comparisons visit their values
|
||||
// twice (once to check for __all__, once to rebuild), which would duplicate warnings.
|
||||
func (v *variableReplacementVisitor) addWarning(w string) {
|
||||
if slices.Contains(v.warnings, w) {
|
||||
return
|
||||
}
|
||||
v.warnings = append(v.warnings, w)
|
||||
}
|
||||
|
||||
// interpolateVariablesInString delegates to Interpolate, folding its warnings into
|
||||
// the visitor.
|
||||
func (v *variableReplacementVisitor) interpolateVariablesInString(s string) (string, bool) {
|
||||
interpolated, hasAll, warnings := Interpolate(s, v.variables)
|
||||
for _, w := range warnings {
|
||||
v.addWarning(w)
|
||||
}
|
||||
return interpolated, hasAll
|
||||
}
|
||||
|
||||
// formatVariableValue formats a variable value for inclusion in the expression.
|
||||
func (v *variableReplacementVisitor) formatVariableValue(value any) string {
|
||||
switch val := value.(type) {
|
||||
|
||||
@@ -421,11 +421,154 @@ func TestReplaceVariablesInExpression(t *testing.T) {
|
||||
},
|
||||
expected: "message NOT CONTAINS 'debug'",
|
||||
},
|
||||
{
|
||||
name: "variable composed with suffix in value",
|
||||
expression: "cluster_name = '$environment-xyz'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"environment": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "prod",
|
||||
},
|
||||
},
|
||||
expected: "cluster_name = 'prod-xyz'",
|
||||
},
|
||||
{
|
||||
name: "variable composed with suffix without quotes",
|
||||
expression: "cluster_name = $environment-xyz",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"environment": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "staging",
|
||||
},
|
||||
},
|
||||
expected: "cluster_name = 'staging-xyz'",
|
||||
},
|
||||
{
|
||||
name: "variable in LIKE pattern with suffix",
|
||||
expression: "service.name LIKE '$env%'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "prod",
|
||||
},
|
||||
},
|
||||
expected: "service.name LIKE 'prod%'",
|
||||
},
|
||||
{
|
||||
name: "variable composed with prefix and suffix",
|
||||
expression: "label = 'prefix-$var-suffix'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"var": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "middle",
|
||||
},
|
||||
},
|
||||
expected: "label = 'prefix-middle-suffix'",
|
||||
},
|
||||
{
|
||||
name: "multiple variables in one string",
|
||||
expression: "path = '$region-$env-cluster'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"region": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "us-west",
|
||||
},
|
||||
"env": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "prod",
|
||||
},
|
||||
},
|
||||
expected: "path = 'us-west-prod-cluster'",
|
||||
},
|
||||
{
|
||||
name: "embedded variable with __all__ value skips condition",
|
||||
expression: "cluster_name = '$environment-xyz'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"environment": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "__all__",
|
||||
},
|
||||
},
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "variable with underscore composed with suffix",
|
||||
expression: "name = '$my_var-test'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"my_var": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "hello",
|
||||
},
|
||||
},
|
||||
expected: "name = 'hello-test'",
|
||||
},
|
||||
{
|
||||
name: "similar variable names - longer matches first",
|
||||
expression: "name = '$env-$environment'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "dev",
|
||||
},
|
||||
"environment": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "production",
|
||||
},
|
||||
},
|
||||
expected: "name = 'dev-production'",
|
||||
},
|
||||
{
|
||||
// Unresolved $-prefixed values come back unquoted so they stay
|
||||
// variable-shaped for downstream resolution.
|
||||
name: "shorter variable does not match inside longer unknown reference",
|
||||
expression: "name = '$environment'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "prod",
|
||||
},
|
||||
},
|
||||
// `env` must not corrupt `$environment` into 'prodironment'
|
||||
expected: "name = $environment",
|
||||
},
|
||||
{
|
||||
name: "unknown embedded variable left untouched",
|
||||
expression: "name = '$unknown-suffix'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "prod",
|
||||
},
|
||||
},
|
||||
expected: "name = $unknown-suffix",
|
||||
},
|
||||
{
|
||||
name: "embedded variable in unquoted value with prefix",
|
||||
expression: "cluster_name = abc-$env",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {
|
||||
Type: qbtypes.DynamicVariableType,
|
||||
Value: "prod",
|
||||
},
|
||||
},
|
||||
expected: "cluster_name = 'abc-prod'",
|
||||
},
|
||||
{
|
||||
name: "multi-select variable in composed string takes first value",
|
||||
expression: "cluster_name = '$env-xyz'",
|
||||
variables: map[string]qbtypes.VariableItem{
|
||||
"env": {
|
||||
Type: qbtypes.CustomVariableType,
|
||||
Value: []any{"prod", "staging"},
|
||||
},
|
||||
},
|
||||
expected: "cluster_name = 'prod-xyz'",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := ReplaceVariablesInExpression(tt.expression, tt.variables)
|
||||
result, _, err := ReplaceVariablesInExpression(tt.expression, tt.variables)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
@@ -438,6 +581,24 @@ func TestReplaceVariablesInExpression(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplaceVariablesInExpressionWarnings ensures multi-value variables used inside a
|
||||
// composed string produce exactly one warning, even though comparisons visit their
|
||||
// values twice (once for the __all__ check, once to rebuild the expression).
|
||||
func TestReplaceVariablesInExpressionWarnings(t *testing.T) {
|
||||
variables := map[string]qbtypes.VariableItem{
|
||||
"env": {
|
||||
Type: qbtypes.CustomVariableType,
|
||||
Value: []any{"prod", "staging"},
|
||||
},
|
||||
}
|
||||
|
||||
result, warnings, err := ReplaceVariablesInExpression("cluster_name = '$env-xyz'", variables)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "cluster_name = 'prod-xyz'", result)
|
||||
assert.Len(t, warnings, 1)
|
||||
assert.Contains(t, warnings[0], "`env` has multiple values")
|
||||
}
|
||||
|
||||
func TestFormatVariableValue(t *testing.T) {
|
||||
visitor := &variableReplacementVisitor{}
|
||||
|
||||
|
||||
25
tests/fixtures/querier.py
vendored
25
tests/fixtures/querier.py
vendored
@@ -972,3 +972,28 @@ def make_scalar_query_request(
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def make_substitute_vars_request(
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
start_ms: int,
|
||||
end_ms: int,
|
||||
queries: list[dict],
|
||||
variables: dict,
|
||||
timeout: int = QUERY_TIMEOUT,
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/substitute_vars"),
|
||||
timeout=timeout,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {"queries": queries},
|
||||
"variables": variables,
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
126
tests/integration/tests/queriervariables/01_substitute_vars.py
Normal file
126
tests/integration/tests/queriervariables/01_substitute_vars.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Tests for the /api/v5/substitute_vars endpoint (GitHub issue #10008).
|
||||
|
||||
Variables are supported both as standalone RHS values (field = $var, field IN $var)
|
||||
and composed inside values ('$env-suffix', LIKE '$env%').
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import Aggregation, BuilderQuery, make_substitute_vars_request
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression,variables,expected",
|
||||
[
|
||||
pytest.param(
|
||||
"service.name = $service",
|
||||
{"service": {"type": "query", "value": "auth-service"}},
|
||||
"service.name = 'auth-service'",
|
||||
id="standalone_variable",
|
||||
),
|
||||
pytest.param(
|
||||
"cluster_name = '$environment-xyz'",
|
||||
{"environment": {"type": "custom", "value": "prod"}},
|
||||
"cluster_name = 'prod-xyz'",
|
||||
id="variable_in_string",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name = $service AND env = $environment",
|
||||
{
|
||||
"service": {"type": "text", "value": "auth-service"},
|
||||
"environment": {"type": "query", "value": "production"},
|
||||
},
|
||||
"service.name = 'auth-service' AND env = 'production'",
|
||||
id="multiple_variables",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name IN $services",
|
||||
{"services": {"type": "query", "value": ["auth-service", "api-service"]}},
|
||||
"service.name IN ['auth-service', 'api-service']",
|
||||
id="array_variable",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name LIKE '$env%'",
|
||||
{"env": {"type": "text", "value": "prod"}},
|
||||
"service.name LIKE 'prod%'",
|
||||
id="like_pattern",
|
||||
),
|
||||
pytest.param(
|
||||
"cluster_name = $environment-xyz",
|
||||
{"environment": {"type": "dynamic", "value": "staging"}},
|
||||
"cluster_name = 'staging-xyz'",
|
||||
id="variable_without_quotes",
|
||||
),
|
||||
# `env` must not match inside the longer, undefined `$environment` reference.
|
||||
# Unresolved $-prefixed values come back unquoted so they stay variable-shaped.
|
||||
pytest.param(
|
||||
"cluster_name = '$environment-xyz'",
|
||||
{"env": {"type": "custom", "value": "prod"}},
|
||||
"cluster_name = $environment-xyz",
|
||||
id="unknown_embedded_variable_untouched",
|
||||
),
|
||||
# expression should be empty when __all__ is used
|
||||
pytest.param(
|
||||
"service.name = $service",
|
||||
{"service": {"type": "dynamic", "value": "__all__"}},
|
||||
"",
|
||||
id="all_value_standalone",
|
||||
),
|
||||
pytest.param(
|
||||
"cluster_name = '$environment-xyz'",
|
||||
{"environment": {"type": "dynamic", "value": "__all__"}},
|
||||
"",
|
||||
id="all_value_in_composed_string",
|
||||
),
|
||||
# only the env condition should remain
|
||||
pytest.param(
|
||||
"service.name = $service AND env = $env",
|
||||
{
|
||||
"service": {"type": "dynamic", "value": "__all__"},
|
||||
"env": {"type": "dynamic", "value": "production"},
|
||||
},
|
||||
"env = 'production'",
|
||||
id="all_value_partial",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_substitute_vars(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
expression: str,
|
||||
variables: dict,
|
||||
expected: str,
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
response = make_substitute_vars_request(
|
||||
signoz,
|
||||
token,
|
||||
int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[
|
||||
BuilderQuery(
|
||||
signal="logs",
|
||||
aggregations=[Aggregation("count()")],
|
||||
step_interval=60,
|
||||
filter_expression=expression,
|
||||
).to_dict()
|
||||
],
|
||||
variables,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
queries = response.json()["data"]["compositeQuery"]["queries"]
|
||||
assert len(queries) == 1
|
||||
assert queries[0]["spec"]["filter"]["expression"] == expected
|
||||
274
tests/integration/tests/queriervariables/02_variable_in_query.py
Normal file
274
tests/integration/tests/queriervariables/02_variable_in_query.py
Normal file
@@ -0,0 +1,274 @@
|
||||
"""
|
||||
End-to-end tests for variables in /api/v5/query_range filter expressions
|
||||
(GitHub issue #10008): standalone ($var, IN $var), composed inside values
|
||||
('$env-suffix', LIKE '$env%'), and the special __all__ value.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
BuilderQuery,
|
||||
OrderBy,
|
||||
TelemetryFieldKey,
|
||||
get_column_data_from_response,
|
||||
get_scalar_table_data,
|
||||
make_query_request,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression,variables,expected_logs",
|
||||
[
|
||||
pytest.param(
|
||||
"service.name = $service",
|
||||
{"service": {"type": "query", "value": "auth-service"}},
|
||||
{"auth-prod-log"},
|
||||
id="standalone_variable",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name IN $services",
|
||||
{"services": {"type": "custom", "value": ["auth-service", "api-service"]}},
|
||||
{"auth-prod-log", "api-prod-log"},
|
||||
id="array_variable",
|
||||
),
|
||||
pytest.param(
|
||||
"cluster_name = '$environment-xyz'",
|
||||
{"environment": {"type": "dynamic", "value": "prod"}},
|
||||
{"auth-prod-log", "api-prod-log"},
|
||||
id="variable_composed_in_string",
|
||||
),
|
||||
pytest.param(
|
||||
"cluster_name LIKE '$env%'",
|
||||
{"env": {"type": "text", "value": "prod"}},
|
||||
{"auth-prod-log", "api-prod-log"},
|
||||
id="variable_in_like_pattern",
|
||||
),
|
||||
pytest.param(
|
||||
"path = '$region-$env-cluster'",
|
||||
{
|
||||
"region": {"type": "query", "value": "us-west"},
|
||||
"env": {"type": "custom", "value": "prod"},
|
||||
},
|
||||
{"auth-prod-log"},
|
||||
id="multiple_variables_in_string",
|
||||
),
|
||||
pytest.param(
|
||||
"label = 'prefix-$var-suffix'",
|
||||
{"var": {"type": "text", "value": "middle"}},
|
||||
{"auth-prod-log"},
|
||||
id="variable_with_prefix_and_suffix",
|
||||
),
|
||||
pytest.param(
|
||||
"cluster_name = $environment-xyz",
|
||||
{"environment": {"type": "custom", "value": "staging"}},
|
||||
{"web-staging-log"},
|
||||
id="variable_without_quotes",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name = $service",
|
||||
{"service": {"type": "query", "value": "auth-service"}},
|
||||
{"auth-prod-log"},
|
||||
id="query_variable_type",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name = $service",
|
||||
{"service": {"type": "custom", "value": "auth-service"}},
|
||||
{"auth-prod-log"},
|
||||
id="custom_variable_type",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name = $service",
|
||||
{"service": {"type": "text", "value": "auth-service"}},
|
||||
{"auth-prod-log"},
|
||||
id="text_variable_type",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name = $service",
|
||||
{"service": {"type": "dynamic", "value": "auth-service"}},
|
||||
{"auth-prod-log"},
|
||||
id="dynamic_variable_type",
|
||||
),
|
||||
# `__all__` removes the filter condition entirely → all logs match
|
||||
pytest.param(
|
||||
"service.name = $service",
|
||||
{"service": {"type": "dynamic", "value": "__all__"}},
|
||||
{"auth-prod-log", "api-prod-log", "web-staging-log"},
|
||||
id="all_value_standalone",
|
||||
),
|
||||
pytest.param(
|
||||
"cluster_name = '$environment-xyz'",
|
||||
{"environment": {"type": "dynamic", "value": "__all__"}},
|
||||
{"auth-prod-log", "api-prod-log", "web-staging-log"},
|
||||
id="all_value_in_composed_string",
|
||||
),
|
||||
pytest.param(
|
||||
"service.name IN $services",
|
||||
{"services": {"type": "dynamic", "value": "__all__"}},
|
||||
{"auth-prod-log", "api-prod-log", "web-staging-log"},
|
||||
id="all_value_in_array",
|
||||
),
|
||||
# `__all__` for service drops only that condition; env filter still applies
|
||||
pytest.param(
|
||||
"service.name = $service AND env = $env",
|
||||
{
|
||||
"service": {"type": "dynamic", "value": "__all__"},
|
||||
"env": {"type": "dynamic", "value": "production"},
|
||||
},
|
||||
{"auth-prod-log", "api-prod-log"},
|
||||
id="all_value_partial_filter",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_variable_in_query(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
variables: dict,
|
||||
expected_logs: set[str],
|
||||
) -> None:
|
||||
"""
|
||||
Runs a raw logs query with the filter expression and variables, then checks the
|
||||
set of matching log bodies.
|
||||
|
||||
Canonical dataset:
|
||||
auth-prod-log: service.name=auth-service, env=production, cluster_name=prod-xyz,
|
||||
path=us-west-prod-cluster, label=prefix-middle-suffix
|
||||
api-prod-log: service.name=api-service, env=production, cluster_name=prod-xyz,
|
||||
path=us-east-prod-cluster, label=prefix-other-suffix
|
||||
web-staging-log: service.name=web-service, env=staging, cluster_name=staging-xyz,
|
||||
path=eu-west-staging-cluster
|
||||
"""
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
body="auth-prod-log",
|
||||
resources={"service.name": "auth-service"},
|
||||
attributes={
|
||||
"env": "production",
|
||||
"cluster_name": "prod-xyz",
|
||||
"path": "us-west-prod-cluster",
|
||||
"label": "prefix-middle-suffix",
|
||||
},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
body="api-prod-log",
|
||||
resources={"service.name": "api-service"},
|
||||
attributes={
|
||||
"env": "production",
|
||||
"cluster_name": "prod-xyz",
|
||||
"path": "us-east-prod-cluster",
|
||||
"label": "prefix-other-suffix",
|
||||
},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
body="web-staging-log",
|
||||
resources={"service.name": "web-service"},
|
||||
attributes={
|
||||
"env": "staging",
|
||||
"cluster_name": "staging-xyz",
|
||||
"path": "eu-west-staging-cluster",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
BuilderQuery(
|
||||
signal="logs",
|
||||
limit=100,
|
||||
filter_expression=expression,
|
||||
order=[
|
||||
OrderBy(key=TelemetryFieldKey(name="timestamp"), direction="desc"),
|
||||
OrderBy(key=TelemetryFieldKey(name="id"), direction="desc"),
|
||||
],
|
||||
).to_dict()
|
||||
],
|
||||
variables=variables,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
assert set(get_column_data_from_response(response.json(), "body")) == expected_logs
|
||||
|
||||
|
||||
def test_variable_in_query_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
A composed variable in the filter works together with groupBy: count logs
|
||||
with cluster_name = '$env-xyz' grouped by service.name.
|
||||
"""
|
||||
now = datetime.now(tz=UTC)
|
||||
logs: list[Logs] = []
|
||||
for i in range(3):
|
||||
for service, cluster in [
|
||||
("auth-service", "prod-xyz"),
|
||||
("api-service", "prod-xyz"),
|
||||
("web-service", "staging-xyz"),
|
||||
]:
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
body=f"{service} log {i}",
|
||||
resources={"service.name": service},
|
||||
attributes={"cluster_name": cluster},
|
||||
)
|
||||
)
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
# BuilderQuery has no groupBy field, so the spec is built inline
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="scalar",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"filter": {"expression": "cluster_name = '$env-xyz'"},
|
||||
"groupBy": [{"name": "service.name", "fieldContext": "resource"}],
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
variables={"env": {"type": "query", "value": "prod"}},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
rows = get_scalar_table_data(response.json())
|
||||
assert sorted(rows) == [["api-service", 3], ["auth-service", 3]]
|
||||
Reference in New Issue
Block a user