Compare commits

...

12 Commits

Author SHA1 Message Date
srikanthccv
349a70156e chore: remove */filter_expr_embedded_vars_test.go 2026-07-14 19:36:47 +05:30
srikanthccv
4decb41df8 chore: move to pkg/variables 2026-07-14 16:52:51 +05:30
srikanthccv
0aa17540a7 chore: merge main and fix embedded variable interpolation
Merge brings the feature onto the refactored where-clause visitor
(Skip/ErrorConditionLiteral sentinels, single-key VisitKey,
buildConditions) and the reorganized integration test tree.

Fixes folded into the merge:
- word-boundary aware interpolation: with only `env` defined,
  "$environment" is left untouched instead of becoming "prodironment";
  single-pass scan keeps replacement deterministic
- __all__ embedded in IN list members, BETWEEN bounds and function
  params now skips the condition instead of leaking a marker into SQL
- value-level skip marker is a private type so no user-supplied string
  can collide with it
- single-select `=` with __all__ skips the condition like IN does
- duplicated multi-value warnings are deduped (values are visited
  twice while checking for __all__)
- `__all__` literal replaced with qbtypes.AllVariableValue constant
- unquoted composed values (e.g. abc-$env) interpolate too, not just
  $-prefixed ones

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-14 16:20:02 +05:30
Srikanth Chekuri
585caa84fe Merge branch 'main' into issue-10008 2026-01-23 20:11:48 +05:30
srikanthccv
814fd40a1d chore: update tests 2026-01-23 05:43:26 +05:30
srikanthccv
0978bdfa7f chore: fix warn 2026-01-23 04:33:07 +05:30
srikanthccv
4f99261743 chore: resolve conflicts 2026-01-23 04:25:39 +05:30
Srikanth Chekuri
836988273f Merge branch 'main' into issue-10008 2026-01-20 16:48:54 +05:30
srikanthccv
a622d65226 chore: lint 2026-01-20 06:30:00 +05:30
srikanthccv
2410e3d411 chore: add integration tests 2026-01-20 06:14:13 +05:30
Srikanth Chekuri
654e2e4b7e Merge branch 'main' into issue-10008 2026-01-20 05:02:15 +05:30
srikanthccv
03ad7a85fa feat(querybuilder): support using variables inside value
prior to this, variables work only as standalone RHS values (e.g. field = $var / field IN $var). this change adds support to use variables inside a value for pattern matching or substitution ex "$env-suffix" or LIKE "%$pattern-var-name%"
2026-01-19 01:41:42 +05:30
11 changed files with 1158 additions and 18 deletions

View File

@@ -53,6 +53,7 @@ jobs:
- queriermetrics
- querierscalar
- queriercommon
- queriervariables
- rawexportdata
- role
- rootuser

View File

@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"strconv"
@@ -246,28 +247,37 @@ func (handler *handler) ReplaceVariables(rw http.ResponseWriter, req *http.Reque
switch spec := item.Spec.(type) {
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
if spec.Filter != nil && spec.Filter.Expression != "" {
replaced, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
replaced, warnings, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
if err != nil {
errs = append(errs, err)
}
if len(warnings) > 0 {
handler.set.Logger.WarnContext(req.Context(), "variable replace warnings", slog.Any("warnings", warnings))
}
spec.Filter.Expression = replaced
}
queryRangeRequest.CompositeQuery.Queries[idx].Spec = spec
case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
if spec.Filter != nil && spec.Filter.Expression != "" {
replaced, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
replaced, warnings, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
if err != nil {
errs = append(errs, err)
}
if len(warnings) > 0 {
handler.set.Logger.WarnContext(req.Context(), "variable replace warnings", slog.Any("warnings", warnings))
}
spec.Filter.Expression = replaced
}
queryRangeRequest.CompositeQuery.Queries[idx].Spec = spec
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
if spec.Filter != nil && spec.Filter.Expression != "" {
replaced, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
replaced, warnings, err := variables.ReplaceVariablesInExpression(spec.Filter.Expression, queryRangeRequest.Variables)
if err != nil {
errs = append(errs, err)
}
if len(warnings) > 0 {
handler.set.Logger.WarnContext(req.Context(), "variable replace warnings", slog.Any("warnings", warnings))
}
spec.Filter.Expression = replaced
}
queryRangeRequest.CompositeQuery.Queries[idx].Spec = spec

View File

@@ -12,6 +12,7 @@ import (
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/variables"
"github.com/antlr4-go/antlr/v4"
sqlbuilder "github.com/huandu/go-sqlbuilder"
@@ -428,6 +429,11 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
values = []any{ret}
}
// a value with an embedded variable resolved to __all__ makes the whole condition moot
if slices.ContainsFunc(values, isSkipConditionValue) {
return SkipConditionLiteral
}
if len(values) == 1 {
if var_, ok := values[0].(string); ok {
// check if this is a variables
@@ -443,7 +449,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
// we have a variable, now check for dynamic variable
if varItem.Type == qbtypes.DynamicVariableType {
// check if it is special value to skip entire filter, if so skip it
if all_, ok := varItem.Value.(string); ok && all_ == "__all__" {
if all_, ok := varItem.Value.(string); ok && all_ == qbtypes.AllVariableValue {
return SkipConditionLiteral
}
}
@@ -498,6 +504,11 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
value1 := v.Visit(values[0])
value2 := v.Visit(values[1])
// a bound with an embedded variable resolved to __all__ makes the whole condition moot
if isSkipConditionValue(value1) || isSkipConditionValue(value2) {
return SkipConditionLiteral
}
switch value1.(type) {
case float64:
if _, ok := value2.(float64); !ok {
@@ -535,6 +546,11 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
if len(values) > 0 {
value := v.Visit(values[0])
// the value had an embedded variable resolved to __all__; drop the condition
if isSkipConditionValue(value) {
return SkipConditionLiteral
}
if var_, ok := value.(string); ok {
// check if this is a variables
var ok bool
@@ -546,6 +562,13 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
}
if ok {
if varItem.Type == qbtypes.DynamicVariableType {
// __all__ used with a single-value operator: there is nothing to
// filter on, drop the condition like the IN clause handling does
if all_, ok := varItem.Value.(string); ok && all_ == qbtypes.AllVariableValue {
return SkipConditionLiteral
}
}
switch varValues := varItem.Value.(type) {
case []any:
if len(varValues) == 0 {
@@ -731,6 +754,11 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
value := params[1:]
// a param with an embedded variable resolved to __all__ makes the whole condition moot
if slices.ContainsFunc(value, isSkipConditionValue) {
return SkipConditionLiteral
}
conds, ok := v.buildConditions(key, matchingFieldKeys(key, v.fieldKeys), operator, value)
if !ok {
return ErrorConditionLiteral
@@ -780,7 +808,16 @@ func (v *filterExpressionVisitor) VisitValue(ctx *grammar.ValueContext) any {
if ctx.QUOTED_TEXT() != nil {
txt := ctx.QUOTED_TEXT().GetText()
// trim quotes and return the value
return trimQuotes(txt)
value := trimQuotes(txt)
// the string may have variables embedded in it (e.g. '$env-suffix')
if strings.Contains(value, "$") {
interpolated, hasAll := v.interpolateVariablesInString(value)
if hasAll {
return skipConditionValue{}
}
return interpolated
}
return value
} else if ctx.NUMBER() != nil {
number, err := strconv.ParseFloat(ctx.NUMBER().GetText(), 64)
if err != nil {
@@ -797,7 +834,16 @@ func (v *filterExpressionVisitor) VisitValue(ctx *grammar.ValueContext) any {
// When the user writes an expression like `service.name=redis`
// The `redis` part is a VALUE context but parsed as a KEY token
// so we return the text as is
return ctx.KEY().GetText()
keyText := ctx.KEY().GetText()
// an unquoted value may compose variables too (e.g. $environment-xyz)
if strings.Contains(keyText, "$") {
interpolated, hasAll := v.interpolateVariablesInString(keyText)
if hasAll {
return skipConditionValue{}
}
return interpolated
}
return keyText
}
return ErrorConditionLiteral // Should not happen with valid input
@@ -898,3 +944,26 @@ func matchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[st
return fieldKeysForName
}
// skipConditionValue is the value-level counterpart of SkipConditionLiteral: VisitValue
// returns it when a variable embedded in the value resolved to __all__, and the
// comparison handling turns it into SkipConditionLiteral for the whole condition. A
// dedicated type (rather than a sentinel string) so no user-supplied value can ever
// collide with it.
type skipConditionValue struct{}
// isSkipConditionValue reports whether a visited value says "drop the enclosing
// condition" (an embedded variable resolved to __all__).
func isSkipConditionValue(value any) bool {
_, ok := value.(skipConditionValue)
return ok
}
// interpolateVariablesInString delegates to variables.Interpolate (embedded $variable
// references, boundary-aware, __all__ detection), folding its warnings into the
// visitor. See that function for the full contract.
func (v *filterExpressionVisitor) interpolateVariablesInString(s string) (string, bool) {
interpolated, hasAll, warnings := variables.Interpolate(s, v.variables)
v.addWarnings(warnings, false)
return interpolated, hasAll
}

View File

@@ -1552,11 +1552,34 @@ func TestVisitComparison_AllVariable(t *testing.T) {
wantSB: "WHERE body_cond",
},
{
// Equality does not trigger __all__ short-circuit; ConditionFor called normally.
name: "equality with __all__ variable no shortcircuit",
// __all__ with a single-value operator means "no filter", same as IN:
// the user picked ALL, so the condition is dropped instead of matching
// the literal string "__all__".
name: "equality with __all__ variable skips condition",
expr: "a = $service",
wantRSB: "",
wantSB: "WHERE a_cond",
wantSB: "",
},
{
// __all__ referenced from inside a composed value drops the condition too.
name: "embedded __all__ variable skips condition",
expr: "a = '$service-suffix'",
wantRSB: "",
wantSB: "",
},
{
// Embedded __all__ inside an IN list member drops the condition.
name: "embedded __all__ variable in IN list skips condition",
expr: "a IN ('$service-suffix', 'other')",
wantRSB: "",
wantSB: "",
},
{
// Embedded __all__ in one bound of BETWEEN drops the condition.
name: "embedded __all__ variable in BETWEEN skips condition",
expr: "a BETWEEN '$service-lo' AND 'hi'",
wantRSB: "",
wantSB: "",
},
{
// __all__→TrueConditionLiteral stripped in AND; y_cond wrapped in paren; NOT wraps.
@@ -1829,3 +1852,281 @@ func TestVisitComparison_SkippableLiteralValues(t *testing.T) {
})
}
}
// TestInterpolateVariablesInString tests the embedded variable interpolation
// feature (GitHub issue #10008).
func TestInterpolateVariablesInString(t *testing.T) {
tests := []struct {
name string
input string
variables map[string]qbtypes.VariableItem
expected string
expectAll bool
wantWarnings int
}{
{
name: "pure variable reference - not interpolated",
input: "$service",
variables: map[string]qbtypes.VariableItem{
"service": {Value: "auth-service"},
},
expected: "$service", // handled by the standalone variable flow
},
{
name: "variable composed with suffix",
input: "$environment-xyz",
variables: map[string]qbtypes.VariableItem{
"environment": {Value: "prod"},
},
expected: "prod-xyz",
},
{
name: "variable with prefix and suffix",
input: "prefix-$var-suffix",
variables: map[string]qbtypes.VariableItem{
"var": {Value: "middle"},
},
expected: "prefix-middle-suffix",
},
{
name: "multiple variables in one string",
input: "$region-$env-cluster",
variables: map[string]qbtypes.VariableItem{
"region": {Value: "us-west"},
"env": {Value: "prod"},
},
expected: "us-west-prod-cluster",
},
{
name: "adjacent variables",
input: "$env$region",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod-"},
"region": {Value: "us"},
},
expected: "prod-us",
},
{
name: "similar variable names - longer matches first",
input: "$env-$environment",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "dev"},
"environment": {Value: "production"},
},
expected: "dev-production",
},
{
name: "unknown variable - preserved as-is",
input: "$unknown-suffix",
variables: map[string]qbtypes.VariableItem{},
expected: "$unknown-suffix",
},
{
name: "shorter variable does not match inside longer unknown reference",
input: "$environment",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
expected: "$environment", // NOT "prodironment"
},
{
name: "underscore extends the variable name",
input: "$env_name",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
expected: "$env_name",
},
{
name: "digit extends the variable name",
input: "$env2",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
expected: "$env2",
},
{
name: "dot is a boundary",
input: "$env.internal",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
expected: "prod.internal",
},
{
name: "LIKE pattern wildcards are boundaries",
input: "%$env%",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
expected: "%prod%",
},
{
name: "variable with underscore in name",
input: "$my_var-test",
variables: map[string]qbtypes.VariableItem{
"my_var": {Value: "hello"},
},
expected: "hello-test",
},
{
name: "map key with dollar prefix",
input: "$env-xyz",
variables: map[string]qbtypes.VariableItem{
"$env": {Value: "prod"},
},
expected: "prod-xyz",
},
{
name: "dollar with no variable after it",
input: "cost is 100$",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
expected: "cost is 100$",
},
{
name: "__all__ value reports hasAll",
input: "$env-suffix",
variables: map[string]qbtypes.VariableItem{
"env": {
Type: qbtypes.DynamicVariableType,
Value: "__all__",
},
},
expected: "",
expectAll: true,
},
{
name: "multi-select takes first value with warning",
input: "$env-suffix",
variables: map[string]qbtypes.VariableItem{
"env": {Value: []any{"prod", "staging", "dev"}},
},
expected: "prod-suffix",
wantWarnings: 1,
},
{
name: "numeric variable value",
input: "code-$status",
variables: map[string]qbtypes.VariableItem{
"status": {Value: float64(200)},
},
expected: "code-200",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
visitor := &filterExpressionVisitor{
variables: tt.variables,
keysWithWarnings: make(map[string]bool),
}
result, hasAll := visitor.interpolateVariablesInString(tt.input)
assert.Equal(t, tt.expectAll, hasAll)
if !tt.expectAll {
assert.Equal(t, tt.expected, result)
}
assert.Len(t, visitor.warnings, tt.wantWarnings)
})
}
}
// valueRecordingConditionBuilder records the value each ConditionFor call receives,
// so tests can assert what the visitor actually hands to the condition builder.
type valueRecordingConditionBuilder struct {
values []any
}
func (b *valueRecordingConditionBuilder) ConditionFor(
_ context.Context,
_ uint64,
_ uint64,
key *telemetrytypes.TelemetryFieldKey,
_ []*telemetrytypes.TelemetryFieldKey,
_ qbtypes.FilterOperator,
value any,
_ *sqlbuilder.SelectBuilder,
) ([]string, []string, error) {
b.values = append(b.values, value)
return []string{fmt.Sprintf("%s_cond", key.Name)}, nil, nil
}
// TestInterpolatedValueReachesConditionBuilder pins the visitor → condition builder
// handoff for embedded variables: the interpolated string (not the raw reference) is
// the value passed to ConditionFor. The condition builder is signal-specific but this
// seam is not, so it is covered here once instead of per telemetry package.
func TestInterpolatedValueReachesConditionBuilder(t *testing.T) {
tests := []struct {
name string
expr string
variables map[string]qbtypes.VariableItem
wantValues []any
}{
{
name: "composed value in quoted string",
expr: "a = '$env-xyz'",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
wantValues: []any{"prod-xyz"},
},
{
name: "composed value in LIKE pattern",
expr: "a LIKE '$env%'",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
wantValues: []any{"prod%"},
},
{
name: "pure reference goes through standalone substitution",
expr: "a = $env",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
wantValues: []any{"prod"},
},
{
name: "multi-select collapses to first value",
expr: "a = '$env-xyz'",
variables: map[string]qbtypes.VariableItem{
"env": {Value: []any{"prod", "staging"}},
},
wantValues: []any{"prod-xyz"},
},
{
name: "composed members of an IN list",
expr: "a IN ('$env-1', '$env-2')",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
wantValues: []any{[]any{"prod-1", "prod-2"}},
},
{
name: "unknown variable passes through untouched",
expr: "a = '$unknown-xyz'",
variables: map[string]qbtypes.VariableItem{
"env": {Value: "prod"},
},
wantValues: []any{"$unknown-xyz"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
recorder := &valueRecordingConditionBuilder{}
opts := FilterExprVisitorOpts{
Context: t.Context(),
FieldKeys: visitTestKeys,
ConditionBuilder: recorder,
Variables: tt.variables,
}
result, err := PrepareWhereClause(tt.expr, opts)
assert.NoError(t, err)
assert.False(t, result.IsEmpty())
assert.Equal(t, tt.wantValues, recorder.values)
})
}
}

View File

@@ -314,6 +314,10 @@ func (VariableType) Enum() []any {
}
}
// AllVariableValue is the sentinel value a dynamic variable carries when "ALL"
// is selected; conditions referencing such a variable are dropped entirely.
const AllVariableValue = "__all__"
type VariableItem struct {
Type VariableType `json:"type"`
Value any `json:"value"`

View File

@@ -0,0 +1,134 @@
package variables
import (
"fmt"
"sort"
"strconv"
"strings"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
)
// isVariableNameChar reports whether c can be part of a variable name.
func isVariableNameChar(c byte) bool {
return c == '_' || ('0' <= c && c <= '9') || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z')
}
// lookupVariable resolves a variable name against the variables map, tolerating a
// `$` prefix on either the name or the map keys.
func lookupVariable(name string, variables map[string]qbtypes.VariableItem) (qbtypes.VariableItem, bool) {
if item, ok := variables[name]; ok {
return item, true
}
if strings.HasPrefix(name, "$") {
item, ok := variables[name[1:]]
return item, ok
}
item, ok := variables["$"+name]
return item, ok
}
// Interpolate replaces $variable references embedded in s with their values, matching
// against the names in the variables map (longest name first). A reference is only
// replaced when the character following it cannot extend a variable name: with only
// `env` defined, "$env-suffix" becomes "prod-suffix" while "$environment" stays
// untouched instead of turning into "prodironment".
// A string that is a single variable reference (e.g. "$env") is returned unchanged so
// callers can process standalone references with their type intact (lists for IN,
// numbers, __all__, etc.).
// The returned bool is true when a referenced dynamic variable has the __all__ value,
// meaning the enclosing condition must be dropped. Warnings (e.g. a multi-value
// variable collapsing to its first value) are returned for the caller to surface.
func Interpolate(s string, variables map[string]qbtypes.VariableItem) (string, bool, []string) {
if len(variables) == 0 {
return s, false, nil
}
if strings.HasPrefix(s, "$") {
if _, ok := lookupVariable(s, variables); ok {
return s, false, nil
}
}
names := make([]string, 0, len(variables))
seen := make(map[string]bool, len(variables))
for name := range variables {
name = strings.TrimPrefix(name, "$")
if name == "" || seen[name] {
continue
}
seen[name] = true
names = append(names, name)
}
// longest first so that $environment is not mistaken for $env plus a suffix
sort.Slice(names, func(i, j int) bool { return len(names[i]) > len(names[j]) })
var warnings []string
var sb strings.Builder
i := 0
for i < len(s) {
if s[i] != '$' {
sb.WriteByte(s[i])
i++
continue
}
rest := s[i+1:]
matched := false
for _, name := range names {
if !strings.HasPrefix(rest, name) {
continue
}
// a name character right after the match means this occurrence references
// a longer, unknown variable; leave it for a shorter-name check or as-is
if len(rest) > len(name) && isVariableNameChar(rest[len(name)]) {
continue
}
varItem, _ := lookupVariable(name, variables)
if varItem.Type == qbtypes.DynamicVariableType {
if allVal, ok := varItem.Value.(string); ok && allVal == qbtypes.AllVariableValue {
return "", true, warnings
}
}
sb.WriteString(formatValueForInterpolation(varItem.Value, name, &warnings))
i += 1 + len(name)
matched = true
break
}
if !matched {
sb.WriteByte('$')
i++
}
}
return sb.String(), false, warnings
}
// formatValueForInterpolation renders a variable value as a plain string for embedding
// inside a larger value. Multi-value variables collapse to their first value with a
// warning, since a pattern like "%$var%" can only hold one.
func formatValueForInterpolation(value any, varName string, warnings *[]string) string {
switch val := value.(type) {
case string:
return val
case []string:
if len(val) == 0 {
return ""
}
if len(val) > 1 {
*warnings = append(*warnings, fmt.Sprintf("variable `%s` has multiple values, using first value `%s` for string interpolation", varName, val[0]))
}
return val[0]
case []any:
if len(val) == 0 {
return ""
}
if len(val) > 1 {
*warnings = append(*warnings, fmt.Sprintf("variable `%s` has multiple values, using first value for string interpolation", varName))
}
return formatValueForInterpolation(val[0], varName, warnings)
case bool:
return strconv.FormatBool(val)
default:
return fmt.Sprintf("%v", val)
}
}

View File

@@ -2,6 +2,7 @@ package variables
import (
"fmt"
"slices"
"strconv"
"strings"
@@ -35,19 +36,22 @@ func (e *ErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol
type variableReplacementVisitor struct {
variables map[string]qbtypes.VariableItem
errors []string
warnings []string
}
// specialSkipMarker is used to indicate that a condition should be removed.
const specialSkipMarker = "__SKIP_CONDITION__"
// ReplaceVariablesInExpression takes a filter expression and returns it with variables replaced.
func ReplaceVariablesInExpression(expression string, variables map[string]qbtypes.VariableItem) (string, error) {
// ReplaceVariablesInExpression takes a filter expression and returns it with variables
// replaced, along with any warnings generated during the replacement.
func ReplaceVariablesInExpression(expression string, variables map[string]qbtypes.VariableItem) (string, []string, error) {
input := antlr.NewInputStream(expression)
lexer := grammar.NewFilterQueryLexer(input)
visitor := &variableReplacementVisitor{
variables: variables,
errors: []string{},
warnings: []string{},
}
lexerErrorListener := NewErrorListener()
@@ -63,21 +67,21 @@ func ReplaceVariablesInExpression(expression string, variables map[string]qbtype
tree := parser.Query()
if len(parserErrorListener.SyntaxErrors) > 0 {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "syntax errors in expression: %v", parserErrorListener.SyntaxErrors)
return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "syntax errors in expression: %v", parserErrorListener.SyntaxErrors)
}
result := visitor.Visit(tree).(string)
if len(visitor.errors) > 0 {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "errors processing expression: %v", visitor.errors)
return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "errors processing expression: %v", visitor.errors)
}
// If the entire expression should be skipped, return empty string
if result == specialSkipMarker {
return "", nil
return "", visitor.warnings, nil
}
return result, nil
return result, visitor.warnings, nil
}
// Visit dispatches to the specific visit method based on node type.
@@ -469,7 +473,7 @@ func (v *variableReplacementVisitor) VisitValue(ctx *grammar.ValueContext) any {
if ok {
// Handle dynamic variable with __all__ value
if varItem.Type == qbtypes.DynamicVariableType {
if allVal, ok := varItem.Value.(string); ok && allVal == "__all__" {
if allVal, ok := varItem.Value.(string); ok && allVal == qbtypes.AllVariableValue {
// Return special marker to indicate this condition should be removed
return specialSkipMarker
}
@@ -480,6 +484,18 @@ func (v *variableReplacementVisitor) VisitValue(ctx *grammar.ValueContext) any {
}
}
// Not a standalone variable reference; the value may still have variables
// embedded in it (e.g. '$env-suffix' or 'prefix-$var-suffix')
if strings.Contains(originalValue, "$") {
interpolated, hasAll := v.interpolateVariablesInString(originalValue)
if hasAll {
return specialSkipMarker
}
if interpolated != originalValue {
return v.formatVariableValue(interpolated)
}
}
// Return original value if not a variable or variable not found
// If it was quoted text and not a variable, return with quotes
if ctx.QUOTED_TEXT() != nil && !strings.HasPrefix(originalValue, "$") {
@@ -505,7 +521,7 @@ func (v *variableReplacementVisitor) VisitKey(ctx *grammar.KeyContext) any {
if ok {
// Handle dynamic variable with __all__ value
if varItem.Type == qbtypes.DynamicVariableType {
if allVal, ok := varItem.Value.(string); ok && allVal == "__all__" {
if allVal, ok := varItem.Value.(string); ok && allVal == qbtypes.AllVariableValue {
return specialSkipMarker
}
}
@@ -517,6 +533,25 @@ func (v *variableReplacementVisitor) VisitKey(ctx *grammar.KeyContext) any {
return keyText
}
// addWarning appends w unless it is already recorded; comparisons visit their values
// twice (once to check for __all__, once to rebuild), which would duplicate warnings.
func (v *variableReplacementVisitor) addWarning(w string) {
if slices.Contains(v.warnings, w) {
return
}
v.warnings = append(v.warnings, w)
}
// interpolateVariablesInString delegates to Interpolate, folding its warnings into
// the visitor.
func (v *variableReplacementVisitor) interpolateVariablesInString(s string) (string, bool) {
interpolated, hasAll, warnings := Interpolate(s, v.variables)
for _, w := range warnings {
v.addWarning(w)
}
return interpolated, hasAll
}
// formatVariableValue formats a variable value for inclusion in the expression.
func (v *variableReplacementVisitor) formatVariableValue(value any) string {
switch val := value.(type) {

View File

@@ -421,11 +421,154 @@ func TestReplaceVariablesInExpression(t *testing.T) {
},
expected: "message NOT CONTAINS 'debug'",
},
{
name: "variable composed with suffix in value",
expression: "cluster_name = '$environment-xyz'",
variables: map[string]qbtypes.VariableItem{
"environment": {
Type: qbtypes.DynamicVariableType,
Value: "prod",
},
},
expected: "cluster_name = 'prod-xyz'",
},
{
name: "variable composed with suffix without quotes",
expression: "cluster_name = $environment-xyz",
variables: map[string]qbtypes.VariableItem{
"environment": {
Type: qbtypes.DynamicVariableType,
Value: "staging",
},
},
expected: "cluster_name = 'staging-xyz'",
},
{
name: "variable in LIKE pattern with suffix",
expression: "service.name LIKE '$env%'",
variables: map[string]qbtypes.VariableItem{
"env": {
Type: qbtypes.DynamicVariableType,
Value: "prod",
},
},
expected: "service.name LIKE 'prod%'",
},
{
name: "variable composed with prefix and suffix",
expression: "label = 'prefix-$var-suffix'",
variables: map[string]qbtypes.VariableItem{
"var": {
Type: qbtypes.DynamicVariableType,
Value: "middle",
},
},
expected: "label = 'prefix-middle-suffix'",
},
{
name: "multiple variables in one string",
expression: "path = '$region-$env-cluster'",
variables: map[string]qbtypes.VariableItem{
"region": {
Type: qbtypes.DynamicVariableType,
Value: "us-west",
},
"env": {
Type: qbtypes.DynamicVariableType,
Value: "prod",
},
},
expected: "path = 'us-west-prod-cluster'",
},
{
name: "embedded variable with __all__ value skips condition",
expression: "cluster_name = '$environment-xyz'",
variables: map[string]qbtypes.VariableItem{
"environment": {
Type: qbtypes.DynamicVariableType,
Value: "__all__",
},
},
expected: "",
},
{
name: "variable with underscore composed with suffix",
expression: "name = '$my_var-test'",
variables: map[string]qbtypes.VariableItem{
"my_var": {
Type: qbtypes.DynamicVariableType,
Value: "hello",
},
},
expected: "name = 'hello-test'",
},
{
name: "similar variable names - longer matches first",
expression: "name = '$env-$environment'",
variables: map[string]qbtypes.VariableItem{
"env": {
Type: qbtypes.DynamicVariableType,
Value: "dev",
},
"environment": {
Type: qbtypes.DynamicVariableType,
Value: "production",
},
},
expected: "name = 'dev-production'",
},
{
// Unresolved $-prefixed values come back unquoted so they stay
// variable-shaped for downstream resolution.
name: "shorter variable does not match inside longer unknown reference",
expression: "name = '$environment'",
variables: map[string]qbtypes.VariableItem{
"env": {
Type: qbtypes.DynamicVariableType,
Value: "prod",
},
},
// `env` must not corrupt `$environment` into 'prodironment'
expected: "name = $environment",
},
{
name: "unknown embedded variable left untouched",
expression: "name = '$unknown-suffix'",
variables: map[string]qbtypes.VariableItem{
"env": {
Type: qbtypes.DynamicVariableType,
Value: "prod",
},
},
expected: "name = $unknown-suffix",
},
{
name: "embedded variable in unquoted value with prefix",
expression: "cluster_name = abc-$env",
variables: map[string]qbtypes.VariableItem{
"env": {
Type: qbtypes.DynamicVariableType,
Value: "prod",
},
},
expected: "cluster_name = 'abc-prod'",
},
{
name: "multi-select variable in composed string takes first value",
expression: "cluster_name = '$env-xyz'",
variables: map[string]qbtypes.VariableItem{
"env": {
Type: qbtypes.CustomVariableType,
Value: []any{"prod", "staging"},
},
},
expected: "cluster_name = 'prod-xyz'",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := ReplaceVariablesInExpression(tt.expression, tt.variables)
result, _, err := ReplaceVariablesInExpression(tt.expression, tt.variables)
if tt.wantErr {
assert.Error(t, err)
@@ -438,6 +581,24 @@ func TestReplaceVariablesInExpression(t *testing.T) {
}
}
// TestReplaceVariablesInExpressionWarnings ensures multi-value variables used inside a
// composed string produce exactly one warning, even though comparisons visit their
// values twice (once for the __all__ check, once to rebuild the expression).
func TestReplaceVariablesInExpressionWarnings(t *testing.T) {
variables := map[string]qbtypes.VariableItem{
"env": {
Type: qbtypes.CustomVariableType,
Value: []any{"prod", "staging"},
},
}
result, warnings, err := ReplaceVariablesInExpression("cluster_name = '$env-xyz'", variables)
assert.NoError(t, err)
assert.Equal(t, "cluster_name = 'prod-xyz'", result)
assert.Len(t, warnings, 1)
assert.Contains(t, warnings[0], "`env` has multiple values")
}
func TestFormatVariableValue(t *testing.T) {
visitor := &variableReplacementVisitor{}

View File

@@ -972,3 +972,28 @@ def make_scalar_query_request(
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
},
)
def make_substitute_vars_request(
signoz: types.SigNoz,
token: str,
start_ms: int,
end_ms: int,
queries: list[dict],
variables: dict,
timeout: int = QUERY_TIMEOUT,
) -> requests.Response:
return requests.post(
signoz.self.host_configs["8080"].get("/api/v5/substitute_vars"),
timeout=timeout,
headers={"authorization": f"Bearer {token}"},
json={
"schemaVersion": "v1",
"start": start_ms,
"end": end_ms,
"requestType": "time_series",
"compositeQuery": {"queries": queries},
"variables": variables,
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
},
)

View File

@@ -0,0 +1,126 @@
"""
Tests for the /api/v5/substitute_vars endpoint (GitHub issue #10008).
Variables are supported both as standalone RHS values (field = $var, field IN $var)
and composed inside values ('$env-suffix', LIKE '$env%').
"""
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import Aggregation, BuilderQuery, make_substitute_vars_request
@pytest.mark.parametrize(
"expression,variables,expected",
[
pytest.param(
"service.name = $service",
{"service": {"type": "query", "value": "auth-service"}},
"service.name = 'auth-service'",
id="standalone_variable",
),
pytest.param(
"cluster_name = '$environment-xyz'",
{"environment": {"type": "custom", "value": "prod"}},
"cluster_name = 'prod-xyz'",
id="variable_in_string",
),
pytest.param(
"service.name = $service AND env = $environment",
{
"service": {"type": "text", "value": "auth-service"},
"environment": {"type": "query", "value": "production"},
},
"service.name = 'auth-service' AND env = 'production'",
id="multiple_variables",
),
pytest.param(
"service.name IN $services",
{"services": {"type": "query", "value": ["auth-service", "api-service"]}},
"service.name IN ['auth-service', 'api-service']",
id="array_variable",
),
pytest.param(
"service.name LIKE '$env%'",
{"env": {"type": "text", "value": "prod"}},
"service.name LIKE 'prod%'",
id="like_pattern",
),
pytest.param(
"cluster_name = $environment-xyz",
{"environment": {"type": "dynamic", "value": "staging"}},
"cluster_name = 'staging-xyz'",
id="variable_without_quotes",
),
# `env` must not match inside the longer, undefined `$environment` reference.
# Unresolved $-prefixed values come back unquoted so they stay variable-shaped.
pytest.param(
"cluster_name = '$environment-xyz'",
{"env": {"type": "custom", "value": "prod"}},
"cluster_name = $environment-xyz",
id="unknown_embedded_variable_untouched",
),
# expression should be empty when __all__ is used
pytest.param(
"service.name = $service",
{"service": {"type": "dynamic", "value": "__all__"}},
"",
id="all_value_standalone",
),
pytest.param(
"cluster_name = '$environment-xyz'",
{"environment": {"type": "dynamic", "value": "__all__"}},
"",
id="all_value_in_composed_string",
),
# only the env condition should remain
pytest.param(
"service.name = $service AND env = $env",
{
"service": {"type": "dynamic", "value": "__all__"},
"env": {"type": "dynamic", "value": "production"},
},
"env = 'production'",
id="all_value_partial",
),
],
)
def test_substitute_vars(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
expression: str,
variables: dict,
expected: str,
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
now = datetime.now(tz=UTC)
response = make_substitute_vars_request(
signoz,
token,
int((now - timedelta(minutes=5)).timestamp() * 1000),
int(now.timestamp() * 1000),
[
BuilderQuery(
signal="logs",
aggregations=[Aggregation("count()")],
step_interval=60,
filter_expression=expression,
).to_dict()
],
variables,
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
queries = response.json()["data"]["compositeQuery"]["queries"]
assert len(queries) == 1
assert queries[0]["spec"]["filter"]["expression"] == expected

View File

@@ -0,0 +1,274 @@
"""
End-to-end tests for variables in /api/v5/query_range filter expressions
(GitHub issue #10008): standalone ($var, IN $var), composed inside values
('$env-suffix', LIKE '$env%'), and the special __all__ value.
"""
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import (
BuilderQuery,
OrderBy,
TelemetryFieldKey,
get_column_data_from_response,
get_scalar_table_data,
make_query_request,
)
@pytest.mark.parametrize(
"expression,variables,expected_logs",
[
pytest.param(
"service.name = $service",
{"service": {"type": "query", "value": "auth-service"}},
{"auth-prod-log"},
id="standalone_variable",
),
pytest.param(
"service.name IN $services",
{"services": {"type": "custom", "value": ["auth-service", "api-service"]}},
{"auth-prod-log", "api-prod-log"},
id="array_variable",
),
pytest.param(
"cluster_name = '$environment-xyz'",
{"environment": {"type": "dynamic", "value": "prod"}},
{"auth-prod-log", "api-prod-log"},
id="variable_composed_in_string",
),
pytest.param(
"cluster_name LIKE '$env%'",
{"env": {"type": "text", "value": "prod"}},
{"auth-prod-log", "api-prod-log"},
id="variable_in_like_pattern",
),
pytest.param(
"path = '$region-$env-cluster'",
{
"region": {"type": "query", "value": "us-west"},
"env": {"type": "custom", "value": "prod"},
},
{"auth-prod-log"},
id="multiple_variables_in_string",
),
pytest.param(
"label = 'prefix-$var-suffix'",
{"var": {"type": "text", "value": "middle"}},
{"auth-prod-log"},
id="variable_with_prefix_and_suffix",
),
pytest.param(
"cluster_name = $environment-xyz",
{"environment": {"type": "custom", "value": "staging"}},
{"web-staging-log"},
id="variable_without_quotes",
),
pytest.param(
"service.name = $service",
{"service": {"type": "query", "value": "auth-service"}},
{"auth-prod-log"},
id="query_variable_type",
),
pytest.param(
"service.name = $service",
{"service": {"type": "custom", "value": "auth-service"}},
{"auth-prod-log"},
id="custom_variable_type",
),
pytest.param(
"service.name = $service",
{"service": {"type": "text", "value": "auth-service"}},
{"auth-prod-log"},
id="text_variable_type",
),
pytest.param(
"service.name = $service",
{"service": {"type": "dynamic", "value": "auth-service"}},
{"auth-prod-log"},
id="dynamic_variable_type",
),
# `__all__` removes the filter condition entirely → all logs match
pytest.param(
"service.name = $service",
{"service": {"type": "dynamic", "value": "__all__"}},
{"auth-prod-log", "api-prod-log", "web-staging-log"},
id="all_value_standalone",
),
pytest.param(
"cluster_name = '$environment-xyz'",
{"environment": {"type": "dynamic", "value": "__all__"}},
{"auth-prod-log", "api-prod-log", "web-staging-log"},
id="all_value_in_composed_string",
),
pytest.param(
"service.name IN $services",
{"services": {"type": "dynamic", "value": "__all__"}},
{"auth-prod-log", "api-prod-log", "web-staging-log"},
id="all_value_in_array",
),
# `__all__` for service drops only that condition; env filter still applies
pytest.param(
"service.name = $service AND env = $env",
{
"service": {"type": "dynamic", "value": "__all__"},
"env": {"type": "dynamic", "value": "production"},
},
{"auth-prod-log", "api-prod-log"},
id="all_value_partial_filter",
),
],
)
def test_variable_in_query(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
expression: str,
variables: dict,
expected_logs: set[str],
) -> None:
"""
Runs a raw logs query with the filter expression and variables, then checks the
set of matching log bodies.
Canonical dataset:
auth-prod-log: service.name=auth-service, env=production, cluster_name=prod-xyz,
path=us-west-prod-cluster, label=prefix-middle-suffix
api-prod-log: service.name=api-service, env=production, cluster_name=prod-xyz,
path=us-east-prod-cluster, label=prefix-other-suffix
web-staging-log: service.name=web-service, env=staging, cluster_name=staging-xyz,
path=eu-west-staging-cluster
"""
now = datetime.now(tz=UTC)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=5),
body="auth-prod-log",
resources={"service.name": "auth-service"},
attributes={
"env": "production",
"cluster_name": "prod-xyz",
"path": "us-west-prod-cluster",
"label": "prefix-middle-suffix",
},
),
Logs(
timestamp=now - timedelta(seconds=4),
body="api-prod-log",
resources={"service.name": "api-service"},
attributes={
"env": "production",
"cluster_name": "prod-xyz",
"path": "us-east-prod-cluster",
"label": "prefix-other-suffix",
},
),
Logs(
timestamp=now - timedelta(seconds=3),
body="web-staging-log",
resources={"service.name": "web-service"},
attributes={
"env": "staging",
"cluster_name": "staging-xyz",
"path": "eu-west-staging-cluster",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
now = datetime.now(tz=UTC)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
BuilderQuery(
signal="logs",
limit=100,
filter_expression=expression,
order=[
OrderBy(key=TelemetryFieldKey(name="timestamp"), direction="desc"),
OrderBy(key=TelemetryFieldKey(name="id"), direction="desc"),
],
).to_dict()
],
variables=variables,
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
assert set(get_column_data_from_response(response.json(), "body")) == expected_logs
def test_variable_in_query_with_group_by(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
A composed variable in the filter works together with groupBy: count logs
with cluster_name = '$env-xyz' grouped by service.name.
"""
now = datetime.now(tz=UTC)
logs: list[Logs] = []
for i in range(3):
for service, cluster in [
("auth-service", "prod-xyz"),
("api-service", "prod-xyz"),
("web-service", "staging-xyz"),
]:
logs.append(
Logs(
timestamp=now - timedelta(seconds=i + 1),
body=f"{service} log {i}",
resources={"service.name": service},
attributes={"cluster_name": cluster},
)
)
insert_logs(logs)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
now = datetime.now(tz=UTC)
# BuilderQuery has no groupBy field, so the spec is built inline
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="scalar",
queries=[
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "logs",
"disabled": False,
"filter": {"expression": "cluster_name = '$env-xyz'"},
"groupBy": [{"name": "service.name", "fieldContext": "resource"}],
"aggregations": [{"expression": "count()"}],
},
}
],
variables={"env": {"type": "query", "value": "prod"}},
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
rows = get_scalar_table_data(response.json())
assert sorted(rows) == [["api-service", 3], ["auth-service", 3]]