Compare commits

...

3 Commits

Author SHA1 Message Date
Tushar Vats
7af25ded00 perf(logs): let body_v2 filters use the body_v2 bloom filters
logs_v2 indexes lower(toString(body_v2)), the serialized JSON, while `body`
resolves to the body_v2.message sub-column — so nothing a body filter compares
matches an index expression and every granule is read.

AND in a LIKE over the indexed expression. `=` and IN anchor on the value with
toJSONString's quotes, pinning the needle to a complete field value; CONTAINS
and LIKE take one needle per literal run of the pattern that is at least as
long as the ngram size. The original predicate still decides the row, so this
only prunes. On 1M rows, 123/123 granules down to 1/123.

ClickHouse escapes the needle with toJSONString, the same function that wrote
the column, rather than mirroring those rules here: a term holding `/` or `"`
is stored escaped and finds nothing without them. replaceAll then escapes the
LIKE metacharacters. The whole expression folds to a constant, which is what
keeps the index usable.

likePatternLiterals honours ClickHouse's own escape rules, where `\` is an
escape only before `%`, `_` and itself — reading `\t` as a bare `t` would build
a needle the pattern does not require and drop matching rows.
2026-08-11 02:36:54 +05:30
Tushar Vats
b09cfd4869 perf(logs): let body equality use the lower(body) bloom filters
logs_v2 indexes lower(body) with a token and an ngram bloom filter, so a plain
`body = ?` matches no index expression and reads every granule. AND in the
lowered comparison as a redundant predicate: the bloom filters can prune on it
and the exact comparison still decides the row. On 1M rows that is 123/123
granules down to 1/123.

IN picks this up for free through the `=` delegation, so a list of bodies
prunes per arm.

ClickHouse folds LOWER(?) on the bound value, so the predicate stays a constant
the index can prune on, and the fold matches lower() on the column by
construction rather than by a reimplementation on our side.

Scoped to the legacy body column: body_v2 keeps the value in a JSON column
indexed on lower(toString(body_v2)), which needs a different predicate.

The integration cases run the same expressions with the flag off and on. The
companion is case-insensitive where the equality is not, so they pin that down:
a body differing only in case must not come back, and the flag-on path — which
skips the companion — has to answer identically.
2026-08-11 02:13:36 +05:30
Tushar Vats
0d633701c4 refactor(qb): build IN as an OR of equalities
The IN and NOT IN cases route each value back through the condition builder
with `=` / `!=`, instead of assembling the comparisons themselves. Whatever a
builder does for a scalar comparison then applies to the list form without
being restated.

Applied to logs, traces, audit and resourcefilter, which all fanned a list out
into per-value comparisons already. Metrics and rulestatehistory build a real
sb.In, so there is nothing to delegate to. telemetrymetadata is left alone as
well: it applies a key-existence guard at a single exit, so a recursed arm
comes back already wrapped, and either the guard nests or the case has to skip
the shared tail and lose the invariant that every case is guarded.

This fixes `body.<path>[*] IN [...]` with use_json_body off, which returned a
500. The list shape made the path extract as Array(String), and comparing that
to each scalar is something ClickHouse rejects outright (code 130); extracting
per value reads the field instead. Covered end-to-end by the new case in
querierlogs/06_json_body.py, which fails on main and passes here.

resourcefilter changes shape without changing results: each value is paired
with its own index filter — `(e1 AND k AND l1) OR (e2 AND k AND l2)` rather
than `(e1 OR e2) AND k AND (l1 OR l2)` — which selects the same rows because
each equality implies its own filter.
2026-08-11 00:58:04 +05:30
13 changed files with 613 additions and 49 deletions

View File

@@ -197,8 +197,8 @@ func TestJSONStmtBuilder_PrimitivePaths(t *testing.T) {
name: "message Contains 'Iron Award'",
filter: "message Contains 'Iron Award'",
expected: TestExpected{
WhereClause: "LOWER(body_v2.message) LIKE LOWER(?)",
Args: []any{"%Iron Award%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
WhereClause: "(LOWER(body_v2.message) LIKE LOWER(?) AND LOWER(toString(body_v2)) LIKE " + jsonSubstringNeedle + ")",
Args: []any{"%Iron Award%", "Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
},

View File

@@ -469,6 +469,8 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
expectedErr: nil,
},
{
// The `[*]` path is extracted per value, not as an Array(String) compared to a
// scalar — ClickHouse rejects that outright (code 130).
name: "IN operator with json search",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
@@ -479,7 +481,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(String)') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSON_VALUE(body, '$.\"user_names\"[*]') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{querybuilder.NewKeyNotFoundWarning("user_names[*]")},
},
@@ -943,6 +945,9 @@ func TestAdjustKey(t *testing.T) {
}
}
// The needle ClickHouse builds for the body_v2 index expression.
const jsonSubstringNeedle = `concat('%', replaceAll(replaceAll(replaceAll(lower(substring(toJSONString(?), 2, length(toJSONString(?)) - 2)), '\\', '\\\\'), '%', '\\%'), '_', '\\_'), '%')`
func TestStmtBuilderBodyField(t *testing.T) {
cases := []struct {
name string
@@ -1009,8 +1014,8 @@ func TestStmtBuilderBodyField(t *testing.T) {
},
enableUseJSONBody: false,
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE body = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (body = ? AND LOWER(body) = LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"", "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
@@ -1024,8 +1029,8 @@ func TestStmtBuilderBodyField(t *testing.T) {
},
enableUseJSONBody: true,
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE LOWER(body_v2.message) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (LOWER(body_v2.message) LIKE LOWER(?) AND LOWER(toString(body_v2)) LIKE " + jsonSubstringNeedle + ") AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"%error%", "error", "error", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{bodySearchDefaultWarning},
},
expectedErr: nil,

View File

@@ -180,20 +180,16 @@ func (b *defaultConditionBuilder) conditionForKey(
if !ok {
return "", qbtypes.ErrInValues
}
// each value carries its own index filter, since `=` derives one from the value
inConditions := make([]string, 0, len(values))
for _, v := range values {
inConditions = append(inConditions, sb.E(fieldName, querybuilder.FormatValueForContains(v)))
}
mainCondition := sb.Or(inConditions...)
valConditions := make([]string, 0, len(values))
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
for _, v := range valuesForIndexFilter {
valConditions = append(valConditions, sb.Like(column.Name, v))
cond, err := b.conditionForKey(ctx, startNs, endNs, key, qbtypes.FilterOperatorEqual, v, sb)
if err != nil {
return "", err
}
inConditions = append(inConditions, cond)
}
mainCondition = sb.And(mainCondition, keyIdxFilter, sb.Or(valConditions...))
return mainCondition, nil
return sb.Or(inConditions...), nil
case qbtypes.FilterOperatorNotIn:
values, ok := value.([]any)
if !ok {
@@ -201,17 +197,13 @@ func (b *defaultConditionBuilder) conditionForKey(
}
notInConditions := make([]string, 0, len(values))
for _, v := range values {
notInConditions = append(notInConditions, sb.NE(fieldName, querybuilder.FormatValueForContains(v)))
}
mainCondition := sb.And(notInConditions...)
valConditions := make([]string, 0, len(values))
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
for _, v := range valuesForIndexFilter {
valConditions = append(valConditions, sb.NotLike(column.Name, v))
cond, err := b.conditionForKey(ctx, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, v, sb)
if err != nil {
return "", err
}
notInConditions = append(notInConditions, cond)
}
mainCondition = sb.And(mainCondition, sb.And(valConditions...))
return mainCondition, nil
return sb.And(notInConditions...), nil
case qbtypes.FilterOperatorExists:
return sb.And(

View File

@@ -109,8 +109,8 @@ func TestConditionBuilder(t *testing.T) {
},
op: qbtypes.FilterOperatorIn,
value: []any{"watch", "redis"},
expected: "(simpleJSONExtractString(labels, 'k8s.namespace.name') = ? OR simpleJSONExtractString(labels, 'k8s.namespace.name') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)",
expectedArgs: []any{"watch", "redis", "%k8s.namespace.name%", "%k8s.namespace.name\":\"watch%", "%k8s.namespace.name\":\"redis%"},
expected: "((simpleJSONExtractString(labels, 'k8s.namespace.name') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'k8s.namespace.name') = ? AND labels LIKE ? AND labels LIKE ?))",
expectedArgs: []any{"watch", "%k8s.namespace.name%", "%k8s.namespace.name\":\"watch%", "redis", "%k8s.namespace.name%", "%k8s.namespace.name\":\"redis%"},
},
{
name: "string_not_in",
@@ -120,8 +120,8 @@ func TestConditionBuilder(t *testing.T) {
},
op: qbtypes.FilterOperatorNotIn,
value: []any{"watch", "redis"},
expected: "(simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND simpleJSONExtractString(labels, 'k8s.namespace.name') <> ?) AND (labels NOT LIKE ? AND labels NOT LIKE ?)",
expectedArgs: []any{"watch", "redis", "%k8s.namespace.name\":\"watch%", "%k8s.namespace.name\":\"redis%"},
expected: "((simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND labels NOT LIKE ?) AND (simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND labels NOT LIKE ?))",
expectedArgs: []any{"watch", "%k8s.namespace.name\":\"watch%", "redis", "%k8s.namespace.name\":\"redis%"},
},
{
name: "string_exists",
@@ -173,8 +173,8 @@ func TestConditionBuilder(t *testing.T) {
},
op: qbtypes.FilterOperatorIn,
value: []any{1, 2},
expected: "(simpleJSONExtractString(labels, 'test_num') = ? OR simpleJSONExtractString(labels, 'test_num') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)",
expectedArgs: []any{"1", "2", "%test_num%", "%test_num\":\"1%", "%test_num\":\"2%"},
expected: "((simpleJSONExtractString(labels, 'test_num') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'test_num') = ? AND labels LIKE ? AND labels LIKE ?))",
expectedArgs: []any{"1", "%test_num%", "%test_num\":\"1%", "2", "%test_num%", "%test_num\":\"2%"},
},
{
name: "number_between",

View File

@@ -229,8 +229,8 @@ func TestResourceFilterStatementBuilder_Traces(t *testing.T) {
start: testStartNs,
end: testEndNs,
expected: &qbtypes.Statement{
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? OR simpleJSONExtractString(labels, 'service.name') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
Args: []any{"redis", "postgres", "%service.name%", "%service.name\":\"redis%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
Args: []any{"redis", "%service.name%", "%service.name\":\"redis%", "postgres", "%service.name%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
},
},
{
@@ -244,8 +244,8 @@ func TestResourceFilterStatementBuilder_Traces(t *testing.T) {
start: testStartNs,
end: testEndNs,
expected: &qbtypes.Statement{
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') <> ? AND simpleJSONExtractString(labels, 'service.name') <> ?) AND (labels NOT LIKE ? AND labels NOT LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
Args: []any{"redis", "postgres", "%service.name\":\"redis%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') <> ? AND labels NOT LIKE ?) AND (simpleJSONExtractString(labels, 'service.name') <> ? AND labels NOT LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
Args: []any{"redis", "%service.name\":\"redis%", "postgres", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
},
},
{

View File

@@ -94,7 +94,11 @@ func (c *conditionBuilder) conditionFor(
}
conditions := []string{}
for _, value := range values {
conditions = append(conditions, sb.E(fieldExpression, value))
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
if err != nil {
return "", err
}
conditions = append(conditions, cond)
}
return sb.Or(conditions...), nil
case qbtypes.FilterOperatorNotIn:
@@ -104,7 +108,11 @@ func (c *conditionBuilder) conditionFor(
}
conditions := []string{}
for _, value := range values {
conditions = append(conditions, sb.NE(fieldExpression, value))
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
if err != nil {
return "", err
}
conditions = append(conditions, cond)
}
return sb.And(conditions...), nil
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"regexp"
"strings"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
@@ -28,6 +29,83 @@ func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionB
return &conditionBuilder{fm: fm, fl: fl}
}
const (
// bodyV2NgramSize is the n of the body_v2 ngrambf_v1 index; a shorter needle yields no
// ngram for it to check.
bodyV2NgramSize = 4
loweredBodyV2 = "LOWER(toString(" + LogsV2BodyV2Column + "))"
)
// bodyIndexLike renders a LIKE on the body_v2 index expression asserting the serialized JSON
// contains value. toJSONString escapes the value the same way the column stores it, and
// replaceAll then escapes the LIKE metacharacters — backslash first, so the ones the later
// steps add are not doubled again. The expression folds to a constant, which is what keeps
// the bloom filters usable. anchored keeps toJSONString's quotes, pinning the needle to a
// complete field value.
func bodyIndexLike(value string, anchored bool, sb *sqlbuilder.SelectBuilder) string {
escaped := fmt.Sprintf("lower(toJSONString(%s))", sb.Var(value))
if !anchored {
quoted := fmt.Sprintf("toJSONString(%s)", sb.Var(value))
escaped = fmt.Sprintf("lower(substring(%s, 2, length(%s) - 2))", quoted, quoted)
}
return fmt.Sprintf(
`%s LIKE concat('%%', replaceAll(replaceAll(replaceAll(%s, '\\', '\\\\'), '%%', '\\%%'), '_', '\\_'), '%%')`,
loweredBodyV2, escaped,
)
}
// likePatternLiterals returns the runs of pattern between unescaped wildcards, with `\`
// escapes resolved. Every value the pattern matches contains each run verbatim. ClickHouse
// treats `\` as an escape only before `%`, `_` and itself; elsewhere it stands for a literal
// backslash, so dropping it would yield a run the pattern does not require.
func likePatternLiterals(pattern string) []string {
var (
literals []string
run strings.Builder
)
for i := 0; i < len(pattern); i++ {
switch c := pattern[i]; c {
case '%', '_':
if run.Len() > 0 {
literals = append(literals, run.String())
run.Reset()
}
case '\\':
if i+1 >= len(pattern) {
run.WriteByte('\\')
continue
}
i++
if escaped := pattern[i]; escaped != '%' && escaped != '_' && escaped != '\\' {
run.WriteByte('\\')
}
run.WriteByte(pattern[i])
default:
run.WriteByte(c)
}
}
if run.Len() > 0 {
literals = append(literals, run.String())
}
return literals
}
// bodyV2SubstringPredicates returns predicates implied by body_v2.message matching pattern,
// one per literal run long enough for the ngram index. The legacy body needs none: it already
// queries LOWER(body), which is its own index expression.
func bodyV2SubstringPredicates(fieldExpression, pattern string, sb *sqlbuilder.SelectBuilder) []string {
if fieldExpression != messageSubColumn {
return nil
}
var preds []string
for _, literal := range likePatternLiterals(pattern) {
if len(literal) >= bodyV2NgramSize {
preds = append(preds, bodyIndexLike(literal, false, sb))
}
}
return preds
}
// conditionForSearch ORs a case-insensitive match of the search term across the key
// context's searchable columns (unspecified context = every column).
func (c *conditionBuilder) conditionForSearch(
@@ -314,8 +392,31 @@ func (c *conditionBuilder) conditionForResolvedKey(
// make use of case insensitive index for body
if fieldExpression == "body" || fieldExpression == messageSubColumn {
switch operator {
case qbtypes.FilterOperatorLike:
return sb.ILike(fieldExpression, value), nil
case qbtypes.FilterOperatorEqual:
// Bloom filters index lower(body) and lower(toString(body_v2)), not the column
// being compared; `=` still decides the row.
str, isStr := value.(string)
if isStr && fieldExpression == LogsV2BodyColumn {
return sb.And(
sb.E(fieldExpression, value),
fmt.Sprintf("LOWER(%s) = LOWER(%s)", fieldExpression, sb.Var(value)),
), nil
}
if isStr && str != "" && fieldExpression == messageSubColumn {
return sb.And(sb.E(fieldExpression, value), bodyIndexLike(str, true, sb)), nil
}
case qbtypes.FilterOperatorLike, qbtypes.FilterOperatorILike, qbtypes.FilterOperatorContains:
pattern, isStr := value.(string)
if operator == qbtypes.FilterOperatorContains {
pattern, isStr = fmt.Sprintf("%%%s%%", value), true
}
if isStr {
cond := sb.ILike(fieldExpression, pattern)
if preds := bodyV2SubstringPredicates(fieldExpression, pattern, sb); len(preds) > 0 {
return sb.And(append([]string{cond}, preds...)...), nil
}
return cond, nil
}
case qbtypes.FilterOperatorNotLike:
return sb.NotILike(fieldExpression, value), nil
case qbtypes.FilterOperatorRegexp:
@@ -410,7 +511,11 @@ func (c *conditionBuilder) conditionForResolvedKey(
// instead of using IN, we use `=` + `OR` to make use of index
conditions := []string{}
for _, value := range values {
conditions = append(conditions, sb.E(fieldExpression, value))
cond, err := c.conditionForResolvedKey(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
if err != nil {
return "", err
}
conditions = append(conditions, cond)
}
return sb.Or(conditions...), nil
case qbtypes.FilterOperatorNotIn:
@@ -421,7 +526,11 @@ func (c *conditionBuilder) conditionForResolvedKey(
// instead of using NOT IN, we use `!=` + `AND` to make use of index
conditions := []string{}
for _, value := range values {
conditions = append(conditions, sb.NE(fieldExpression, value))
cond, err := c.conditionForResolvedKey(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
if err != nil {
return "", err
}
conditions = append(conditions, cond)
}
return sb.And(conditions...), nil

View File

@@ -168,9 +168,9 @@ func TestConditionFor(t *testing.T) {
FieldContext: telemetrytypes.FieldContextLog,
},
operator: qbtypes.FilterOperatorEqual,
value: "error message",
expectedSQL: "body = ?",
expectedArgs: []any{"error message"},
value: "Error Message",
expectedSQL: "(body = ? AND LOWER(body) = LOWER(?))",
expectedArgs: []any{"Error Message", "Error Message"},
expectedError: nil,
},
{
@@ -619,8 +619,8 @@ func TestConditionForMultipleKeys(t *testing.T) {
},
operator: qbtypes.FilterOperatorEqual,
value: "error message",
expectedSQL: "body = ? AND severity_text = ?",
expectedArgs: []any{"error message", "error message"},
expectedSQL: "(body = ? AND LOWER(body) = LOWER(?)) AND severity_text = ?",
expectedArgs: []any{"error message", "error message", "error message"},
expectedError: nil,
},
}
@@ -905,3 +905,116 @@ func TestConditionForJSONBodySearch(t *testing.T) {
})
}
}
// IN on the body column routes each value back through the `=` path, so every arm picks up
// the lower(body) companion — including the values a mixed-type list stringifies.
func TestConditionForBodyIn(t *testing.T) {
testCases := []struct {
name string
values []any
expectedSQL string
expectedArgs []any
}{
{
name: "strings",
values: []any{"alpha", "beta"},
expectedSQL: "((body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)))",
expectedArgs: []any{"alpha", "alpha", "beta", "beta"},
},
{
name: "mixed types are stringified before they reach the column",
values: []any{"alpha", float64(1), true},
expectedSQL: "((body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)))",
expectedArgs: []any{"alpha", "alpha", "1", "1", "true", "true"},
},
}
fl := flaggertest.New(t)
fm := NewFieldMapper(fl)
conditionBuilder := NewConditionBuilder(fm, fl)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
key := telemetrytypes.TelemetryFieldKey{
Name: "body",
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
sb := sqlbuilder.NewSelectBuilder()
sb.Select("1").From("t")
cond, _, err := conditionBuilder.ConditionFor(context.Background(), valuer.UUID{}, 0, 0, &key,
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, qbtypes.ConditionBuilderOptions{},
qbtypes.FilterOperatorIn, tc.values, sb)
require.NoError(t, err)
sb.Where(cond...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, sql, tc.expectedSQL)
assert.Equal(t, tc.expectedArgs, args)
})
}
}
// The escaping is ClickHouse's own toJSONString, verified end-to-end in
// querier_json_body/07_body_contains.py; what matters here is that the needle stays a
// constant expression the bloom filters can fold.
func TestBodyIndexLike(t *testing.T) {
testCases := []struct {
name string
value string
anchored bool
expected string
expectedArgs []any
}{
{
name: "anchored keeps the quotes toJSONString adds",
value: "GET /api/v1/users",
anchored: true,
expected: `LOWER(toString(body_v2)) LIKE concat('%', replaceAll(replaceAll(replaceAll(lower(toJSONString(?)), '\\', '\\\\'), '%', '\\%'), '_', '\\_'), '%')`,
expectedArgs: []any{"GET /api/v1/users"},
},
{
name: "substring strips them and binds the value twice",
value: "/api/v1",
anchored: false,
expected: `LOWER(toString(body_v2)) LIKE concat('%', replaceAll(replaceAll(replaceAll(lower(substring(toJSONString(?), 2, length(toJSONString(?)) - 2)), '\\', '\\\\'), '%', '\\%'), '_', '\\_'), '%')`,
expectedArgs: []any{"/api/v1", "/api/v1"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select("1").From("t")
sb.Where(bodyIndexLike(tc.value, tc.anchored, sb))
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, query, tc.expected)
assert.Equal(t, tc.expectedArgs, args)
})
}
}
// ClickHouse treats `\` as an escape only before `%`, `_` and itself — every expectation
// here was checked against it, where the pattern matches the run and nothing shorter.
func TestLikePatternLiterals(t *testing.T) {
testCases := []struct {
name string
pattern string
expected []string
}{
{"contains wraps a plain value", "%error%", []string{"error"}},
{"wildcards split runs", "%foo%bar%", []string{"foo", "bar"}},
{"underscore splits too", "a_b", []string{"a", "b"}},
{"escaped wildcards stay literal", `%100\%\_off%`, []string{`100%_off`}},
{"escaped backslash collapses", `%C:\\tmp%`, []string{`C:\tmp`}},
{"backslash before other chars is literal", `%C:\tmp%`, []string{`C:\tmp`}},
{"trailing backslash is literal", `%path\`, []string{`path\`}},
{"no literals at all", "%_%", nil},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.expected, likePatternLiterals(tc.pattern))
})
}
}

View File

@@ -135,7 +135,11 @@ func (c *conditionBuilder) conditionFor(
// instead of using IN, we use `=` + `OR` to make use of index
conditions := []string{}
for _, value := range values {
conditions = append(conditions, sb.E(fieldExpression, value))
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
if err != nil {
return "", err
}
conditions = append(conditions, cond)
}
return sb.Or(conditions...), nil
case qbtypes.FilterOperatorNotIn:
@@ -146,7 +150,11 @@ func (c *conditionBuilder) conditionFor(
// instead of using NOT IN, we use `!=` + `AND` to make use of index
conditions := []string{}
for _, value := range values {
conditions = append(conditions, sb.NE(fieldExpression, value))
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
if err != nil {
return "", err
}
conditions = append(conditions, cond)
}
return sb.And(conditions...), nil

View File

@@ -0,0 +1,90 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import build_order_by, build_raw_query, get_rows, make_query_request
LOWER = "alpha"
UPPER = "ALPHA"
PLAIN = "beta"
NON_ASCII = "Mixed CASE Ünïcode"
SLASH = "GET /api/v1/users"
SUPERSTRING = "GET /api/v1/users/42"
QUOTE = 'say "hi" now'
BACKSLASH = "C:\\tmp\\log"
LIKE_META = "100% _off"
TAB = "tab\there"
CTRL = "ctrl\x01here"
BODIES = [LOWER, UPPER, PLAIN, NON_ASCII, SLASH, SUPERSTRING, QUOTE, BACKSLASH, LIKE_META, TAB, CTRL]
# querierlogs/16_body_equality.py with use_json_body on: `body` resolves to body_v2.message,
# which the lower(body) companion skips, and the same expressions must still answer alike.
@pytest.mark.parametrize(
"expression,expected_bodies",
[
pytest.param(f"body = '{LOWER}'", {LOWER}, id="equality_exact"),
pytest.param(f"body = '{UPPER}'", {UPPER}, id="equality_other_case"),
pytest.param("body = 'Alpha'", set(), id="equality_case_must_match"),
pytest.param(f"body = '{NON_ASCII}'", {NON_ASCII}, id="equality_non_ascii"),
pytest.param("body = 'gamma'", set(), id="equality_no_match"),
pytest.param(f"body = '{SLASH}'", {SLASH}, id="equality_slash"),
pytest.param("body = 'say \"hi\" now'", {QUOTE}, id="equality_quote"),
pytest.param(r"body = 'C:\\tmp\\log'", {BACKSLASH}, id="equality_backslash"),
pytest.param(f"body = '{LIKE_META}'", {LIKE_META}, id="equality_like_metacharacters"),
pytest.param("body = 'tab\there'", {TAB}, id="equality_tab"),
pytest.param("body = 'ctrl\x01here'", {CTRL}, id="equality_control_char"),
pytest.param("body = 'GET /api/v1'", set(), id="equality_prefix_does_not_match"),
pytest.param(f"body IN ('{LOWER}', '{PLAIN}')", {LOWER, PLAIN}, id="in_excludes_other_case"),
pytest.param(f"body IN ('{SLASH}', '{LIKE_META}')", {SLASH, LIKE_META}, id="in_escaped_values"),
pytest.param(f"body NOT IN ('{LOWER}', '{UPPER}')", set(BODIES) - {LOWER, UPPER}, id="not_in"),
],
)
def test_logs_body_equality_json(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
expression: str,
expected_bodies: set[str],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=i + 1),
resources={"service.name": "api"},
body=body,
)
for i, body in enumerate(BODIES)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
build_raw_query(
"A",
"logs",
filter_expression=expression,
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
limit=100,
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["status"] == "success"
# body_v2 comes back parsed; a plain-string body is {"message": <body>}.
assert {row["data"]["body"]["message"] for row in get_rows(response)} == expected_bodies

View File

@@ -0,0 +1,87 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import build_order_by, build_raw_query, get_rows, make_query_request
SLASH = "GET /api/v1/users"
SUPERSTRING = "GET /api/v1/users/42"
QUOTE = 'say "hi" now'
BACKSLASH = "C:\\tmp\\log"
LIKE_META = "100% _off"
NON_ASCII = "Mixed CASE Ünïcode"
TAB = "tab\there"
CTRL = "ctrl\x01here"
BODIES = [SLASH, SUPERSTRING, QUOTE, BACKSLASH, LIKE_META, NON_ASCII, TAB, CTRL]
# CONTAINS and LIKE on body_v2 carry a needle over lower(toString(body_v2)), built by
# ClickHouse's own toJSONString so it matches the escapes the column stores — a term holding
# `/` or `"` finds nothing without them. The needle only ever narrows, never widens, so a
# term that should miss must still miss.
@pytest.mark.parametrize(
"expression,expected_bodies",
[
pytest.param("body CONTAINS '/api/v1'", {SLASH, SUPERSTRING}, id="contains_slash"),
pytest.param("body CONTAINS '\"hi\"'", {QUOTE}, id="contains_quote"),
pytest.param(r"body CONTAINS 'C:\\tmp'", {BACKSLASH}, id="contains_backslash"),
pytest.param("body CONTAINS 'CASE Ünïcode'", {NON_ASCII}, id="contains_non_ascii"),
pytest.param("body CONTAINS 'here'", {TAB, CTRL}, id="contains_beside_control_chars"),
pytest.param("body CONTAINS 'nothing here'", set(), id="contains_no_match"),
# a run shorter than the ngram size gets no needle, so the match alone decides
pytest.param("body CONTAINS 'ab'", {TAB}, id="contains_run_below_ngram_size"),
# a `\%` the user wrote has to stay a literal `%` through unescape and re-escape
pytest.param(r"body LIKE '%100\% \_off%'", {LIKE_META}, id="like_user_escaped_wildcards"),
pytest.param(r"body LIKE '%100\%\_off%'", set(), id="like_escaped_wildcards_are_not_wildcards"),
pytest.param("body LIKE 'GET /api/v1/users%'", {SLASH, SUPERSTRING}, id="like_trailing_wildcard"),
pytest.param("body LIKE '%CASE Ünïcode'", {NON_ASCII}, id="like_leading_wildcard"),
],
)
def test_logs_body_contains_json(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
expression: str,
expected_bodies: set[str],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=i + 1),
resources={"service.name": "api"},
body=body,
)
for i, body in enumerate(BODIES)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
build_raw_query(
"A",
"logs",
filter_expression=expression,
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
limit=100,
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["status"] == "success"
# body_v2 comes back parsed; a plain-string body is {"message": <body>}.
assert {row["data"]["body"]["message"] for row in get_rows(response)} == expected_bodies

View File

@@ -3,11 +3,13 @@ from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import build_order_by, build_raw_query, get_rows, make_query_request
def test_logs_json_body_simple_searches(
@@ -911,3 +913,61 @@ def test_logs_json_body_listing(
assert len(results) == 1
count = results[0]["data"][0][0]
assert count == 4 # 4 logs have status="success"
@pytest.mark.parametrize(
"expression,expected_services",
[
pytest.param("body.service IN ['auth', 'payment']", {"auth", "payment"}, id="in_scalar_path"),
pytest.param("body.status IN [200, 500]", {"auth", "payment"}, id="in_number_path"),
pytest.param("body.service NOT IN ['auth']", {"payment", "search"}, id="not_in_scalar_path"),
# An `[]` path is extracted as an array. Comparing that array to each scalar in the
# list is something ClickHouse rejects outright (code 130), so this shape used to
# fail the whole query; per-value extraction reads the first element instead.
pytest.param("body.user_names[*] IN ['alpha', 'gamma']", {"auth", "payment"}, id="in_array_path"),
],
)
def test_logs_json_body_in_operator(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
expression: str,
expected_services: set[str],
) -> None:
"""IN over a body JSON path fans out to one comparison per value."""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
specs = [("auth", 200, ["alpha", "beta"]), ("payment", 500, ["gamma"]), ("search", 404, ["beta", "alpha"])]
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=i + 1),
resources={"service.name": "api"},
body=json.dumps({"service": service, "status": status, "user_names": user_names}),
)
for i, (service, status, user_names) in enumerate(specs)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
build_raw_query(
"A",
"logs",
filter_expression=expression,
order=[build_order_by("timestamp", "desc")],
limit=100,
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["status"] == "success"
# flag off: the body comes back as the raw JSON string
assert {json.loads(row["data"]["body"])["service"] for row in get_rows(response)} == expected_services

View File

@@ -0,0 +1,92 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import build_order_by, build_raw_query, get_column_data_from_response, make_query_request
LOWER = "alpha"
UPPER = "ALPHA"
PLAIN = "beta"
NON_ASCII = "Mixed CASE Ünïcode"
SLASH = "GET /api/v1/users"
SUPERSTRING = "GET /api/v1/users/42"
QUOTE = 'say "hi" now'
BACKSLASH = "C:\\tmp\\log"
LIKE_META = "100% _off"
TAB = "tab\there"
CTRL = "ctrl\x01here"
BODIES = [LOWER, UPPER, PLAIN, NON_ASCII, SLASH, SUPERSTRING, QUOTE, BACKSLASH, LIKE_META, TAB, CTRL]
# `body = ?` carries a case-insensitive LOWER(body) companion for the bloom filters, so a
# body differing only in case must still not come back.
@pytest.mark.parametrize(
"expression,expected_bodies",
[
pytest.param(f"body = '{LOWER}'", {LOWER}, id="equality_exact"),
pytest.param(f"body = '{UPPER}'", {UPPER}, id="equality_other_case"),
pytest.param("body = 'Alpha'", set(), id="equality_case_must_match"),
pytest.param(f"body = '{NON_ASCII}'", {NON_ASCII}, id="equality_non_ascii"),
pytest.param("body = ''", set(), id="equality_empty"),
pytest.param("body = 'gamma'", set(), id="equality_no_match"),
# the companion is a LIKE-free equality, so none of these are metacharacters to it
pytest.param(f"body = '{SLASH}'", {SLASH}, id="equality_slash"),
pytest.param("body = 'say \"hi\" now'", {QUOTE}, id="equality_quote"),
pytest.param(r"body = 'C:\\tmp\\log'", {BACKSLASH}, id="equality_backslash"),
pytest.param(f"body = '{LIKE_META}'", {LIKE_META}, id="equality_like_metacharacters"),
pytest.param("body = 'tab\there'", {TAB}, id="equality_tab"),
pytest.param("body = 'ctrl\x01here'", {CTRL}, id="equality_control_char"),
# a prefix of another body must not match it
pytest.param("body = 'GET /api/v1'", set(), id="equality_prefix_does_not_match"),
pytest.param(f"body IN ('{LOWER}', '{PLAIN}')", {LOWER, PLAIN}, id="in_excludes_other_case"),
pytest.param(f"body IN ('{SLASH}', '{LIKE_META}')", {SLASH, LIKE_META}, id="in_escaped_values"),
pytest.param(f"body NOT IN ('{LOWER}', '{UPPER}')", set(BODIES) - {LOWER, UPPER}, id="not_in"),
],
)
def test_logs_body_equality(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
expression: str,
expected_bodies: set[str],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=i + 1),
resources={"service.name": "api"},
body=body,
)
for i, body in enumerate(BODIES)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
build_raw_query(
"A",
"logs",
filter_expression=expression,
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
limit=100,
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["status"] == "success"
assert set(get_column_data_from_response(response.json(), "body")) == expected_bodies