Compare commits

...

2 Commits

Author SHA1 Message Date
Tushar Vats
10e8c19b47 perf(traces): let attribute filters use the mapValues bloom filters
signoz_index_v3 carries the same value indexes as logs_v2 - an ngram filter
over mapValues(attributes_string) and a plain bloom filter over
mapValues(attributes_number), the exact shape has() asserts - while the
subscript a filter reads matches neither. The same two companions apply:

- a numeric equality carries the membership it implies, the paired
  mapContains making it hold for the zero an absent key reads;
- a case-insensitive match carries the raw-value LIKE for a pattern
  holding no ASCII letter, LOWER being the identity on those bytes.

The gate is the rendered expression rather than the resolved column:
traces render promoted attributes as materialized columns and semconv
families as one multiIf, and neither reads the map subscript the
mapValues indexes serve - only a plain attributes_*['key'] read does.
It is captured before the collision handler wraps the subscript in
toFloat64, which would defeat the check.
2026-09-01 03:51:08 +05:30
Tushar Vats
4d5992bd4d perf(logs): let attribute filters use the mapValues bloom filters
attributes_string and attributes_number each carry a bloom filter over mapValues, but the
subscript a filter reads matches no index expression, so a filter on a key every row carries -
a status code, a client address - reads every granule. The mapKeys index prunes only when the
key itself is rare. Both cases below AND in a predicate over mapValues that the original filter
already implies. On 1M rows, 1000001 rows read down to 8192, returning the same rows.

- A numeric equality carries the membership it implies. conditionForKey pairs a positive
  comparison with mapContains, so the key is present and the value compared is one of the map's
  own - which is what makes the assertion hold for the zero a subscript returns for an absent
  key. A non-numeric value means the collision handler compared the column as text, where an
  Array(Float64) membership check has no supertype, so it carries no predicate.
- A case-insensitive match reaches the raw-valued index only for a pattern holding no ASCII
  letter, LOWER being the identity on those bytes; a letter breaks the implication, since an `a`
  in the pattern may have come from an `A` in the value. That covers the values case folding
  cannot alter: addresses, timestamps, ports, numeric ids.

Both read a column the query already reads - a Map subscript decompresses keys and values
either way - so when the filter does not prune, bytes read are unchanged.
2026-09-01 03:23:39 +05:30
8 changed files with 508 additions and 78 deletions

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"regexp"
"strings"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
@@ -74,6 +75,35 @@ func (c *conditionBuilder) conditionForSearch(
return []string{sb.Or(conditions...)}, nil, nil
}
// numberAttributeIndexPredicate returns what an equality on a numeric attribute implies over
// mapValues(attributes_number), which its bloom filter indexes while the subscript the comparison
// reads matches nothing. The paired mapContains is what makes membership hold for the zero default.
func numberAttributeIndexPredicate(columns []*schema.Column, value any, sb *sqlbuilder.SelectBuilder) string {
if len(columns) != 1 || columns[0].Name != LogsV2AttributesNumberColumn {
return ""
}
// a non-numeric value means the collision handler compared the column as text, where an
// Array(Float64) membership check has no supertype
switch value.(type) {
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
return fmt.Sprintf("has(mapValues(%s), %s)", LogsV2AttributesNumberColumn, sb.Var(value))
}
return ""
}
// stringAttributeIndexPredicate returns the raw-value match a case-insensitive one implies when
// the pattern holds no ASCII letter, LOWER being the identity on those bytes. A letter breaks it:
// an `a` in the pattern may have come from an `A` in the value.
func stringAttributeIndexPredicate(columns []*schema.Column, fieldExpression, pattern string, sb *sqlbuilder.SelectBuilder) string {
if len(columns) != 1 || columns[0].Name != LogsV2AttributesStringColumn {
return ""
}
if strings.ContainsFunc(pattern, func(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') }) {
return ""
}
return sb.Like(fieldExpression, pattern)
}
// isBodyJSONSearch reports whether a key addresses a path within the body JSON. Only
// an explicit Body context qualifies; a bare, context-less `body` (e.g. full-text
// `count_distinct(body)` or `body EXISTS`) is a full-text match, not a `$.body` path.
@@ -333,6 +363,9 @@ func (c *conditionBuilder) conditionForResolvedKey(
switch operator {
// regular operators
case qbtypes.FilterOperatorEqual:
if predicate := numberAttributeIndexPredicate(columns, value, sb); predicate != "" {
return sb.And(sb.E(fieldExpression, value), predicate), nil
}
return sb.E(fieldExpression, value), nil
case qbtypes.FilterOperatorNotEqual:
return sb.NE(fieldExpression, value), nil
@@ -351,6 +384,11 @@ func (c *conditionBuilder) conditionForResolvedKey(
case qbtypes.FilterOperatorNotLike:
return sb.NotLike(fieldExpression, value), nil
case qbtypes.FilterOperatorILike:
if pattern, ok := value.(string); ok {
if predicate := stringAttributeIndexPredicate(columns, fieldExpression, pattern, sb); predicate != "" {
return sb.And(sb.ILike(fieldExpression, pattern), predicate), nil
}
}
return sb.ILike(fieldExpression, value), nil
case qbtypes.FilterOperatorNotILike:
return sb.NotILike(fieldExpression, value), nil
@@ -369,7 +407,13 @@ func (c *conditionBuilder) conditionForResolvedKey(
return sqlbuilder.Escape(pred), nil
case qbtypes.FilterOperatorContains:
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
// The map value indexes are over raw mapValues, which a case-insensitive match reaches only
// for the patterns stringAttributeIndexPredicate can assert the raw value from.
pattern := fmt.Sprintf("%%%s%%", value)
if predicate := stringAttributeIndexPredicate(columns, fieldExpression, pattern, sb); predicate != "" {
return sb.And(sb.ILike(fieldExpression, pattern), predicate), nil
}
return sb.ILike(fieldExpression, pattern), nil
case qbtypes.FilterOperatorNotContains:
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil

View File

@@ -258,8 +258,8 @@ func TestConditionFor(t *testing.T) {
},
operator: qbtypes.FilterOperatorContains,
value: 521509198310,
expectedSQL: "LOWER(attributes_string['user.id']) LIKE LOWER(?)",
expectedArgs: []any{"%521509198310%"},
expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND attributes_string['user.id'] LIKE ?)",
expectedArgs: []any{"%521509198310%", "%521509198310%"},
expectedError: nil,
},
{

View File

@@ -495,32 +495,32 @@ func TestFilterExprLogs(t *testing.T) {
category: "FREETEXT with parentheses",
query: "error (status.code=500 OR status.code=503)",
shouldPass: true,
expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')))))",
expectedArgs: []any{"error", float64(500), float64(503)},
expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND ((((toFloat64(attributes_number['status.code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status.code')) OR ((toFloat64(attributes_number['status.code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status.code')))))",
expectedArgs: []any{"error", float64(500), float64(500), float64(503), float64(503)},
expectedErrorContains: "",
},
{
category: "FREETEXT with parentheses",
query: "(status.code=500 OR status.code=503) error",
shouldPass: true,
expectedQuery: "WHERE ((((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')))) AND match(LOWER(body), LOWER(?)))",
expectedArgs: []any{float64(500), float64(503), "error"},
expectedQuery: "WHERE (((((toFloat64(attributes_number['status.code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status.code')) OR ((toFloat64(attributes_number['status.code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status.code')))) AND match(LOWER(body), LOWER(?)))",
expectedArgs: []any{float64(500), float64(500), float64(503), float64(503), "error"},
expectedErrorContains: "",
},
{
category: "FREETEXT with parentheses",
query: "error AND (status.code=500 OR status.code=503)",
shouldPass: true,
expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')))))",
expectedArgs: []any{"error", float64(500), float64(503)},
expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND ((((toFloat64(attributes_number['status.code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status.code')) OR ((toFloat64(attributes_number['status.code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status.code')))))",
expectedArgs: []any{"error", float64(500), float64(500), float64(503), float64(503)},
expectedErrorContains: "",
},
{
category: "FREETEXT with parentheses",
query: "(status.code=500 OR status.code=503) AND error",
shouldPass: true,
expectedQuery: "WHERE ((((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')))) AND match(LOWER(body), LOWER(?)))",
expectedArgs: []any{float64(500), float64(503), "error"},
expectedQuery: "WHERE (((((toFloat64(attributes_number['status.code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status.code')) OR ((toFloat64(attributes_number['status.code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status.code')))) AND match(LOWER(body), LOWER(?)))",
expectedArgs: []any{float64(500), float64(500), float64(503), float64(503), "error"},
expectedErrorContains: "",
},
@@ -827,16 +827,16 @@ func TestFilterExprLogs(t *testing.T) {
category: "Basic equality",
query: "status=200",
shouldPass: true,
expectedQuery: "WHERE (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))",
expectedArgs: []any{float64(200)},
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status'))",
expectedArgs: []any{float64(200), float64(200)},
expectedErrorContains: "",
},
{
category: "Basic equality",
query: "code=400",
shouldPass: true,
expectedQuery: "WHERE (toFloat64(attributes_number['code']) = ? AND mapContains(attributes_number, 'code'))",
expectedArgs: []any{float64(400)},
expectedQuery: "WHERE ((toFloat64(attributes_number['code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'code'))",
expectedArgs: []any{float64(400), float64(400)},
expectedErrorContains: "",
},
{
@@ -867,8 +867,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Basic equality",
query: "count=0",
shouldPass: true,
expectedQuery: "WHERE (toFloat64(attributes_number['count']) = ? AND mapContains(attributes_number, 'count'))",
expectedArgs: []any{float64(0)},
expectedQuery: "WHERE ((toFloat64(attributes_number['count']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'count'))",
expectedArgs: []any{float64(0), float64(0)},
expectedErrorContains: "",
},
{
@@ -1187,16 +1187,16 @@ func TestFilterExprLogs(t *testing.T) {
category: "IN operator (parentheses)",
query: "status IN (200, 201, 202)",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status'))",
expectedArgs: []any{float64(200), float64(201), float64(202)},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) OR (toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) OR (toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?))) AND mapContains(attributes_number, 'status'))",
expectedArgs: []any{float64(200), float64(200), float64(201), float64(201), float64(202), float64(202)},
expectedErrorContains: "",
},
{
category: "IN operator (parentheses)",
query: "error.code IN (404, 500, 503)",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ?) AND mapContains(attributes_number, 'error.code'))",
expectedArgs: []any{float64(404), float64(500), float64(503)},
expectedQuery: "WHERE (((toFloat64(attributes_number['error.code']) = ? AND has(mapValues(attributes_number), ?)) OR (toFloat64(attributes_number['error.code']) = ? AND has(mapValues(attributes_number), ?)) OR (toFloat64(attributes_number['error.code']) = ? AND has(mapValues(attributes_number), ?))) AND mapContains(attributes_number, 'error.code'))",
expectedArgs: []any{float64(404), float64(404), float64(500), float64(500), float64(503), float64(503)},
expectedErrorContains: "",
},
{
@@ -1221,16 +1221,16 @@ func TestFilterExprLogs(t *testing.T) {
category: "IN operator (brackets)",
query: "status IN [200, 201, 202]",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status'))",
expectedArgs: []any{float64(200), float64(201), float64(202)},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) OR (toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) OR (toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?))) AND mapContains(attributes_number, 'status'))",
expectedArgs: []any{float64(200), float64(200), float64(201), float64(201), float64(202), float64(202)},
expectedErrorContains: "",
},
{
category: "IN operator (brackets)",
query: "error.code IN [404, 500, 503]",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ?) AND mapContains(attributes_number, 'error.code'))",
expectedArgs: []any{float64(404), float64(500), float64(503)},
expectedQuery: "WHERE (((toFloat64(attributes_number['error.code']) = ? AND has(mapValues(attributes_number), ?)) OR (toFloat64(attributes_number['error.code']) = ? AND has(mapValues(attributes_number), ?)) OR (toFloat64(attributes_number['error.code']) = ? AND has(mapValues(attributes_number), ?))) AND mapContains(attributes_number, 'error.code'))",
expectedArgs: []any{float64(404), float64(404), float64(500), float64(500), float64(503), float64(503)},
expectedErrorContains: "",
},
{
@@ -1609,8 +1609,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Explicit AND",
query: "status=200 AND service.name=\"api\"",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), "api"},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), float64(200), "api"},
expectedErrorContains: "",
},
{
@@ -1635,8 +1635,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Explicit OR",
query: "status=200 OR status=201",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))",
expectedArgs: []any{float64(200), float64(201)},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) OR ((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')))",
expectedArgs: []any{float64(200), float64(200), float64(201), float64(201)},
expectedErrorContains: "",
},
{
@@ -1661,8 +1661,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "NOT with expressions",
query: "NOT status=200",
shouldPass: true,
expectedQuery: "WHERE NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))",
expectedArgs: []any{float64(200)},
expectedQuery: "WHERE NOT (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')))",
expectedArgs: []any{float64(200), float64(200)},
expectedErrorContains: "",
},
{
@@ -1687,8 +1687,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "AND + OR combinations",
query: "status=200 AND (service.name=\"api\" OR service.name=\"web\")",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))))",
expectedArgs: []any{float64(200), "api", "web"},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))))",
expectedArgs: []any{float64(200), float64(200), "api", "web"},
expectedErrorContains: "",
},
{
@@ -1713,8 +1713,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "AND + NOT combinations",
query: "status=200 AND NOT service.name=\"api\"",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))",
expectedArgs: []any{float64(200), "api"},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))",
expectedArgs: []any{float64(200), float64(200), "api"},
expectedErrorContains: "",
},
{
@@ -1731,8 +1731,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "OR + NOT combinations",
query: "NOT status=200 OR NOT service.name=\"api\"",
shouldPass: true,
expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))) OR NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))",
expectedArgs: []any{float64(200), "api"},
expectedQuery: "WHERE (NOT (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status'))) OR NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))",
expectedArgs: []any{float64(200), float64(200), "api"},
expectedErrorContains: "",
},
{
@@ -1749,8 +1749,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "AND + OR + NOT combinations",
query: "status=200 AND (service.name=\"api\" OR NOT duration>1000)",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR NOT ((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration'))))))",
expectedArgs: []any{float64(200), "api", float64(1000)},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR NOT ((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration'))))))",
expectedArgs: []any{float64(200), float64(200), "api", float64(1000)},
expectedErrorContains: "",
},
{
@@ -1765,8 +1765,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "AND + OR + NOT combinations",
query: "NOT (status=200 AND service.name=\"api\") OR count>0",
shouldPass: true,
expectedQuery: "WHERE (NOT ((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))) OR (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')))",
expectedArgs: []any{float64(200), "api", float64(0)},
expectedQuery: "WHERE (NOT (((((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))) OR (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')))",
expectedArgs: []any{float64(200), float64(200), "api", float64(0)},
expectedErrorContains: "",
},
@@ -1775,8 +1775,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Implicit AND",
query: "status=200 service.name=\"api\"",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), "api"},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), float64(200), "api"},
expectedErrorContains: "",
},
{
@@ -1801,8 +1801,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Mixed implicit/explicit AND",
query: "status=200 AND service.name=\"api\" duration<1000",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')))",
expectedArgs: []any{float64(200), "api", float64(1000)},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')))",
expectedArgs: []any{float64(200), float64(200), "api", float64(1000)},
expectedErrorContains: "",
},
{
@@ -1819,8 +1819,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Simple grouping",
query: "(status=200)",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))",
expectedArgs: []any{float64(200)},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')))",
expectedArgs: []any{float64(200), float64(200)},
expectedErrorContains: "",
},
{
@@ -1845,8 +1845,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Nested grouping",
query: "((status=200))",
shouldPass: true,
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))))",
expectedArgs: []any{float64(200)},
expectedQuery: "WHERE ((((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status'))))",
expectedArgs: []any{float64(200), float64(200)},
expectedErrorContains: "",
},
{
@@ -1871,8 +1871,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Complex nested grouping",
query: "(status=200 AND (service.name=\"api\" OR service.name=\"web\"))",
shouldPass: true,
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))",
expectedArgs: []any{float64(200), "api", "web"},
expectedQuery: "WHERE ((((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))",
expectedArgs: []any{float64(200), float64(200), "api", "web"},
expectedErrorContains: "",
},
{
@@ -1897,8 +1897,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Deep nesting",
query: "(((status=200 OR status=201) AND service.name=\"api\") OR ((status=202 OR status=203) AND service.name=\"web\"))",
shouldPass: true,
expectedQuery: "WHERE (((((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))) OR (((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))",
expectedArgs: []any{float64(200), float64(201), "api", float64(202), float64(203), "web"},
expectedQuery: "WHERE ((((((((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) OR ((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))) OR ((((((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) OR ((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))",
expectedArgs: []any{float64(200), float64(200), float64(201), float64(201), "api", float64(202), float64(202), float64(203), float64(203), "web"},
expectedErrorContains: "",
},
{
@@ -1949,32 +1949,32 @@ func TestFilterExprLogs(t *testing.T) {
category: "Numeric values",
query: "status=200",
shouldPass: true,
expectedQuery: "WHERE (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))",
expectedArgs: []any{float64(200)},
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status'))",
expectedArgs: []any{float64(200), float64(200)},
expectedErrorContains: "",
},
{
category: "Numeric values",
query: "count=0",
shouldPass: true,
expectedQuery: "WHERE (toFloat64(attributes_number['count']) = ? AND mapContains(attributes_number, 'count'))",
expectedArgs: []any{float64(0)},
expectedQuery: "WHERE ((toFloat64(attributes_number['count']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'count'))",
expectedArgs: []any{float64(0), float64(0)},
expectedErrorContains: "",
},
{
category: "Numeric values",
query: "duration=1000.5",
shouldPass: true,
expectedQuery: "WHERE (toFloat64(attributes_number['duration']) = ? AND mapContains(attributes_number, 'duration'))",
expectedArgs: []any{float64(1000.5)},
expectedQuery: "WHERE ((toFloat64(attributes_number['duration']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'duration'))",
expectedArgs: []any{float64(1000.5), float64(1000.5)},
expectedErrorContains: "",
},
{
category: "Numeric values",
query: "amount=-10.25",
shouldPass: true,
expectedQuery: "WHERE (toFloat64(attributes_number['amount']) = ? AND mapContains(attributes_number, 'amount'))",
expectedArgs: []any{float64(-10.25)},
expectedQuery: "WHERE ((toFloat64(attributes_number['amount']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'amount'))",
expectedArgs: []any{float64(-10.25), float64(-10.25)},
expectedErrorContains: "",
},
@@ -2052,8 +2052,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Nested object paths",
query: "response.body.data.items[].id=123",
shouldPass: true,
expectedQuery: `WHERE ((toFloat64(attributes_number['response.body.data.items[].id']) = ? AND mapContains(attributes_number, 'response.body.data.items[].id')) OR (JSONExtract(JSON_VALUE(body, '$."response"."body"."data"."items"[*]."id"'), 'Float64') = ? AND JSON_EXISTS(body, '$."response"."body"."data"."items"[*]."id"')))`,
expectedArgs: []any{float64(123), float64(123)},
expectedQuery: `WHERE (((toFloat64(attributes_number['response.body.data.items[].id']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'response.body.data.items[].id')) OR (JSONExtract(JSON_VALUE(body, '$."response"."body"."data"."items"[*]."id"'), 'Float64') = ? AND JSON_EXISTS(body, '$."response"."body"."data"."items"[*]."id"')))`,
expectedArgs: []any{float64(123), float64(123), float64(123)},
expectedErrorContains: "",
},
{
@@ -2083,29 +2083,29 @@ func TestFilterExprLogs(t *testing.T) {
category: "Operator precedence",
query: "NOT status=200 AND service.name=\"api\"",
shouldPass: true,
expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), "api"}, // Should be (NOT status=200) AND service.name="api"
expectedQuery: "WHERE (NOT (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status'))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), float64(200), "api"}, // Should be (NOT status=200) AND service.name="api"
},
{
category: "Operator precedence",
query: "status=200 AND service.name=\"api\" OR service.name=\"web\"",
shouldPass: true,
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), "api", "web"}, // Should be (status=200 AND service.name="api") OR service.name="web"
expectedQuery: "WHERE ((((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), float64(200), "api", "web"}, // Should be (status=200 AND service.name="api") OR service.name="web"
},
{
category: "Operator precedence",
query: "NOT status=200 OR NOT service.name=\"api\"",
shouldPass: true,
expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))) OR NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))",
expectedArgs: []any{float64(200), "api"}, // Should be (NOT status=200) OR (NOT service.name="api")
expectedQuery: "WHERE (NOT (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status'))) OR NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))",
expectedArgs: []any{float64(200), float64(200), "api"}, // Should be (NOT status=200) OR (NOT service.name="api")
},
{
category: "Operator precedence",
query: "status=200 OR service.name=\"api\" AND level=\"ERROR\"",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) OR ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level'))))",
expectedArgs: []any{float64(200), "api", "ERROR"}, // Should be status=200 OR (service.name="api" AND level="ERROR")
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) OR ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level'))))",
expectedArgs: []any{float64(200), float64(200), "api", "ERROR"}, // Should be status=200 OR (service.name="api" AND level="ERROR")
},
// Different whitespace patterns
@@ -2129,8 +2129,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Whitespace patterns",
query: "status=200 AND service.name=\"api\"",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), "api"}, // Multiple spaces
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), float64(200), "api"}, // Multiple spaces
},
// More Unicode characters
@@ -2365,8 +2365,8 @@ func TestFilterExprLogs(t *testing.T) {
category: "Unusual whitespace",
query: "status = 200 AND service.name = \"api\"",
shouldPass: true,
expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), "api"},
expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))",
expectedArgs: []any{float64(200), float64(200), "api"},
},
{
category: "Unusual whitespace",
@@ -2426,9 +2426,9 @@ func TestFilterExprLogs(t *testing.T) {
)
`,
shouldPass: true,
expectedQuery: "WHERE ((((((((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status')) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status')))) OR (((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status')) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status')) AND NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))))))) AND ((((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? OR multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? OR multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ?) AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (((multiIf(resource.`service.type` IS NOT NULL, resource.`service.type`::String, mapContains(resources_string, 'service.type'), resources_string['service.type'], NULL) = ? AND multiIf(resource.`service.type` IS NOT NULL, resource.`service.type`::String, mapContains(resources_string, 'service.type'), resources_string['service.type'], NULL) IS NOT NULL) AND NOT ((multiIf(resource.`service.deprecated` IS NOT NULL, resource.`service.deprecated`::String, mapContains(resources_string, 'service.deprecated'), resources_string['service.deprecated'], NULL) = ? AND multiIf(resource.`service.deprecated` IS NOT NULL, resource.`service.deprecated`::String, mapContains(resources_string, 'service.deprecated'), resources_string['service.deprecated'], NULL) IS NOT NULL)))))))) AND (((((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')) OR ((toFloat64(attributes_number['duration']) BETWEEN ? AND ? AND mapContains(attributes_number, 'duration'))))) AND ((multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) <> ? OR (((multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) = ? AND multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) IS NOT NULL) AND (attributes_bool['is_automated_test'] = ? AND mapContains(attributes_bool, 'is_automated_test')))))))) AND NOT ((((((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')) OR (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')))) AND (attributes_string['severity'] = ? AND mapContains(attributes_string, 'severity'))))))",
expectedQuery: "WHERE ((((((((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status')) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status')))) OR (((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status')) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status')) AND NOT (((toFloat64(attributes_number['status']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'status'))))))) AND ((((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? OR multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? OR multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ?) AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (((multiIf(resource.`service.type` IS NOT NULL, resource.`service.type`::String, mapContains(resources_string, 'service.type'), resources_string['service.type'], NULL) = ? AND multiIf(resource.`service.type` IS NOT NULL, resource.`service.type`::String, mapContains(resources_string, 'service.type'), resources_string['service.type'], NULL) IS NOT NULL) AND NOT ((multiIf(resource.`service.deprecated` IS NOT NULL, resource.`service.deprecated`::String, mapContains(resources_string, 'service.deprecated'), resources_string['service.deprecated'], NULL) = ? AND multiIf(resource.`service.deprecated` IS NOT NULL, resource.`service.deprecated`::String, mapContains(resources_string, 'service.deprecated'), resources_string['service.deprecated'], NULL) IS NOT NULL)))))))) AND (((((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')) OR ((toFloat64(attributes_number['duration']) BETWEEN ? AND ? AND mapContains(attributes_number, 'duration'))))) AND ((multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) <> ? OR (((multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) = ? AND multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) IS NOT NULL) AND (attributes_bool['is_automated_test'] = ? AND mapContains(attributes_bool, 'is_automated_test')))))))) AND NOT ((((((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')) OR (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')))) AND (attributes_string['severity'] = ? AND mapContains(attributes_string, 'severity'))))))",
expectedArgs: []any{
float64(200), float64(300), float64(400), float64(500), float64(404),
float64(200), float64(300), float64(400), float64(500), float64(404), float64(404),
"api", "web", "auth",
"internal", true,
float64(1000), float64(1000), float64(5000),

View File

@@ -31,6 +31,37 @@ func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionB
return &conditionBuilder{fm: fm, fl: fl}
}
// numberAttributeIndexPredicate returns what an equality on a numeric attribute implies over
// mapValues(attributes_number), which its bloom filter indexes while the subscript the comparison
// reads matches nothing. The paired mapContains is what makes membership hold for the zero default.
// mapExpression gates it to a plain subscript read — a materialized column or a family multiIf
// carries nothing.
func numberAttributeIndexPredicate(mapExpression string, value any, sb *sqlbuilder.SelectBuilder) string {
if !strings.HasPrefix(mapExpression, SpanAttributesNumberColumn+"['") {
return ""
}
// a non-numeric value means the collision handler compared the column as text, where an
// Array(Float64) membership check has no supertype
switch value.(type) {
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
return fmt.Sprintf("has(mapValues(%s), %s)", SpanAttributesNumberColumn, sb.Var(value))
}
return ""
}
// stringAttributeIndexPredicate returns the raw-value match a case-insensitive one implies when
// the pattern holds no ASCII letter, LOWER being the identity on those bytes. A letter breaks it:
// an `a` in the pattern may have come from an `A` in the value.
func stringAttributeIndexPredicate(mapExpression, pattern string, sb *sqlbuilder.SelectBuilder) string {
if !strings.HasPrefix(mapExpression, SpanAttributesStringColumn+"['") {
return ""
}
if strings.ContainsFunc(pattern, func(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') }) {
return ""
}
return sb.Like(mapExpression, pattern)
}
func (c *conditionBuilder) conditionFor(
ctx context.Context,
orgID valuer.UUID,
@@ -69,6 +100,9 @@ func (c *conditionBuilder) conditionFor(
}
}
// the raw map read, before the collision handler wraps the subscript
mapExpression := fieldExpression
// Coercion switches only on the data type, which every member shares, so
// the first member stands in for the field.
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(logical.Single(), value, fieldExpression, operator)
@@ -77,6 +111,9 @@ func (c *conditionBuilder) conditionFor(
switch operator {
// regular operators
case qbtypes.FilterOperatorEqual:
if predicate := numberAttributeIndexPredicate(mapExpression, value, sb); predicate != "" {
return sb.And(sb.E(fieldExpression, value), predicate), nil
}
return sb.E(fieldExpression, value), nil
case qbtypes.FilterOperatorNotEqual:
return sb.NE(fieldExpression, value), nil
@@ -95,12 +132,23 @@ func (c *conditionBuilder) conditionFor(
case qbtypes.FilterOperatorNotLike:
return sb.NotLike(fieldExpression, value), nil
case qbtypes.FilterOperatorILike:
if pattern, ok := value.(string); ok {
if predicate := stringAttributeIndexPredicate(mapExpression, pattern, sb); predicate != "" {
return sb.And(sb.ILike(fieldExpression, pattern), predicate), nil
}
}
return sb.ILike(fieldExpression, value), nil
case qbtypes.FilterOperatorNotILike:
return sb.NotILike(fieldExpression, value), nil
case qbtypes.FilterOperatorContains:
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
// The map value indexes are over raw mapValues, which a case-insensitive match reaches only
// for the patterns stringAttributeIndexPredicate can assert the raw value from.
pattern := fmt.Sprintf("%%%s%%", value)
if predicate := stringAttributeIndexPredicate(mapExpression, pattern, sb); predicate != "" {
return sb.And(sb.ILike(fieldExpression, pattern), predicate), nil
}
return sb.ILike(fieldExpression, pattern), nil
case qbtypes.FilterOperatorNotContains:
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil

View File

@@ -65,6 +65,32 @@ func TestConditionFor(t *testing.T) {
expectedArgs: []any{float64(1024)},
expectedError: nil,
},
{
name: "Equal operator - number attribute carries the mapValues membership",
key: telemetrytypes.TelemetryFieldKey{
Name: "resp_code",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeNumber,
},
operator: qbtypes.FilterOperatorEqual,
value: float64(503),
expectedSQL: "((toFloat64(attributes_number['resp_code']) = ? AND has(mapValues(attributes_number), ?)) AND mapContains(attributes_number, 'resp_code'))",
expectedArgs: []any{float64(503), float64(503)},
expectedError: nil,
},
{
name: "Contains operator - letter-free value carries the raw-value match",
key: telemetrytypes.TelemetryFieldKey{
Name: "client.ip",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
operator: qbtypes.FilterOperatorContains,
value: "192.168.77",
expectedSQL: "((LOWER(attributes_string['client.ip']) LIKE LOWER(?) AND attributes_string['client.ip'] LIKE ?) AND mapContains(attributes_string, 'client.ip'))",
expectedArgs: []any{"%192.168.77%", "%192.168.77%"},
expectedError: nil,
},
{
name: "Greater Than Or Equal operator - timestamp",
key: telemetrytypes.TelemetryFieldKey{

View File

@@ -253,6 +253,27 @@ def get_preview_sql(response: requests.Response, name: str) -> str:
return statements[0]["db.statement.query"]
def get_preview_skip_indexes(response: requests.Response, name: str) -> dict[str, dict[str, Any]]:
"""The skip-index steps of the named query's read funnel, keyed by index name.
Needs a verbose preview. ClickHouse lists a skip index only when the predicate matches its
expression, so an absent entry means it was never consulted."""
statements = get_preview_statements(response, name)
assert len(statements) == 1, f"expected 1 statement for query {name}, got {len(statements)}"
granules = statements[0]["granules"]
assert granules is not None, f"query {name} reads no MergeTree table: {statements[0]}"
return {step["name"]: step for read in granules["reads"] for step in read["steps"] if step["type"] == "Skip"}
def get_preview_selected_granules(response: requests.Response, name: str) -> int:
"""Granules surviving every index step of the named query's read funnel."""
statements = get_preview_statements(response, name)
assert len(statements) == 1, f"expected 1 statement for query {name}, got {len(statements)}"
granules = statements[0]["granules"]
assert granules is not None, f"query {name} reads no MergeTree table: {statements[0]}"
return granules["selected"]
def aligned_epoch(ago: timedelta, step_seconds: int = DEFAULT_STEP_INTERVAL) -> int:
"""Epoch seconds for `now - ago`, floored to a step boundary so seeded
points land exactly on the query's toStartOfInterval buckets."""

View File

@@ -0,0 +1,135 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import (
build_order_by,
build_raw_query,
get_preview_selected_granules,
get_preview_skip_indexes,
make_preview_query_request,
)
# The attribute maps carry a bloom filter over mapValues, but the subscript a comparison reads
# matches no index expression. mapContains prunes on the key alone, so these use a key every row
# carries to isolate what the value predicate contributes.
ATTRIBUTE_NUMBER_VALUE_INDEX = "attributes_number_idx_val"
ATTRIBUTE_STRING_VALUE_INDEX = "attributes_string_idx_val"
@pytest.mark.parametrize(
"expression,prunes_every_granule",
[
pytest.param("attribute.resp_code = 503", False, id="value_present"),
pytest.param("attribute.resp_code = 60599", True, id="value_absent"),
# IN delegates to the equalities, so every arm carries its own membership assertion
pytest.param("attribute.resp_code IN (503, 200)", False, id="in_present_values"),
pytest.param("attribute.resp_code IN (60599, 60600)", True, id="in_absent_values"),
],
)
def test_attribute_number_equality_prunes_granules(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
expression: str,
prunes_every_granule: bool,
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs([Logs(timestamp=now - timedelta(seconds=i + 1), resources={"service.name": "api"}, attributes={"resp_code": code}) for i, code in enumerate([200, 200, 503])])
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_preview_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
build_raw_query(
"A",
"logs",
filter_expression=expression,
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
limit=100,
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
skip_indexes = get_preview_skip_indexes(response, "A")
assert ATTRIBUTE_NUMBER_VALUE_INDEX in skip_indexes, f"mapValues filter not consulted, only: {sorted(skip_indexes)}"
selected = get_preview_selected_granules(response, "A")
if prunes_every_granule:
assert selected == 0, f"expected every granule pruned, {selected} survived"
else:
assert selected > 0, "the granule holding the match must survive"
# The mapValues filter indexes the values raw, so a case-insensitive match reaches it only for a
# pattern holding no ASCII letter, where LOWER changes nothing. A letter leaves it unconsulted.
@pytest.mark.parametrize(
"expression,value_index_consulted,prunes_every_granule",
[
pytest.param("attribute.client.ip CONTAINS '192.168.77'", True, False, id="letter_free_value_present"),
pytest.param("attribute.client.ip CONTAINS '192.168.99'", True, True, id="letter_free_value_absent"),
pytest.param("attribute.env CONTAINS 'production'", False, False, id="letters_leave_it_unconsulted"),
# the filter is consulted for any letter-free pattern, but a run below the index ngram
# leaves it nothing to check
pytest.param("attribute.client.ip CONTAINS '.7'", True, False, id="run_shorter_than_the_ngram"),
],
)
def test_attribute_letter_free_match_prunes_granules(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
expression: str,
value_index_consulted: bool,
prunes_every_granule: bool,
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=i + 1),
resources={"service.name": "api"},
attributes={"client.ip": ip, "env": "production"},
)
for i, ip in enumerate(["10.0.0.1", "10.0.0.2", "192.168.77.31"])
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_preview_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
build_raw_query(
"A",
"logs",
filter_expression=expression,
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
limit=100,
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
skip_indexes = get_preview_skip_indexes(response, "A")
assert (ATTRIBUTE_STRING_VALUE_INDEX in skip_indexes) == value_index_consulted, f"consulted: {sorted(skip_indexes)}"
selected = get_preview_selected_granules(response, "A")
if prunes_every_granule:
assert selected == 0, f"expected every granule pruned, {selected} survived"
else:
assert selected > 0, "the granule holding the match must survive"

View File

@@ -0,0 +1,156 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import (
build_order_by,
build_raw_query,
get_preview_selected_granules,
get_preview_skip_indexes,
make_preview_query_request,
)
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
# The attribute maps carry a bloom filter over mapValues, but the subscript a comparison reads
# matches no index expression. mapContains prunes on the key alone, so these use a key every row
# carries to isolate what the value predicate contributes.
ATTRIBUTE_NUMBER_VALUE_INDEX = "attributes_number_idx_val"
ATTRIBUTE_STRING_VALUE_INDEX = "attributes_string_idx_val"
@pytest.mark.parametrize(
"expression,prunes_every_granule",
[
pytest.param("attribute.resp_code = 503", False, id="value_present"),
pytest.param("attribute.resp_code = 60599", True, id="value_absent"),
# IN delegates to the equalities, so every arm carries its own membership assertion
pytest.param("attribute.resp_code IN (503, 200)", False, id="in_present_values"),
pytest.param("attribute.resp_code IN (60599, 60600)", True, id="in_absent_values"),
],
)
def test_attribute_number_equality_prunes_granules(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
expression: str,
prunes_every_granule: bool,
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=i + 1),
duration=timedelta(milliseconds=100),
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
name="handler",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "api"},
attributes={"resp_code": code},
)
for i, code in enumerate([200, 200, 503])
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_preview_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
build_raw_query(
"A",
"traces",
filter_expression=expression,
order=[build_order_by("timestamp", "desc")],
limit=100,
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
skip_indexes = get_preview_skip_indexes(response, "A")
assert ATTRIBUTE_NUMBER_VALUE_INDEX in skip_indexes, f"mapValues filter not consulted, only: {sorted(skip_indexes)}"
selected = get_preview_selected_granules(response, "A")
if prunes_every_granule:
assert selected == 0, f"expected every granule pruned, {selected} survived"
else:
assert selected > 0, "the granule holding the match must survive"
# The mapValues filter indexes the values raw, so a case-insensitive match reaches it only for a
# pattern holding no ASCII letter, where LOWER changes nothing. A letter leaves it unconsulted.
@pytest.mark.parametrize(
"expression,value_index_consulted,prunes_every_granule",
[
pytest.param("attribute.client.ip CONTAINS '192.168.77'", True, False, id="letter_free_value_present"),
pytest.param("attribute.client.ip CONTAINS '192.168.99'", True, True, id="letter_free_value_absent"),
pytest.param("attribute.env CONTAINS 'production'", False, False, id="letters_leave_it_unconsulted"),
# the filter is consulted for any letter-free pattern, but a run below the index ngram
# leaves it nothing to check
pytest.param("attribute.client.ip CONTAINS '.7'", True, False, id="run_shorter_than_the_ngram"),
],
)
def test_attribute_letter_free_match_prunes_granules(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
expression: str,
value_index_consulted: bool,
prunes_every_granule: bool,
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=i + 1),
duration=timedelta(milliseconds=100),
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
name="handler",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "api"},
attributes={"client.ip": ip, "env": "production"},
)
for i, ip in enumerate(["10.0.0.1", "10.0.0.2", "192.168.77.31"])
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_preview_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
build_raw_query(
"A",
"traces",
filter_expression=expression,
order=[build_order_by("timestamp", "desc")],
limit=100,
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
skip_indexes = get_preview_skip_indexes(response, "A")
assert (ATTRIBUTE_STRING_VALUE_INDEX in skip_indexes) == value_index_consulted, f"consulted: {sorted(skip_indexes)}"
selected = get_preview_selected_granules(response, "A")
if prunes_every_granule:
assert selected == 0, f"expected every granule pruned, {selected} survived"
else:
assert selected > 0, "the granule holding the match must survive"