mirror of
https://github.com/SigNoz/signoz.git
synced 2026-09-16 08:20:41 +01:00
Compare commits
3 Commits
feat/sqlco
...
tvats-body
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7af25ded00 | ||
|
|
b09cfd4869 | ||
|
|
0d633701c4 |
@@ -197,8 +197,8 @@ func TestJSONStmtBuilder_PrimitivePaths(t *testing.T) {
|
||||
name: "message Contains 'Iron Award'",
|
||||
filter: "message Contains 'Iron Award'",
|
||||
expected: TestExpected{
|
||||
WhereClause: "LOWER(body_v2.message) LIKE LOWER(?)",
|
||||
Args: []any{"%Iron Award%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
WhereClause: "(LOWER(body_v2.message) LIKE LOWER(?) AND LOWER(toString(body_v2)) LIKE " + jsonSubstringNeedle + ")",
|
||||
Args: []any{"%Iron Award%", "Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
@@ -469,6 +469,8 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
// The `[*]` path is extracted per value, not as an Array(String) compared to a
|
||||
// scalar — ClickHouse rejects that outright (code 130).
|
||||
name: "IN operator with json search",
|
||||
requestType: qbtypes.RequestTypeRaw,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -479,7 +481,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
Limit: 10,
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(String)') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSON_VALUE(body, '$.\"user_names\"[*]') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Warnings: []string{querybuilder.NewKeyNotFoundWarning("user_names[*]")},
|
||||
},
|
||||
@@ -943,6 +945,9 @@ func TestAdjustKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The needle ClickHouse builds for the body_v2 index expression.
|
||||
const jsonSubstringNeedle = `concat('%', replaceAll(replaceAll(replaceAll(lower(substring(toJSONString(?), 2, length(toJSONString(?)) - 2)), '\\', '\\\\'), '%', '\\%'), '_', '\\_'), '%')`
|
||||
|
||||
func TestStmtBuilderBodyField(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -1009,8 +1014,8 @@ func TestStmtBuilderBodyField(t *testing.T) {
|
||||
},
|
||||
enableUseJSONBody: false,
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE body = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (body = ? AND LOWER(body) = LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"", "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
@@ -1024,8 +1029,8 @@ func TestStmtBuilderBodyField(t *testing.T) {
|
||||
},
|
||||
enableUseJSONBody: true,
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE LOWER(body_v2.message) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (LOWER(body_v2.message) LIKE LOWER(?) AND LOWER(toString(body_v2)) LIKE " + jsonSubstringNeedle + ") AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"%error%", "error", "error", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Warnings: []string{bodySearchDefaultWarning},
|
||||
},
|
||||
expectedErr: nil,
|
||||
|
||||
@@ -180,20 +180,16 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
if !ok {
|
||||
return "", qbtypes.ErrInValues
|
||||
}
|
||||
// each value carries its own index filter, since `=` derives one from the value
|
||||
inConditions := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
inConditions = append(inConditions, sb.E(fieldName, querybuilder.FormatValueForContains(v)))
|
||||
}
|
||||
mainCondition := sb.Or(inConditions...)
|
||||
valConditions := make([]string, 0, len(values))
|
||||
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
|
||||
for _, v := range valuesForIndexFilter {
|
||||
valConditions = append(valConditions, sb.Like(column.Name, v))
|
||||
cond, err := b.conditionForKey(ctx, startNs, endNs, key, qbtypes.FilterOperatorEqual, v, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
inConditions = append(inConditions, cond)
|
||||
}
|
||||
mainCondition = sb.And(mainCondition, keyIdxFilter, sb.Or(valConditions...))
|
||||
|
||||
return mainCondition, nil
|
||||
return sb.Or(inConditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
@@ -201,17 +197,13 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
}
|
||||
notInConditions := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
notInConditions = append(notInConditions, sb.NE(fieldName, querybuilder.FormatValueForContains(v)))
|
||||
}
|
||||
mainCondition := sb.And(notInConditions...)
|
||||
valConditions := make([]string, 0, len(values))
|
||||
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
|
||||
for _, v := range valuesForIndexFilter {
|
||||
valConditions = append(valConditions, sb.NotLike(column.Name, v))
|
||||
cond, err := b.conditionForKey(ctx, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, v, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
notInConditions = append(notInConditions, cond)
|
||||
}
|
||||
mainCondition = sb.And(mainCondition, sb.And(valConditions...))
|
||||
return mainCondition, nil
|
||||
return sb.And(notInConditions...), nil
|
||||
|
||||
case qbtypes.FilterOperatorExists:
|
||||
return sb.And(
|
||||
|
||||
@@ -109,8 +109,8 @@ func TestConditionBuilder(t *testing.T) {
|
||||
},
|
||||
op: qbtypes.FilterOperatorIn,
|
||||
value: []any{"watch", "redis"},
|
||||
expected: "(simpleJSONExtractString(labels, 'k8s.namespace.name') = ? OR simpleJSONExtractString(labels, 'k8s.namespace.name') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)",
|
||||
expectedArgs: []any{"watch", "redis", "%k8s.namespace.name%", "%k8s.namespace.name\":\"watch%", "%k8s.namespace.name\":\"redis%"},
|
||||
expected: "((simpleJSONExtractString(labels, 'k8s.namespace.name') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'k8s.namespace.name') = ? AND labels LIKE ? AND labels LIKE ?))",
|
||||
expectedArgs: []any{"watch", "%k8s.namespace.name%", "%k8s.namespace.name\":\"watch%", "redis", "%k8s.namespace.name%", "%k8s.namespace.name\":\"redis%"},
|
||||
},
|
||||
{
|
||||
name: "string_not_in",
|
||||
@@ -120,8 +120,8 @@ func TestConditionBuilder(t *testing.T) {
|
||||
},
|
||||
op: qbtypes.FilterOperatorNotIn,
|
||||
value: []any{"watch", "redis"},
|
||||
expected: "(simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND simpleJSONExtractString(labels, 'k8s.namespace.name') <> ?) AND (labels NOT LIKE ? AND labels NOT LIKE ?)",
|
||||
expectedArgs: []any{"watch", "redis", "%k8s.namespace.name\":\"watch%", "%k8s.namespace.name\":\"redis%"},
|
||||
expected: "((simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND labels NOT LIKE ?) AND (simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND labels NOT LIKE ?))",
|
||||
expectedArgs: []any{"watch", "%k8s.namespace.name\":\"watch%", "redis", "%k8s.namespace.name\":\"redis%"},
|
||||
},
|
||||
{
|
||||
name: "string_exists",
|
||||
@@ -173,8 +173,8 @@ func TestConditionBuilder(t *testing.T) {
|
||||
},
|
||||
op: qbtypes.FilterOperatorIn,
|
||||
value: []any{1, 2},
|
||||
expected: "(simpleJSONExtractString(labels, 'test_num') = ? OR simpleJSONExtractString(labels, 'test_num') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)",
|
||||
expectedArgs: []any{"1", "2", "%test_num%", "%test_num\":\"1%", "%test_num\":\"2%"},
|
||||
expected: "((simpleJSONExtractString(labels, 'test_num') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'test_num') = ? AND labels LIKE ? AND labels LIKE ?))",
|
||||
expectedArgs: []any{"1", "%test_num%", "%test_num\":\"1%", "2", "%test_num%", "%test_num\":\"2%"},
|
||||
},
|
||||
{
|
||||
name: "number_between",
|
||||
|
||||
@@ -229,8 +229,8 @@ func TestResourceFilterStatementBuilder_Traces(t *testing.T) {
|
||||
start: testStartNs,
|
||||
end: testEndNs,
|
||||
expected: &qbtypes.Statement{
|
||||
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? OR simpleJSONExtractString(labels, 'service.name') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
|
||||
Args: []any{"redis", "postgres", "%service.name%", "%service.name\":\"redis%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
|
||||
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
|
||||
Args: []any{"redis", "%service.name%", "%service.name\":\"redis%", "postgres", "%service.name%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -244,8 +244,8 @@ func TestResourceFilterStatementBuilder_Traces(t *testing.T) {
|
||||
start: testStartNs,
|
||||
end: testEndNs,
|
||||
expected: &qbtypes.Statement{
|
||||
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') <> ? AND simpleJSONExtractString(labels, 'service.name') <> ?) AND (labels NOT LIKE ? AND labels NOT LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
|
||||
Args: []any{"redis", "postgres", "%service.name\":\"redis%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
|
||||
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') <> ? AND labels NOT LIKE ?) AND (simpleJSONExtractString(labels, 'service.name') <> ? AND labels NOT LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
|
||||
Args: []any{"redis", "%service.name\":\"redis%", "postgres", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -94,7 +94,11 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -104,7 +108,11 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
@@ -28,6 +29,83 @@ func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionB
|
||||
return &conditionBuilder{fm: fm, fl: fl}
|
||||
}
|
||||
|
||||
const (
|
||||
// bodyV2NgramSize is the n of the body_v2 ngrambf_v1 index; a shorter needle yields no
|
||||
// ngram for it to check.
|
||||
bodyV2NgramSize = 4
|
||||
loweredBodyV2 = "LOWER(toString(" + LogsV2BodyV2Column + "))"
|
||||
)
|
||||
|
||||
// bodyIndexLike renders a LIKE on the body_v2 index expression asserting the serialized JSON
|
||||
// contains value. toJSONString escapes the value the same way the column stores it, and
|
||||
// replaceAll then escapes the LIKE metacharacters — backslash first, so the ones the later
|
||||
// steps add are not doubled again. The expression folds to a constant, which is what keeps
|
||||
// the bloom filters usable. anchored keeps toJSONString's quotes, pinning the needle to a
|
||||
// complete field value.
|
||||
func bodyIndexLike(value string, anchored bool, sb *sqlbuilder.SelectBuilder) string {
|
||||
escaped := fmt.Sprintf("lower(toJSONString(%s))", sb.Var(value))
|
||||
if !anchored {
|
||||
quoted := fmt.Sprintf("toJSONString(%s)", sb.Var(value))
|
||||
escaped = fmt.Sprintf("lower(substring(%s, 2, length(%s) - 2))", quoted, quoted)
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
`%s LIKE concat('%%', replaceAll(replaceAll(replaceAll(%s, '\\', '\\\\'), '%%', '\\%%'), '_', '\\_'), '%%')`,
|
||||
loweredBodyV2, escaped,
|
||||
)
|
||||
}
|
||||
|
||||
// likePatternLiterals returns the runs of pattern between unescaped wildcards, with `\`
|
||||
// escapes resolved. Every value the pattern matches contains each run verbatim. ClickHouse
|
||||
// treats `\` as an escape only before `%`, `_` and itself; elsewhere it stands for a literal
|
||||
// backslash, so dropping it would yield a run the pattern does not require.
|
||||
func likePatternLiterals(pattern string) []string {
|
||||
var (
|
||||
literals []string
|
||||
run strings.Builder
|
||||
)
|
||||
for i := 0; i < len(pattern); i++ {
|
||||
switch c := pattern[i]; c {
|
||||
case '%', '_':
|
||||
if run.Len() > 0 {
|
||||
literals = append(literals, run.String())
|
||||
run.Reset()
|
||||
}
|
||||
case '\\':
|
||||
if i+1 >= len(pattern) {
|
||||
run.WriteByte('\\')
|
||||
continue
|
||||
}
|
||||
i++
|
||||
if escaped := pattern[i]; escaped != '%' && escaped != '_' && escaped != '\\' {
|
||||
run.WriteByte('\\')
|
||||
}
|
||||
run.WriteByte(pattern[i])
|
||||
default:
|
||||
run.WriteByte(c)
|
||||
}
|
||||
}
|
||||
if run.Len() > 0 {
|
||||
literals = append(literals, run.String())
|
||||
}
|
||||
return literals
|
||||
}
|
||||
|
||||
// bodyV2SubstringPredicates returns predicates implied by body_v2.message matching pattern,
|
||||
// one per literal run long enough for the ngram index. The legacy body needs none: it already
|
||||
// queries LOWER(body), which is its own index expression.
|
||||
func bodyV2SubstringPredicates(fieldExpression, pattern string, sb *sqlbuilder.SelectBuilder) []string {
|
||||
if fieldExpression != messageSubColumn {
|
||||
return nil
|
||||
}
|
||||
var preds []string
|
||||
for _, literal := range likePatternLiterals(pattern) {
|
||||
if len(literal) >= bodyV2NgramSize {
|
||||
preds = append(preds, bodyIndexLike(literal, false, sb))
|
||||
}
|
||||
}
|
||||
return preds
|
||||
}
|
||||
|
||||
// conditionForSearch ORs a case-insensitive match of the search term across the key
|
||||
// context's searchable columns (unspecified context = every column).
|
||||
func (c *conditionBuilder) conditionForSearch(
|
||||
@@ -314,8 +392,31 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
// make use of case insensitive index for body
|
||||
if fieldExpression == "body" || fieldExpression == messageSubColumn {
|
||||
switch operator {
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
// Bloom filters index lower(body) and lower(toString(body_v2)), not the column
|
||||
// being compared; `=` still decides the row.
|
||||
str, isStr := value.(string)
|
||||
if isStr && fieldExpression == LogsV2BodyColumn {
|
||||
return sb.And(
|
||||
sb.E(fieldExpression, value),
|
||||
fmt.Sprintf("LOWER(%s) = LOWER(%s)", fieldExpression, sb.Var(value)),
|
||||
), nil
|
||||
}
|
||||
if isStr && str != "" && fieldExpression == messageSubColumn {
|
||||
return sb.And(sb.E(fieldExpression, value), bodyIndexLike(str, true, sb)), nil
|
||||
}
|
||||
case qbtypes.FilterOperatorLike, qbtypes.FilterOperatorILike, qbtypes.FilterOperatorContains:
|
||||
pattern, isStr := value.(string)
|
||||
if operator == qbtypes.FilterOperatorContains {
|
||||
pattern, isStr = fmt.Sprintf("%%%s%%", value), true
|
||||
}
|
||||
if isStr {
|
||||
cond := sb.ILike(fieldExpression, pattern)
|
||||
if preds := bodyV2SubstringPredicates(fieldExpression, pattern, sb); len(preds) > 0 {
|
||||
return sb.And(append([]string{cond}, preds...)...), nil
|
||||
}
|
||||
return cond, nil
|
||||
}
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
@@ -410,7 +511,11 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
cond, err := c.conditionForResolvedKey(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -421,7 +526,11 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
cond, err := c.conditionForResolvedKey(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
|
||||
@@ -168,9 +168,9 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: "error message",
|
||||
expectedSQL: "body = ?",
|
||||
expectedArgs: []any{"error message"},
|
||||
value: "Error Message",
|
||||
expectedSQL: "(body = ? AND LOWER(body) = LOWER(?))",
|
||||
expectedArgs: []any{"Error Message", "Error Message"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -619,8 +619,8 @@ func TestConditionForMultipleKeys(t *testing.T) {
|
||||
},
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: "error message",
|
||||
expectedSQL: "body = ? AND severity_text = ?",
|
||||
expectedArgs: []any{"error message", "error message"},
|
||||
expectedSQL: "(body = ? AND LOWER(body) = LOWER(?)) AND severity_text = ?",
|
||||
expectedArgs: []any{"error message", "error message", "error message"},
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
@@ -905,3 +905,116 @@ func TestConditionForJSONBodySearch(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// IN on the body column routes each value back through the `=` path, so every arm picks up
|
||||
// the lower(body) companion — including the values a mixed-type list stringifies.
|
||||
func TestConditionForBodyIn(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
values []any
|
||||
expectedSQL string
|
||||
expectedArgs []any
|
||||
}{
|
||||
{
|
||||
name: "strings",
|
||||
values: []any{"alpha", "beta"},
|
||||
expectedSQL: "((body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)))",
|
||||
expectedArgs: []any{"alpha", "alpha", "beta", "beta"},
|
||||
},
|
||||
{
|
||||
name: "mixed types are stringified before they reach the column",
|
||||
values: []any{"alpha", float64(1), true},
|
||||
expectedSQL: "((body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)))",
|
||||
expectedArgs: []any{"alpha", "alpha", "1", "1", "true", "true"},
|
||||
},
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
conditionBuilder := NewConditionBuilder(fm, fl)
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
key := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "body",
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("1").From("t")
|
||||
cond, _, err := conditionBuilder.ConditionFor(context.Background(), valuer.UUID{}, 0, 0, &key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, qbtypes.ConditionBuilderOptions{},
|
||||
qbtypes.FilterOperatorIn, tc.values, sb)
|
||||
require.NoError(t, err)
|
||||
sb.Where(cond...)
|
||||
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, tc.expectedSQL)
|
||||
assert.Equal(t, tc.expectedArgs, args)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The escaping is ClickHouse's own toJSONString, verified end-to-end in
|
||||
// querier_json_body/07_body_contains.py; what matters here is that the needle stays a
|
||||
// constant expression the bloom filters can fold.
|
||||
func TestBodyIndexLike(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
value string
|
||||
anchored bool
|
||||
expected string
|
||||
expectedArgs []any
|
||||
}{
|
||||
{
|
||||
name: "anchored keeps the quotes toJSONString adds",
|
||||
value: "GET /api/v1/users",
|
||||
anchored: true,
|
||||
expected: `LOWER(toString(body_v2)) LIKE concat('%', replaceAll(replaceAll(replaceAll(lower(toJSONString(?)), '\\', '\\\\'), '%', '\\%'), '_', '\\_'), '%')`,
|
||||
expectedArgs: []any{"GET /api/v1/users"},
|
||||
},
|
||||
{
|
||||
name: "substring strips them and binds the value twice",
|
||||
value: "/api/v1",
|
||||
anchored: false,
|
||||
expected: `LOWER(toString(body_v2)) LIKE concat('%', replaceAll(replaceAll(replaceAll(lower(substring(toJSONString(?), 2, length(toJSONString(?)) - 2)), '\\', '\\\\'), '%', '\\%'), '_', '\\_'), '%')`,
|
||||
expectedArgs: []any{"/api/v1", "/api/v1"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("1").From("t")
|
||||
sb.Where(bodyIndexLike(tc.value, tc.anchored, sb))
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, query, tc.expected)
|
||||
assert.Equal(t, tc.expectedArgs, args)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ClickHouse treats `\` as an escape only before `%`, `_` and itself — every expectation
|
||||
// here was checked against it, where the pattern matches the run and nothing shorter.
|
||||
func TestLikePatternLiterals(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
pattern string
|
||||
expected []string
|
||||
}{
|
||||
{"contains wraps a plain value", "%error%", []string{"error"}},
|
||||
{"wildcards split runs", "%foo%bar%", []string{"foo", "bar"}},
|
||||
{"underscore splits too", "a_b", []string{"a", "b"}},
|
||||
{"escaped wildcards stay literal", `%100\%\_off%`, []string{`100%_off`}},
|
||||
{"escaped backslash collapses", `%C:\\tmp%`, []string{`C:\tmp`}},
|
||||
{"backslash before other chars is literal", `%C:\tmp%`, []string{`C:\tmp`}},
|
||||
{"trailing backslash is literal", `%path\`, []string{`path\`}},
|
||||
{"no literals at all", "%_%", nil},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.expected, likePatternLiterals(tc.pattern))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +135,11 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -146,7 +150,11 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import build_order_by, build_raw_query, get_rows, make_query_request
|
||||
|
||||
LOWER = "alpha"
|
||||
UPPER = "ALPHA"
|
||||
PLAIN = "beta"
|
||||
NON_ASCII = "Mixed CASE Ünïcode"
|
||||
SLASH = "GET /api/v1/users"
|
||||
SUPERSTRING = "GET /api/v1/users/42"
|
||||
QUOTE = 'say "hi" now'
|
||||
BACKSLASH = "C:\\tmp\\log"
|
||||
LIKE_META = "100% _off"
|
||||
TAB = "tab\there"
|
||||
CTRL = "ctrl\x01here"
|
||||
|
||||
BODIES = [LOWER, UPPER, PLAIN, NON_ASCII, SLASH, SUPERSTRING, QUOTE, BACKSLASH, LIKE_META, TAB, CTRL]
|
||||
|
||||
|
||||
# querierlogs/16_body_equality.py with use_json_body on: `body` resolves to body_v2.message,
|
||||
# which the lower(body) companion skips, and the same expressions must still answer alike.
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_bodies",
|
||||
[
|
||||
pytest.param(f"body = '{LOWER}'", {LOWER}, id="equality_exact"),
|
||||
pytest.param(f"body = '{UPPER}'", {UPPER}, id="equality_other_case"),
|
||||
pytest.param("body = 'Alpha'", set(), id="equality_case_must_match"),
|
||||
pytest.param(f"body = '{NON_ASCII}'", {NON_ASCII}, id="equality_non_ascii"),
|
||||
pytest.param("body = 'gamma'", set(), id="equality_no_match"),
|
||||
pytest.param(f"body = '{SLASH}'", {SLASH}, id="equality_slash"),
|
||||
pytest.param("body = 'say \"hi\" now'", {QUOTE}, id="equality_quote"),
|
||||
pytest.param(r"body = 'C:\\tmp\\log'", {BACKSLASH}, id="equality_backslash"),
|
||||
pytest.param(f"body = '{LIKE_META}'", {LIKE_META}, id="equality_like_metacharacters"),
|
||||
pytest.param("body = 'tab\there'", {TAB}, id="equality_tab"),
|
||||
pytest.param("body = 'ctrl\x01here'", {CTRL}, id="equality_control_char"),
|
||||
pytest.param("body = 'GET /api/v1'", set(), id="equality_prefix_does_not_match"),
|
||||
pytest.param(f"body IN ('{LOWER}', '{PLAIN}')", {LOWER, PLAIN}, id="in_excludes_other_case"),
|
||||
pytest.param(f"body IN ('{SLASH}', '{LIKE_META}')", {SLASH, LIKE_META}, id="in_escaped_values"),
|
||||
pytest.param(f"body NOT IN ('{LOWER}', '{UPPER}')", set(BODIES) - {LOWER, UPPER}, id="not_in"),
|
||||
],
|
||||
)
|
||||
def test_logs_body_equality_json(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_bodies: set[str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=body,
|
||||
)
|
||||
for i, body in enumerate(BODIES)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["status"] == "success"
|
||||
# body_v2 comes back parsed; a plain-string body is {"message": <body>}.
|
||||
assert {row["data"]["body"]["message"] for row in get_rows(response)} == expected_bodies
|
||||
@@ -0,0 +1,87 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import build_order_by, build_raw_query, get_rows, make_query_request
|
||||
|
||||
SLASH = "GET /api/v1/users"
|
||||
SUPERSTRING = "GET /api/v1/users/42"
|
||||
QUOTE = 'say "hi" now'
|
||||
BACKSLASH = "C:\\tmp\\log"
|
||||
LIKE_META = "100% _off"
|
||||
NON_ASCII = "Mixed CASE Ünïcode"
|
||||
TAB = "tab\there"
|
||||
CTRL = "ctrl\x01here"
|
||||
|
||||
BODIES = [SLASH, SUPERSTRING, QUOTE, BACKSLASH, LIKE_META, NON_ASCII, TAB, CTRL]
|
||||
|
||||
|
||||
# CONTAINS and LIKE on body_v2 carry a needle over lower(toString(body_v2)), built by
|
||||
# ClickHouse's own toJSONString so it matches the escapes the column stores — a term holding
|
||||
# `/` or `"` finds nothing without them. The needle only ever narrows, never widens, so a
|
||||
# term that should miss must still miss.
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_bodies",
|
||||
[
|
||||
pytest.param("body CONTAINS '/api/v1'", {SLASH, SUPERSTRING}, id="contains_slash"),
|
||||
pytest.param("body CONTAINS '\"hi\"'", {QUOTE}, id="contains_quote"),
|
||||
pytest.param(r"body CONTAINS 'C:\\tmp'", {BACKSLASH}, id="contains_backslash"),
|
||||
pytest.param("body CONTAINS 'CASE Ünïcode'", {NON_ASCII}, id="contains_non_ascii"),
|
||||
pytest.param("body CONTAINS 'here'", {TAB, CTRL}, id="contains_beside_control_chars"),
|
||||
pytest.param("body CONTAINS 'nothing here'", set(), id="contains_no_match"),
|
||||
# a run shorter than the ngram size gets no needle, so the match alone decides
|
||||
pytest.param("body CONTAINS 'ab'", {TAB}, id="contains_run_below_ngram_size"),
|
||||
# a `\%` the user wrote has to stay a literal `%` through unescape and re-escape
|
||||
pytest.param(r"body LIKE '%100\% \_off%'", {LIKE_META}, id="like_user_escaped_wildcards"),
|
||||
pytest.param(r"body LIKE '%100\%\_off%'", set(), id="like_escaped_wildcards_are_not_wildcards"),
|
||||
pytest.param("body LIKE 'GET /api/v1/users%'", {SLASH, SUPERSTRING}, id="like_trailing_wildcard"),
|
||||
pytest.param("body LIKE '%CASE Ünïcode'", {NON_ASCII}, id="like_leading_wildcard"),
|
||||
],
|
||||
)
|
||||
def test_logs_body_contains_json(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_bodies: set[str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=body,
|
||||
)
|
||||
for i, body in enumerate(BODIES)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["status"] == "success"
|
||||
# body_v2 comes back parsed; a plain-string body is {"message": <body>}.
|
||||
assert {row["data"]["body"]["message"] for row in get_rows(response)} == expected_bodies
|
||||
@@ -3,11 +3,13 @@ from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import build_order_by, build_raw_query, get_rows, make_query_request
|
||||
|
||||
|
||||
def test_logs_json_body_simple_searches(
|
||||
@@ -911,3 +913,61 @@ def test_logs_json_body_listing(
|
||||
assert len(results) == 1
|
||||
count = results[0]["data"][0][0]
|
||||
assert count == 4 # 4 logs have status="success"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_services",
|
||||
[
|
||||
pytest.param("body.service IN ['auth', 'payment']", {"auth", "payment"}, id="in_scalar_path"),
|
||||
pytest.param("body.status IN [200, 500]", {"auth", "payment"}, id="in_number_path"),
|
||||
pytest.param("body.service NOT IN ['auth']", {"payment", "search"}, id="not_in_scalar_path"),
|
||||
# An `[]` path is extracted as an array. Comparing that array to each scalar in the
|
||||
# list is something ClickHouse rejects outright (code 130), so this shape used to
|
||||
# fail the whole query; per-value extraction reads the first element instead.
|
||||
pytest.param("body.user_names[*] IN ['alpha', 'gamma']", {"auth", "payment"}, id="in_array_path"),
|
||||
],
|
||||
)
|
||||
def test_logs_json_body_in_operator(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_services: set[str],
|
||||
) -> None:
|
||||
"""IN over a body JSON path fans out to one comparison per value."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
specs = [("auth", 200, ["alpha", "beta"]), ("payment", 500, ["gamma"]), ("search", 404, ["beta", "alpha"])]
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=json.dumps({"service": service, "status": status, "user_names": user_names}),
|
||||
)
|
||||
for i, (service, status, user_names) in enumerate(specs)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["status"] == "success"
|
||||
# flag off: the body comes back as the raw JSON string
|
||||
assert {json.loads(row["data"]["body"])["service"] for row in get_rows(response)} == expected_services
|
||||
|
||||
92
tests/integration/tests/querierlogs/16_body_equality.py
Normal file
92
tests/integration/tests/querierlogs/16_body_equality.py
Normal file
@@ -0,0 +1,92 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import build_order_by, build_raw_query, get_column_data_from_response, make_query_request
|
||||
|
||||
LOWER = "alpha"
|
||||
UPPER = "ALPHA"
|
||||
PLAIN = "beta"
|
||||
NON_ASCII = "Mixed CASE Ünïcode"
|
||||
SLASH = "GET /api/v1/users"
|
||||
SUPERSTRING = "GET /api/v1/users/42"
|
||||
QUOTE = 'say "hi" now'
|
||||
BACKSLASH = "C:\\tmp\\log"
|
||||
LIKE_META = "100% _off"
|
||||
TAB = "tab\there"
|
||||
CTRL = "ctrl\x01here"
|
||||
|
||||
BODIES = [LOWER, UPPER, PLAIN, NON_ASCII, SLASH, SUPERSTRING, QUOTE, BACKSLASH, LIKE_META, TAB, CTRL]
|
||||
|
||||
|
||||
# `body = ?` carries a case-insensitive LOWER(body) companion for the bloom filters, so a
|
||||
# body differing only in case must still not come back.
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_bodies",
|
||||
[
|
||||
pytest.param(f"body = '{LOWER}'", {LOWER}, id="equality_exact"),
|
||||
pytest.param(f"body = '{UPPER}'", {UPPER}, id="equality_other_case"),
|
||||
pytest.param("body = 'Alpha'", set(), id="equality_case_must_match"),
|
||||
pytest.param(f"body = '{NON_ASCII}'", {NON_ASCII}, id="equality_non_ascii"),
|
||||
pytest.param("body = ''", set(), id="equality_empty"),
|
||||
pytest.param("body = 'gamma'", set(), id="equality_no_match"),
|
||||
# the companion is a LIKE-free equality, so none of these are metacharacters to it
|
||||
pytest.param(f"body = '{SLASH}'", {SLASH}, id="equality_slash"),
|
||||
pytest.param("body = 'say \"hi\" now'", {QUOTE}, id="equality_quote"),
|
||||
pytest.param(r"body = 'C:\\tmp\\log'", {BACKSLASH}, id="equality_backslash"),
|
||||
pytest.param(f"body = '{LIKE_META}'", {LIKE_META}, id="equality_like_metacharacters"),
|
||||
pytest.param("body = 'tab\there'", {TAB}, id="equality_tab"),
|
||||
pytest.param("body = 'ctrl\x01here'", {CTRL}, id="equality_control_char"),
|
||||
# a prefix of another body must not match it
|
||||
pytest.param("body = 'GET /api/v1'", set(), id="equality_prefix_does_not_match"),
|
||||
pytest.param(f"body IN ('{LOWER}', '{PLAIN}')", {LOWER, PLAIN}, id="in_excludes_other_case"),
|
||||
pytest.param(f"body IN ('{SLASH}', '{LIKE_META}')", {SLASH, LIKE_META}, id="in_escaped_values"),
|
||||
pytest.param(f"body NOT IN ('{LOWER}', '{UPPER}')", set(BODIES) - {LOWER, UPPER}, id="not_in"),
|
||||
],
|
||||
)
|
||||
def test_logs_body_equality(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_bodies: set[str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=body,
|
||||
)
|
||||
for i, body in enumerate(BODIES)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["status"] == "success"
|
||||
assert set(get_column_data_from_response(response.json(), "body")) == expected_bodies
|
||||
Reference in New Issue
Block a user