mirror of
https://github.com/SigNoz/signoz.git
synced 2026-09-21 10:50:41 +01:00
9c886be120
The semantic convention family as first call citizen revealed that the
current state of the query builder needs a bit refactoring for long term
maintenance.
The `FieldMapper` and `ConditionBuilder` are now one abstraction
`Storage`.
A storage now answers
- what the compiler cannot know i.e one read per field key (the bare
SQL, the membership present or absent, what an absent row reads, and
whether the read keeps its type or filters only).
- the fallback for a key metadata does not report
- its traits
- and one Condition compilation part.
And we introduce a new type to use in the system, `Resolved`
```
// Resolved is what resolution produces for one key: its meanings, and how
// they came to be. It is the only thing the compilers receive. Compile it
// with the operator and value it was resolved with.
type Resolved struct {
Key *telemetrytypes.TelemetryFieldKey
Fields []*telemetrytypes.LogicalField
// FromFallback: the fields came from the storage's fallback, not from
// metadata matches.
FromFallback bool
// Ambiguous: the matches held several interpretations.
Ambiguous bool
// Skipped: the storage contributes nothing for this key.
Skipped bool
Warnings []string
}
```
The prepared SQL has no changes, where it changed, it specifically made
the expression better by removing the redundant part.
- The prepared SQL remains identical with this refactoring
- No changes to integration tests
Assisted-by: Claude Fable 5.1
1353 lines
87 KiB
Go
1353 lines
87 KiB
Go
package logsstatementbuilder
|
|
|
|
import (
|
|
"context"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
|
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
|
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
|
"github.com/SigNoz/signoz/pkg/querybuilder"
|
|
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
|
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
|
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
|
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
|
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
|
"github.com/SigNoz/signoz/pkg/valuer"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const (
|
|
testJSONQueryPrefix = "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE"
|
|
testJSONQuerySuffix = "AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?"
|
|
)
|
|
|
|
type TestExpected struct {
|
|
WhereClause string
|
|
Args []any
|
|
Warnings []string
|
|
}
|
|
|
|
func (t TestExpected) GetQuery() string {
|
|
return strings.Join([]string{testJSONQueryPrefix, t.WhereClause, testJSONQuerySuffix}, " ")
|
|
}
|
|
|
|
func TestJSONStmtBuilder_TimeSeries(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "Group By Simple Path",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Aggregations: []qbtypes.LogAggregation{
|
|
{
|
|
Expression: "count()",
|
|
},
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "user.age",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "WITH __limit_cte AS (SELECT toString(multiIf((dynamicElement(body_v2.`user.age`, 'Int64') IS NOT NULL), toString(dynamicElement(body_v2.`user.age`, 'Int64')), (dynamicElement(body_v2.`user.age`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.age`, 'String'), NULL)) AS `__GROUP_BY_KEY_0_user.age`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_user.age` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf((dynamicElement(body_v2.`user.age`, 'Int64') IS NOT NULL), toString(dynamicElement(body_v2.`user.age`, 'Int64')), (dynamicElement(body_v2.`user.age`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.age`, 'String'), NULL)) AS `__GROUP_BY_KEY_0_user.age`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_user.age`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_user.age` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_user.age`",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
|
},
|
|
},
|
|
{
|
|
name: "Group By One Array Join",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Aggregations: []qbtypes.LogAggregation{
|
|
{
|
|
Expression: "count()",
|
|
},
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "education[].awards[].type",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "WITH __limit_cte AS (SELECT toString(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))) AS `__GROUP_BY_KEY_0_education[].awards[].type`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_education[].awards[].type` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))) AS `__GROUP_BY_KEY_0_education[].awards[].type`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_education[].awards[].type`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_education[].awards[].type` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_education[].awards[].type`",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
/* Promoted path tests commented out — Materialized now means type hint (direct sub-column),
|
|
not a body_promoted.* column. These tests assumed the old coalesce(body_promoted.x, body_v2.x) path.
|
|
|
|
func TestStmtBuilderTimeSeriesBodyGroupByPromoted(t *testing.T) {
|
|
statementBuilder := buildJSONTestStatementBuilder(t, "user.age", "user.name")
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "Two simple paths",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Aggregations: []qbtypes.LogAggregation{
|
|
{
|
|
Expression: "count()",
|
|
},
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "user.age",
|
|
},
|
|
},
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "user.name",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Args: []any{ "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
*/
|
|
|
|
func TestJSONStmtBuilder_PrimitivePaths(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
cases := []struct {
|
|
name string
|
|
filter string
|
|
expected TestExpected
|
|
expectedErr error
|
|
}{
|
|
// ── intrinsic sub-column (body_v2.message) ──────────────────────────────
|
|
{
|
|
name: "message Exists",
|
|
filter: "message Exists",
|
|
expected: TestExpected{
|
|
WhereClause: "body_v2.message <> ''",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "message Contains 'Iron Award'",
|
|
filter: "message Contains 'Iron Award'",
|
|
expected: TestExpected{
|
|
WhereClause: "LOWER(body_v2.message) LIKE LOWER(?)",
|
|
Args: []any{"%Iron Award%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── flat nested primitive (x.y) ─────────────────────────────────────────
|
|
{
|
|
name: "Simple string filter",
|
|
filter: "body.user.name = 'x'",
|
|
expected: TestExpected{
|
|
WhereClause: "((dynamicElement(body_v2.`user.name`, 'String') = ?) AND has(JSONAllPaths(body_v2), 'user.name'))",
|
|
Args: []any{"x", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Simple Int field Contains value",
|
|
filter: "body.user.age Contains 25",
|
|
expected: TestExpected{
|
|
WhereClause: "(((LOWER(toString(dynamicElement(body_v2.`user.age`, 'Int64'))) LIKE LOWER(?)) AND has(JSONAllPaths(body_v2), 'user.age')) OR ((LOWER(dynamicElement(body_v2.`user.age`, 'String')) LIKE LOWER(?)) AND has(JSONAllPaths(body_v2), 'user.age')))",
|
|
Args: []any{"%25%", "%25%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `user.age` is ambiguous, found 2 different combinations of field context / data type: [name=user.age,context=body,datatype=int64 name=user.age,context=body,datatype=string]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Simple Float field Contains value",
|
|
filter: "body.user.height Contains 5.8",
|
|
expected: TestExpected{
|
|
WhereClause: "((LOWER(toString(dynamicElement(body_v2.`user.height`, 'Float64'))) LIKE LOWER(?)) AND has(JSONAllPaths(body_v2), 'user.height'))",
|
|
Args: []any{"%5.8%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Float GT",
|
|
filter: "body.user.height > 5.8",
|
|
expected: TestExpected{
|
|
WhereClause: "((toFloat64(dynamicElement(body_v2.`user.height`, 'Float64')) > ?) AND has(JSONAllPaths(body_v2), 'user.height'))",
|
|
Args: []any{5.8, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "String Like",
|
|
filter: "body.user.name LIKE '%ali%'",
|
|
expected: TestExpected{
|
|
WhereClause: "((dynamicElement(body_v2.`user.name`, 'String') LIKE ?) AND has(JSONAllPaths(body_v2), 'user.name'))",
|
|
Args: []any{"%ali%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "String Regexp",
|
|
filter: "body.user.name REGEXP 'al.*'",
|
|
expected: TestExpected{
|
|
WhereClause: "((match(dynamicElement(body_v2.`user.name`, 'String'), ?)) AND has(JSONAllPaths(body_v2), 'user.name'))",
|
|
Args: []any{"al.*", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive Bool NotEqual",
|
|
filter: "body.user.active != true",
|
|
expected: TestExpected{
|
|
WhereClause: "(assumeNotNull(dynamicElement(body_v2.`user.active`, 'Bool')) <> ?)",
|
|
Args: []any{true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive Bool NotExists",
|
|
filter: "body.user.active NOT EXISTS",
|
|
expected: TestExpected{
|
|
WhereClause: "(dynamicElement(body_v2.`user.active`, 'Bool') IS NULL)",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive String NotContains",
|
|
filter: "body.user.name NOT CONTAINS 'IIT'",
|
|
expected: TestExpected{
|
|
WhereClause: "(LOWER(assumeNotNull(dynamicElement(body_v2.`user.name`, 'String'))) NOT LIKE LOWER(?))",
|
|
Args: []any{"%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive String NotLike",
|
|
filter: "body.user.name NOT LIKE '%ali%'",
|
|
expected: TestExpected{
|
|
WhereClause: "(assumeNotNull(dynamicElement(body_v2.`user.name`, 'String')) NOT LIKE ?)",
|
|
Args: []any{"%ali%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive String NotRegexp",
|
|
filter: "body.user.name NOT REGEXP 'al.*'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT match(assumeNotNull(dynamicElement(body_v2.`user.name`, 'String')), ?))",
|
|
Args: []any{"al.*", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive String NotIn",
|
|
filter: "body.user.name NOT IN ['alice', 'bob']",
|
|
expected: TestExpected{
|
|
WhereClause: "(assumeNotNull(dynamicElement(body_v2.`user.name`, 'String')) NOT IN (?, ?))",
|
|
Args: []any{"alice", "bob", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── deeply nested primitive (x.y.z) ─────────────────────────────────────
|
|
{
|
|
name: "Primitive Int64 NotBetween",
|
|
filter: "body.user.address.zip NOT BETWEEN 100000 AND 999999",
|
|
expected: TestExpected{
|
|
WhereClause: "(toFloat64(assumeNotNull(dynamicElement(body_v2.`user.address.zip`, 'Int64'))) NOT BETWEEN ? AND ?)",
|
|
Args: []any{float64(100000), float64(999999), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, qbtypes.RequestTypeRaw,
|
|
qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: c.filter},
|
|
Limit: 10,
|
|
}, nil)
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.GetQuery(), q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
})
|
|
}
|
|
}
|
|
|
|
/* Promoted path list-query tests commented out — Materialized now means type hint
|
|
(direct sub-column access), not a body_promoted.* column.
|
|
|
|
func TestStatementBuilderListQueryBodyPromoted(t *testing.T) {
|
|
statementBuilder := buildJSONTestStatementBuilder(t, "education", "tags")
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected TestExpected
|
|
expectedErr error
|
|
}{
|
|
{
|
|
name: "Has Array promoted uses body fallback",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "has(body.tags, 'production')"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) exists",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].name Exists"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) -> Array(Dynamic) + Array(JSON) exists",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].awards[].name Exists"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) -> Array(Dynamic) + Array(JSON) = 'Iron Award'",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].awards[].name = 'Iron Award'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "Iron Award", "Iron Award", "Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) contains value",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].parameters Contains 1.65"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{uint64(1747945619), uint64(1747983448), "%1.65%", 1.65, "%1.65%", 1.65, "%1.65%", 1.65, "%1.65%", 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true name=education[].parameters,context=body,datatype=[]dynamic,materialized=true]."},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) contains String value",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].name Contains 'IIT'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "%IIT%", "%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Dynamic array contains boolean",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].parameters Contains true"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{uint64(1747945619), uint64(1747983448), "%true%", true, "%true%", true, "%true%", true, "%true%", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true name=education[].parameters,context=body,datatype=[]dynamic,materialized=true]."},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Dynamic array contains String",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].parameters Contains 'passed'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{uint64(1747945619), uint64(1747983448), "%passed%", "passed", "%passed%", "passed", "%passed%", "passed", "%passed%", "passed", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true name=education[].parameters,context=body,datatype=[]dynamic,materialized=true]."},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Array(Dynamic) + Array(JSON) both can contain a value",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].awards[].type = 'sports'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "sports", "sports", "sports", "sports", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Multi nested Array(Dynamic) + Array(JSON) both can contain a value",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].awards[].participated[].team[].branch Contains 'Civil'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.GetQuery(), q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
*/
|
|
|
|
func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
cases := []struct {
|
|
name string
|
|
filter string
|
|
expected TestExpected
|
|
expectedErr error
|
|
}{
|
|
// ── single-hop (x[].y) ──────────────────────────────────────────────────
|
|
{
|
|
name: "Key inside Array(JSON) exists",
|
|
filter: "body.education[].name Exists",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) contains String value",
|
|
filter: "body.education[].name Contains 'IIT'",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> LOWER(dynamicElement(`body_v2.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{"%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Int in array Contains value",
|
|
filter: "body.education[].year Contains 2020",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> LOWER(toString(dynamicElement(`body_v2.education`.`year`, 'Int64'))) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{"%2020%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Array simple path NotEqual",
|
|
filter: "body.education[].type != '10001'",
|
|
expected: TestExpected{
|
|
WhereClause: "((NOT arrayExists(`body_v2.education`-> toFloat64OrNull(dynamicElement(`body_v2.education`.`type`, 'String')) = ?, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND (NOT arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`type`, 'Int64') = ?, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))",
|
|
Args: []any{int64(10001), int64(10001), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].type` is ambiguous, found 2 different combinations of field context / data type: [name=education[].type,context=body,datatype=string name=education[].type,context=body,datatype=int64]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Array simple path NotExists",
|
|
filter: "body.education[].name NOT EXISTS",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Array simple path NotContains",
|
|
filter: "body.education[].name NOT CONTAINS 'IIT'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> LOWER(dynamicElement(`body_v2.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── single-hop terminal array (x[].y[]) ─────────────────────────────────
|
|
{
|
|
name: "Key inside Array(JSON) contains Float value",
|
|
filter: "body.education[].parameters Contains 1.65",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')) OR ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))) OR arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')))",
|
|
Args: []any{"%1.65%", 1.65, "%1.65%", 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Dynamic array element comparison",
|
|
filter: "body.education[].parameters = 'passed'",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.education`-> arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')) OR ((arrayExists(`body_v2.education`-> arrayExists(x -> toString(x) = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')))",
|
|
Args: []any{"passed", "passed", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Dynamic array contains String",
|
|
filter: "body.education[].parameters Contains 'passed'",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')) OR ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))) OR arrayExists(x -> toString(x) = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')))",
|
|
Args: []any{"%passed%", "passed", "%passed%", "passed", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Dynamic array IN Operator",
|
|
filter: "body.education[].parameters IN [1.65, 1.99]",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) IN (?, ?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')) OR ((arrayExists(`body_v2.education`-> arrayExists(x -> toString(x) IN (?, ?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')))",
|
|
Args: []any{1.65, 1.99, "1.65", "1.99", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Terminal array NotContains float",
|
|
filter: "body.education[].parameters NOT CONTAINS 1.65",
|
|
expected: TestExpected{
|
|
WhereClause: "((NOT arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND (NOT arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))) OR arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))",
|
|
Args: []any{"%1.65%", float64(1.65), "%1.65%", float64(1.65), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Simple has filter",
|
|
filter: "has(body.education[].parameters, 1.65)",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{1.65, 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{
|
|
"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic].",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "Flat path hasAll filter",
|
|
filter: "hasAll(body.user.permissions, ['read', 'write'])",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(x -> x = ?, dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))')) AND arrayExists(x -> x = ?, dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))')))",
|
|
Args: []any{"read", "write", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── non-array intermediate hop (x[].y.z) ────────────────────────────────
|
|
{
|
|
name: "Non-array intermediate NotEqual",
|
|
filter: "body.http-events[].request-info.host != 'example.com'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.http-events`-> dynamicElement(`body_v2.http-events`.`request-info.host`, 'String') = ?, dynamicElement(body_v2.`http-events`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"example.com", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── double-hop (x[].y[].z) ──────────────────────────────────────────────
|
|
{
|
|
name: "Key inside Array(JSON) -> Array(Dynamic) + Array(JSON) exists",
|
|
filter: "body.education[].awards[].name Exists",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) -> Array(Dynamic) + Array(JSON) = 'Iron Award'",
|
|
filter: "body.education[].awards[].name = 'Iron Award'",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{"Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Integer BETWEEN Operator",
|
|
filter: "education[].awards[].semester BETWEEN 2 AND 4",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{float64(2), float64(4), float64(2), float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Integer IN Operator",
|
|
filter: "education[].awards[].semester IN [2, 4]",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{float64(2), float64(4), float64(2), float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Nested array NotEqual",
|
|
filter: "body.education[].awards[].type != 'sports'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"sports", "sports", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Nested array NotExists",
|
|
filter: "body.education[].awards[].name NOT EXISTS",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── deep multi-hop (x[].y[].z[].w...) ──────────────────────────────────
|
|
{
|
|
name: "Super nested array contains Int64 value",
|
|
filter: "body.interests[].entities[].reviews[].entries[].metadata[].positions[].ratings Contains 4",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'interests')) OR ((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))')) OR arrayExists(x -> toFloat64OrNull(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'interests')))",
|
|
Args: []any{"%4%", float64(4), "%4%", float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `interests[].entities[].reviews[].entries[].metadata[].positions[].ratings` is ambiguous, found 2 different combinations of field context / data type: [name=interests[].entities[].reviews[].entries[].metadata[].positions[].ratings,context=body,datatype=[]int64 name=interests[].entities[].reviews[].entries[].metadata[].positions[].ratings,context=body,datatype=[]string]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Multi nested Array(Dynamic) + Array(JSON) both can contain a value",
|
|
filter: "body.education[].awards[].participated[].team[].branch Contains 'Civil'",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{"%Civil%", "%Civil%", "%Civil%", "%Civil%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// IN on a single-type Int64 array terminal: any element of scores IN {90, 95}
|
|
name: "Terminal Int64 array IN operator",
|
|
filter: "body.education[].scores IN [90, 95]",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) IN (?, ?), dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{float64(90), float64(95), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// NOT IN on a single-type Int64 array terminal: no element of scores IN {90, 95}.
|
|
// Negative operator → NOT wraps outer arrayExists; no path-index guard added.
|
|
name: "Terminal Int64 array NOT IN operator",
|
|
filter: "body.education[].scores NOT IN [90, 95]",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) IN (?, ?), dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{float64(90), float64(95), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// BETWEEN on a single-type Int64 array terminal: any element of scores BETWEEN 80 AND 95.
|
|
// BETWEEN is an IsArrayOperator → value arrives as []any{80,95} → c.valueType.IsArray=true.
|
|
name: "Terminal Int64 array BETWEEN operator",
|
|
filter: "body.education[].scores BETWEEN 80 AND 95",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{float64(80), float64(95), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// NOT BETWEEN on a single-type Int64 array terminal: no element BETWEEN 80 AND 95.
|
|
// Negative → NOT wraps outer arrayExists; no path-index guard.
|
|
name: "Terminal Int64 array NOT BETWEEN operator",
|
|
filter: "body.education[].scores NOT BETWEEN 80 AND 95",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{float64(80), float64(95), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Nested path hasAny filter",
|
|
filter: "hasAny(education[].awards[].participated[].members, ['Piyush', 'Tushar'])",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"Piyush", "Tushar", "Piyush", "Tushar", "Piyush", "Tushar", "Piyush", "Tushar", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "dynamic_array_element_compare_CONTAINS",
|
|
filter: "interests[].entities[].product_codes Contains 1",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))) OR arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)')))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'interests'))",
|
|
Args: []any{"%1%", float64(1), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "dynamic_array_element_compare_HAS_STRING",
|
|
filter: "has(interests[].entities[].product_codes, '2002')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{int64(2002), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "dynamic_array_element_compare_NOT_EQUAL",
|
|
filter: "interests[].entities[].product_codes != '1'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{int64(1), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "dynamic_array_element_compare_HAS_INT",
|
|
filter: "has(interests[].entities[].product_codes, 1001)",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{float64(1001), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── scalar leaf reached through an array ───
|
|
{
|
|
name: "Nested primitive leaf has",
|
|
filter: "has(body.education[].name, 'IIT')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"IIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Nested primitive leaf hasAny",
|
|
filter: "hasAny(body.education[].name, ['IIT', 'MIT'])",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') IN (?, ?), false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"IIT", "MIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// contains-all: single-value hasAll over a nested leaf collapses to has.
|
|
name: "Nested primitive leaf hasAll single collapses to has",
|
|
filter: "hasAll(body.education[].name, 'IIT')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"IIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// contains-all: "some element is IIT AND some element is MIT" (AND of per-value has).
|
|
name: "Nested primitive leaf hasAll multi (contains-all)",
|
|
filter: "hasAll(body.education[].name, ['IIT', 'MIT'])",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) AND arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"IIT", "MIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── numeric literal against an Int64 array is collision-handled ───
|
|
{
|
|
name: "Nested Int64 array has collision",
|
|
filter: "has(body.education[].scores, 90)",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{float64(90), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── hasAny folds multiple scalar arguments into one value set ─────
|
|
{
|
|
name: "hasAny folds multiple scalar args",
|
|
filter: "hasAny(body.user.permissions, 'read', 'write')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(x -> x IN (?, ?), dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'))",
|
|
Args: []any{"read", "write", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── hasToken over JSON string fields (nested leaf, top-level array, nested array) ──
|
|
{
|
|
name: "hasToken nested string leaf",
|
|
filter: "hasToken(body.education[].name, 'harvard')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> ifNull(hasToken(LOWER(dynamicElement(`body_v2.education`.`name`, 'String')), LOWER(?)), false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"harvard", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "hasToken top-level string array",
|
|
filter: "hasToken(body.user.permissions, 'admin')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'))",
|
|
Args: []any{"admin", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "hasToken nested string array",
|
|
filter: "hasToken(body.education[].awards[].participated[].members, 'piyush')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"piyush", "piyush", "piyush", "piyush", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── hasToken over the message field: bare body and explicit body.message are
|
|
// equivalent, both target the body.message column directly (no dynamicElement) ──
|
|
{
|
|
name: "hasToken bare body",
|
|
filter: "hasToken(body, 'production')",
|
|
expected: TestExpected{
|
|
WhereClause: "hasToken(LOWER(body.message), LOWER(?))",
|
|
Args: []any{"production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{bodySearchDefaultWarning},
|
|
},
|
|
},
|
|
{
|
|
name: "hasToken explicit body.message",
|
|
filter: "hasToken(body.message, 'production')",
|
|
expected: TestExpected{
|
|
WhereClause: "hasToken(LOWER(body.message), LOWER(?))",
|
|
Args: []any{"production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── scalar (non-array) leaf: treated as a single-element set ─────────────
|
|
{
|
|
name: "Scalar leaf has",
|
|
filter: "has(body.user.name, 'alice')",
|
|
expected: TestExpected{
|
|
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false)",
|
|
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Scalar leaf hasAny",
|
|
filter: "hasAny(body.user.name, ['alice', 'bob'])",
|
|
expected: TestExpected{
|
|
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') IN (?, ?), false)",
|
|
Args: []any{"alice", "bob", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// contains-all: single-value hasAll over a plain scalar collapses to has.
|
|
name: "Scalar leaf hasAll single collapses to has",
|
|
filter: "hasAll(body.user.name, 'alice')",
|
|
expected: TestExpected{
|
|
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false)",
|
|
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// contains-all over a one-element set: the scalar must equal every value, so
|
|
// distinct values never match.
|
|
name: "Scalar leaf hasAll multi (contains-all)",
|
|
filter: "hasAll(body.user.name, ['alice', 'bob'])",
|
|
expected: TestExpected{
|
|
WhereClause: "(ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false) AND ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false))",
|
|
Args: []any{"alice", "bob", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, qbtypes.RequestTypeRaw,
|
|
qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: c.filter},
|
|
Limit: 10,
|
|
}, nil)
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.GetQuery(), q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestJSONStmtBuilder_IndexedPaths(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, true)
|
|
cases := []struct {
|
|
name string
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected TestExpected
|
|
expectedErr error
|
|
}{
|
|
// ── indexed equal: emits assumeNotNull(dynamicElement) = ? ───────────
|
|
{
|
|
name: "Indexed String Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.name = 'alice'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "((lower(assumeNotNull(dynamicElement(body_v2.user.name, 'String'))) = ?) AND has(JSONAllPaths(body_v2), 'user.name'))",
|
|
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Indexed Int64 Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.address.zip = 110001"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "((toFloat64(assumeNotNull(dynamicElement(body_v2.user.address.zip, 'Int64'))) = ?) AND has(JSONAllPaths(body_v2), 'user.address.zip'))",
|
|
Args: []any{float64(110001), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── indexed exists: index is skipped; emits plain IS NOT NULL ───────────
|
|
// EXISTS/NOT EXISTS bypass the index because indexed columns store a default
|
|
// empty value for absent fields, making != "" unreliable for existence checks
|
|
// (a field with value "" would be incorrectly excluded).
|
|
{
|
|
name: "Indexed String Exists",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.name Exists"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "(dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL)",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── indexed not-equal: assumeNotNull wrapping + no path index ─────────
|
|
{
|
|
name: "Indexed String NotEqual",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.name != 'alice'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "(lower(assumeNotNull(dynamicElement(body_v2.user.name, 'String'))) <> ?)",
|
|
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── special-character indexed paths ───────────────────────────────────
|
|
{
|
|
name: "Indexed hyphen key Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.http-status = 200"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "(((toFloat64(assumeNotNull(dynamicElement(body_v2.`http-status`, 'Int64'))) = ?) AND has(JSONAllPaths(body_v2), 'http-status')) OR ((toFloat64OrNull(lower(assumeNotNull(dynamicElement(body_v2.`http-status`, 'String')))) = ?) AND has(JSONAllPaths(body_v2), 'http-status')))",
|
|
Args: []any{float64(200), float64(200), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Indexed nested hyphen key Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.response.time-taken > 1.5"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "((toFloat64(assumeNotNull(dynamicElement(body_v2.response.`time-taken`, 'Float64'))) > ?) AND has(JSONAllPaths(body_v2), 'response.time-taken'))",
|
|
Args: []any{float64(1.5), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── Bool is NOT IndexSupported — index branch must be skipped ─────────
|
|
// user.active is Bool; even with indexed builder it must not use the index.
|
|
{
|
|
name: "Bool skips index — Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.active = true"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "((dynamicElement(body_v2.`user.active`, 'Bool') = ?) AND has(JSONAllPaths(body_v2), 'user.active'))",
|
|
Args: []any{true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, qbtypes.RequestTypeRaw, c.query, nil)
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.GetQuery(), q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestJSONStmtBuilder_SelectField(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "select_x_education[].awards[].participated[].members",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
SelectFields: []telemetrytypes.TelemetryFieldKey{
|
|
{
|
|
Name: "education[].awards[].participated[].members",
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "user.name exists",
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `__SELECT_KEY_0_education[].awards[].participated[].members` FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "select_x_education[].awards[].type",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
SelectFields: []telemetrytypes.TelemetryFieldKey{
|
|
{
|
|
Name: "education[].awards[].type",
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "user.name exists",
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `__SELECT_KEY_0_education[].awards[].type` FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "select_x_user.name",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
SelectFields: []telemetrytypes.TelemetryFieldKey{
|
|
{
|
|
Name: "user.name",
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, dynamicElement(body_v2.`user.name`, 'String') AS `__SELECT_KEY_0_user.name` FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestJSONStmtBuilder_OrderBy(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "order_by_education[].awards[].participated[].members",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
Order: []qbtypes.OrderBy{
|
|
{
|
|
Key: qbtypes.OrderByKey{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "education[].awards[].participated[].members",
|
|
},
|
|
},
|
|
Direction: qbtypes.OrderDirectionAsc,
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "user.name exists",
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) asc LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "order_by_user.name",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
Order: []qbtypes.OrderBy{
|
|
{
|
|
Key: qbtypes.OrderByKey{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "user.name",
|
|
},
|
|
},
|
|
Direction: qbtypes.OrderDirectionAsc,
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY dynamicElement(body_v2.`user.name`, 'String') asc LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) {
|
|
statementBuilder, metadataStore := buildJSONTestStatementBuilder(t, false)
|
|
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
|
keysMap := logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
|
for _, keys := range keysMap {
|
|
for _, key := range keys {
|
|
metadataStore.SetKey(key)
|
|
}
|
|
}
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "resource_aggregation_and_group_by_with_json_enabled",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "region",
|
|
},
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "user.name exists",
|
|
},
|
|
Aggregations: []qbtypes.LogAggregation{
|
|
{
|
|
Expression: "count_distinct(service.name)",
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`region` IS NOT NULL, resource.`region`::String, NULL)) AS `__GROUP_BY_KEY_0_region`, countDistinct(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) OR mapContains(attributes_string, 'user.name')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, `__GROUP_BY_KEY_0_region`",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
|
Warnings: []string{"Key `user.name` is ambiguous, found 2 different combinations of field context / data type: [name=user.name,context=body,datatype=string name=user.name,context=attribute,datatype=string]."},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetrytypestest.MockMetadataStore {
|
|
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
|
mockMetadataStore.SetStaticFields(logstelemetryschema.IntrinsicFields)
|
|
types, _ := telemetrytypes.TestJSONTypeSet()
|
|
for path, fieldDataTypes := range types {
|
|
for _, fdt := range fieldDataTypes {
|
|
key := &telemetrytypes.TelemetryFieldKey{
|
|
Name: path,
|
|
Signal: telemetrytypes.SignalLogs,
|
|
FieldContext: telemetrytypes.FieldContextBody,
|
|
FieldDataType: fdt,
|
|
}
|
|
if addIndexes {
|
|
jsonType := telemetrytypes.MappingFieldDataTypeToJSONDataType[fdt]
|
|
idx := slices.IndexFunc(telemetrytypes.TestIndexedPaths, func(entry telemetrytypes.TestIndexedPathEntry) bool {
|
|
return entry.Path == path && entry.Type == jsonType
|
|
})
|
|
if idx >= 0 {
|
|
key.Indexes = append(key.Indexes, telemetrytypes.TelemetryFieldKeySkipIndex{
|
|
Name: path,
|
|
FieldContext: telemetrytypes.FieldContextBody,
|
|
FieldDataType: fdt,
|
|
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
|
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(logstelemetryschema.LogsV2BodyV2Column, path, jsonType.StringValue()),
|
|
})
|
|
}
|
|
}
|
|
|
|
err := key.SetJSONAccessPlan(telemetrytypes.JSONColumnMetadata{
|
|
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
|
PromotedColumn: logstelemetryschema.LogsV2BodyPromotedColumn,
|
|
}, types)
|
|
require.NoError(t, err)
|
|
|
|
mockMetadataStore.SetKey(key)
|
|
}
|
|
}
|
|
|
|
return mockMetadataStore
|
|
}
|
|
|
|
func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStatementBuilder, *telemetrytypestest.MockMetadataStore) {
|
|
t.Helper()
|
|
|
|
mockMetadataStore := buildTestTelemetryMetadataStore(t, addIndexes)
|
|
fl := flaggertest.WithUseJSONBody(t, true)
|
|
storage := logstelemetryschema.NewStorage()
|
|
|
|
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, storage, fl, telemetrytypes.SignalLogs)
|
|
|
|
statementBuilder := NewLogQueryStatementBuilder(
|
|
instrumentationtest.New().ToProviderSettings(),
|
|
mockMetadataStore,
|
|
storage,
|
|
aggExprRewriter,
|
|
logstelemetryschema.DefaultFullTextColumn,
|
|
fl,
|
|
nil,
|
|
statementbuilder.Config{SkipResourceFingerprint: statementbuilder.SkipResourceFingerprint{Enabled: false, Threshold: 100000}},
|
|
)
|
|
|
|
return statementBuilder, mockMetadataStore
|
|
}
|