mirror of
https://github.com/SigNoz/signoz.git
synced 2026-08-06 05:00:42 +01:00
8eb3f6bc1b
- Embed statementbuilder.Config into querier.Config with mapstructure ",squash"; keys move to querier.skip_resource_fingerprint.* (env SIGNOZ_QUERIER_*). - Drop the standalone statementbuilder section and its config factory. - Pass statementbuilder.Config wholesale into NewLogQueryStatementBuilder.
1355 lines
88 KiB
Go
1355 lines
88 KiB
Go
package logsstatementbuilder
|
|
|
|
import (
|
|
"context"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
|
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
|
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
|
"github.com/SigNoz/signoz/pkg/querybuilder"
|
|
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
|
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
|
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
|
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
|
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
|
"github.com/SigNoz/signoz/pkg/valuer"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const (
|
|
testJSONQueryPrefix = "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE"
|
|
testJSONQuerySuffix = "AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?"
|
|
)
|
|
|
|
type TestExpected struct {
|
|
WhereClause string
|
|
Args []any
|
|
Warnings []string
|
|
}
|
|
|
|
func (t TestExpected) GetQuery() string {
|
|
return strings.Join([]string{testJSONQueryPrefix, t.WhereClause, testJSONQuerySuffix}, " ")
|
|
}
|
|
|
|
func TestJSONStmtBuilder_TimeSeries(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "Group By Simple Path",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Aggregations: []qbtypes.LogAggregation{
|
|
{
|
|
Expression: "count()",
|
|
},
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "user.age",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "WITH __limit_cte AS (SELECT toString(multiIf((dynamicElement(body_v2.`user.age`, 'Int64') IS NOT NULL), toString(dynamicElement(body_v2.`user.age`, 'Int64')), (dynamicElement(body_v2.`user.age`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.age`, 'String'), NULL)) AS `__GROUP_BY_KEY_0_user.age`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_user.age` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf((dynamicElement(body_v2.`user.age`, 'Int64') IS NOT NULL), toString(dynamicElement(body_v2.`user.age`, 'Int64')), (dynamicElement(body_v2.`user.age`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.age`, 'String'), NULL)) AS `__GROUP_BY_KEY_0_user.age`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_user.age`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_user.age` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_user.age`",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
|
},
|
|
},
|
|
{
|
|
name: "Group By One Array Join",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Aggregations: []qbtypes.LogAggregation{
|
|
{
|
|
Expression: "count()",
|
|
},
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "education[].awards[].type",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "WITH __limit_cte AS (SELECT toString(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))) AS `__GROUP_BY_KEY_0_education[].awards[].type`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_education[].awards[].type` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))) AS `__GROUP_BY_KEY_0_education[].awards[].type`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_education[].awards[].type`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_education[].awards[].type` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_education[].awards[].type`",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
/* Promoted path tests commented out — Materialized now means type hint (direct sub-column),
|
|
not a body_promoted.* column. These tests assumed the old coalesce(body_promoted.x, body_v2.x) path.
|
|
|
|
func TestStmtBuilderTimeSeriesBodyGroupByPromoted(t *testing.T) {
|
|
statementBuilder := buildJSONTestStatementBuilder(t, "user.age", "user.name")
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "Two simple paths",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Aggregations: []qbtypes.LogAggregation{
|
|
{
|
|
Expression: "count()",
|
|
},
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "user.age",
|
|
},
|
|
},
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "user.name",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Args: []any{ "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
*/
|
|
|
|
func TestJSONStmtBuilder_PrimitivePaths(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
cases := []struct {
|
|
name string
|
|
filter string
|
|
expected TestExpected
|
|
expectedErr error
|
|
}{
|
|
// ── intrinsic sub-column (body_v2.message) ──────────────────────────────
|
|
{
|
|
name: "message Exists",
|
|
filter: "message Exists",
|
|
expected: TestExpected{
|
|
WhereClause: "body_v2.message <> ''",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "message Contains 'Iron Award'",
|
|
filter: "message Contains 'Iron Award'",
|
|
expected: TestExpected{
|
|
WhereClause: "LOWER(body_v2.message) LIKE LOWER(?)",
|
|
Args: []any{"%Iron Award%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── flat nested primitive (x.y) ─────────────────────────────────────────
|
|
{
|
|
name: "Simple string filter",
|
|
filter: "body.user.name = 'x'",
|
|
expected: TestExpected{
|
|
WhereClause: "((dynamicElement(body_v2.`user.name`, 'String') = ?) AND has(JSONAllPaths(body_v2), 'user.name'))",
|
|
Args: []any{"x", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Simple Int field Contains value",
|
|
filter: "body.user.age Contains 25",
|
|
expected: TestExpected{
|
|
WhereClause: "(((LOWER(toString(dynamicElement(body_v2.`user.age`, 'Int64'))) LIKE LOWER(?)) AND has(JSONAllPaths(body_v2), 'user.age')) OR ((LOWER(dynamicElement(body_v2.`user.age`, 'String')) LIKE LOWER(?)) AND has(JSONAllPaths(body_v2), 'user.age')))",
|
|
Args: []any{"%25%", "%25%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `user.age` is ambiguous, found 2 different combinations of field context / data type: [name=user.age,context=body,datatype=int64 name=user.age,context=body,datatype=string]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Simple Float field Contains value",
|
|
filter: "body.user.height Contains 5.8",
|
|
expected: TestExpected{
|
|
WhereClause: "((LOWER(toString(dynamicElement(body_v2.`user.height`, 'Float64'))) LIKE LOWER(?)) AND has(JSONAllPaths(body_v2), 'user.height'))",
|
|
Args: []any{"%5.8%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Float GT",
|
|
filter: "body.user.height > 5.8",
|
|
expected: TestExpected{
|
|
WhereClause: "((toFloat64(dynamicElement(body_v2.`user.height`, 'Float64')) > ?) AND has(JSONAllPaths(body_v2), 'user.height'))",
|
|
Args: []any{5.8, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "String Like",
|
|
filter: "body.user.name LIKE '%ali%'",
|
|
expected: TestExpected{
|
|
WhereClause: "((dynamicElement(body_v2.`user.name`, 'String') LIKE ?) AND has(JSONAllPaths(body_v2), 'user.name'))",
|
|
Args: []any{"%ali%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "String Regexp",
|
|
filter: "body.user.name REGEXP 'al.*'",
|
|
expected: TestExpected{
|
|
WhereClause: "((match(dynamicElement(body_v2.`user.name`, 'String'), ?)) AND has(JSONAllPaths(body_v2), 'user.name'))",
|
|
Args: []any{"al.*", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive Bool NotEqual",
|
|
filter: "body.user.active != true",
|
|
expected: TestExpected{
|
|
WhereClause: "(assumeNotNull(dynamicElement(body_v2.`user.active`, 'Bool')) <> ?)",
|
|
Args: []any{true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive Bool NotExists",
|
|
filter: "body.user.active NOT EXISTS",
|
|
expected: TestExpected{
|
|
WhereClause: "(dynamicElement(body_v2.`user.active`, 'Bool') IS NULL)",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive String NotContains",
|
|
filter: "body.user.name NOT CONTAINS 'IIT'",
|
|
expected: TestExpected{
|
|
WhereClause: "(LOWER(assumeNotNull(dynamicElement(body_v2.`user.name`, 'String'))) NOT LIKE LOWER(?))",
|
|
Args: []any{"%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive String NotLike",
|
|
filter: "body.user.name NOT LIKE '%ali%'",
|
|
expected: TestExpected{
|
|
WhereClause: "(assumeNotNull(dynamicElement(body_v2.`user.name`, 'String')) NOT LIKE ?)",
|
|
Args: []any{"%ali%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive String NotRegexp",
|
|
filter: "body.user.name NOT REGEXP 'al.*'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT match(assumeNotNull(dynamicElement(body_v2.`user.name`, 'String')), ?))",
|
|
Args: []any{"al.*", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Primitive String NotIn",
|
|
filter: "body.user.name NOT IN ['alice', 'bob']",
|
|
expected: TestExpected{
|
|
WhereClause: "(assumeNotNull(dynamicElement(body_v2.`user.name`, 'String')) NOT IN (?, ?))",
|
|
Args: []any{"alice", "bob", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── deeply nested primitive (x.y.z) ─────────────────────────────────────
|
|
{
|
|
name: "Primitive Int64 NotBetween",
|
|
filter: "body.user.address.zip NOT BETWEEN 100000 AND 999999",
|
|
expected: TestExpected{
|
|
WhereClause: "(toFloat64(assumeNotNull(dynamicElement(body_v2.`user.address.zip`, 'Int64'))) NOT BETWEEN ? AND ?)",
|
|
Args: []any{float64(100000), float64(999999), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, qbtypes.RequestTypeRaw,
|
|
qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: c.filter},
|
|
Limit: 10,
|
|
}, nil)
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.GetQuery(), q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
})
|
|
}
|
|
}
|
|
|
|
/* Promoted path list-query tests commented out — Materialized now means type hint
|
|
(direct sub-column access), not a body_promoted.* column.
|
|
|
|
func TestStatementBuilderListQueryBodyPromoted(t *testing.T) {
|
|
statementBuilder := buildJSONTestStatementBuilder(t, "education", "tags")
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected TestExpected
|
|
expectedErr error
|
|
}{
|
|
{
|
|
name: "Has Array promoted uses body fallback",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "has(body.tags, 'production')"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) exists",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].name Exists"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) -> Array(Dynamic) + Array(JSON) exists",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].awards[].name Exists"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) -> Array(Dynamic) + Array(JSON) = 'Iron Award'",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].awards[].name = 'Iron Award'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "Iron Award", "Iron Award", "Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) contains value",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].parameters Contains 1.65"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{uint64(1747945619), uint64(1747983448), "%1.65%", 1.65, "%1.65%", 1.65, "%1.65%", 1.65, "%1.65%", 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true name=education[].parameters,context=body,datatype=[]dynamic,materialized=true]."},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) contains String value",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].name Contains 'IIT'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "%IIT%", "%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Dynamic array contains boolean",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].parameters Contains true"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{uint64(1747945619), uint64(1747983448), "%true%", true, "%true%", true, "%true%", true, "%true%", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true name=education[].parameters,context=body,datatype=[]dynamic,materialized=true]."},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Dynamic array contains String",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].parameters Contains 'passed'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{uint64(1747945619), uint64(1747983448), "%passed%", "passed", "%passed%", "passed", "%passed%", "passed", "%passed%", "passed", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64,materialized=true name=education[].parameters,context=body,datatype=[]dynamic,materialized=true]."},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Array(Dynamic) + Array(JSON) both can contain a value",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].awards[].type = 'sports'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "sports", "sports", "sports", "sports", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "Multi nested Array(Dynamic) + Array(JSON) both can contain a value",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.education[].awards[].participated[].team[].branch Contains 'Civil'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
Args: []any{ "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "%Civil%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.GetQuery(), q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
*/
|
|
|
|
func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
cases := []struct {
|
|
name string
|
|
filter string
|
|
expected TestExpected
|
|
expectedErr error
|
|
}{
|
|
// ── single-hop (x[].y) ──────────────────────────────────────────────────
|
|
{
|
|
name: "Key inside Array(JSON) exists",
|
|
filter: "body.education[].name Exists",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) contains String value",
|
|
filter: "body.education[].name Contains 'IIT'",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> LOWER(dynamicElement(`body_v2.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{"%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Int in array Contains value",
|
|
filter: "body.education[].year Contains 2020",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> LOWER(toString(dynamicElement(`body_v2.education`.`year`, 'Int64'))) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{"%2020%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Array simple path NotEqual",
|
|
filter: "body.education[].type != '10001'",
|
|
expected: TestExpected{
|
|
WhereClause: "((NOT arrayExists(`body_v2.education`-> toFloat64OrNull(dynamicElement(`body_v2.education`.`type`, 'String')) = ?, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND (NOT arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`type`, 'Int64') = ?, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))",
|
|
Args: []any{int64(10001), int64(10001), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].type` is ambiguous, found 2 different combinations of field context / data type: [name=education[].type,context=body,datatype=string name=education[].type,context=body,datatype=int64]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Array simple path NotExists",
|
|
filter: "body.education[].name NOT EXISTS",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') IS NOT NULL, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Array simple path NotContains",
|
|
filter: "body.education[].name NOT CONTAINS 'IIT'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> LOWER(dynamicElement(`body_v2.education`.`name`, 'String')) LIKE LOWER(?), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"%IIT%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── single-hop terminal array (x[].y[]) ─────────────────────────────────
|
|
{
|
|
name: "Key inside Array(JSON) contains Float value",
|
|
filter: "body.education[].parameters Contains 1.65",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')) OR ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))) OR arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')))",
|
|
Args: []any{"%1.65%", 1.65, "%1.65%", 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Dynamic array element comparison",
|
|
filter: "body.education[].parameters = 'passed'",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.education`-> arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')) OR ((arrayExists(`body_v2.education`-> arrayExists(x -> toString(x) = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')))",
|
|
Args: []any{"passed", "passed", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Dynamic array contains String",
|
|
filter: "body.education[].parameters Contains 'passed'",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toString(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')) OR ((arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))) OR arrayExists(x -> toString(x) = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')))",
|
|
Args: []any{"%passed%", "passed", "%passed%", "passed", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Dynamic array IN Operator",
|
|
filter: "body.education[].parameters IN [1.65, 1.99]",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) IN (?, ?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')) OR ((arrayExists(`body_v2.education`-> arrayExists(x -> toString(x) IN (?, ?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education')))",
|
|
Args: []any{1.65, 1.99, "1.65", "1.99", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Terminal array NotContains float",
|
|
filter: "body.education[].parameters NOT CONTAINS 1.65",
|
|
expected: TestExpected{
|
|
WhereClause: "((NOT arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND (NOT arrayExists(`body_v2.education`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))) OR arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)')))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))",
|
|
Args: []any{"%1.65%", float64(1.65), "%1.65%", float64(1.65), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Simple has filter",
|
|
filter: "has(body.education[].parameters, 1.65)",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`parameters`, 'Array(Nullable(Float64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.education`.`parameters`, 'Array(Dynamic)'))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{1.65, 1.65, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{
|
|
"Key `education[].parameters` is ambiguous, found 2 different combinations of field context / data type: [name=education[].parameters,context=body,datatype=[]float64 name=education[].parameters,context=body,datatype=[]dynamic].",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "Flat path hasAll filter",
|
|
filter: "hasAll(body.user.permissions, ['read', 'write'])",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(x -> x = ?, dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))')) AND arrayExists(x -> x = ?, dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))')))",
|
|
Args: []any{"read", "write", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── non-array intermediate hop (x[].y.z) ────────────────────────────────
|
|
{
|
|
name: "Non-array intermediate NotEqual",
|
|
filter: "body.http-events[].request-info.host != 'example.com'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.http-events`-> dynamicElement(`body_v2.http-events`.`request-info.host`, 'String') = ?, dynamicElement(body_v2.`http-events`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"example.com", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── double-hop (x[].y[].z) ──────────────────────────────────────────────
|
|
{
|
|
name: "Key inside Array(JSON) -> Array(Dynamic) + Array(JSON) exists",
|
|
filter: "body.education[].awards[].name Exists",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Key inside Array(JSON) -> Array(Dynamic) + Array(JSON) = 'Iron Award'",
|
|
filter: "body.education[].awards[].name = 'Iron Award'",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{"Iron Award", "Iron Award", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Integer BETWEEN Operator",
|
|
filter: "education[].awards[].semester BETWEEN 2 AND 4",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) BETWEEN ? AND ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{float64(2), float64(4), float64(2), float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Integer IN Operator",
|
|
filter: "education[].awards[].semester IN [2, 4]",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> toFloat64(dynamicElement(`body_v2.education[].awards`.`semester`, 'Int64')) IN (?, ?), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{float64(2), float64(4), float64(2), float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Nested array NotEqual",
|
|
filter: "body.education[].awards[].type != 'sports'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`type`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"sports", "sports", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Nested array NotExists",
|
|
filter: "body.education[].awards[].name NOT EXISTS",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') IS NOT NULL, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── deep multi-hop (x[].y[].z[].w...) ──────────────────────────────────
|
|
{
|
|
name: "Super nested array contains Int64 value",
|
|
filter: "body.interests[].entities[].reviews[].entries[].metadata[].positions[].ratings Contains 4",
|
|
expected: TestExpected{
|
|
WhereClause: "(((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))')) OR arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(Int64))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'interests')) OR ((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(`body_v2.interests[].entities[].reviews`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata`-> arrayExists(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`-> (arrayExists(x -> LOWER(x) LIKE LOWER(?), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))')) OR arrayExists(x -> toFloat64OrNull(x) = ?, dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata[].positions`.`ratings`, 'Array(Nullable(String))'))), dynamicElement(`body_v2.interests[].entities[].reviews[].entries[].metadata`.`positions`, 'Array(JSON(max_dynamic_types=0, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews[].entries`.`metadata`, 'Array(JSON(max_dynamic_types=1, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities[].reviews`.`entries`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests[].entities`.`reviews`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'interests')))",
|
|
Args: []any{"%4%", float64(4), "%4%", float64(4), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{"Key `interests[].entities[].reviews[].entries[].metadata[].positions[].ratings` is ambiguous, found 2 different combinations of field context / data type: [name=interests[].entities[].reviews[].entries[].metadata[].positions[].ratings,context=body,datatype=[]int64 name=interests[].entities[].reviews[].entries[].metadata[].positions[].ratings,context=body,datatype=[]string]."},
|
|
},
|
|
},
|
|
{
|
|
name: "Multi nested Array(Dynamic) + Array(JSON) both can contain a value",
|
|
filter: "body.education[].awards[].participated[].team[].branch Contains 'Civil'",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=2, max_dynamic_paths=0))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(`body_v2.education[].awards[].participated[].team`-> LOWER(dynamicElement(`body_v2.education[].awards[].participated[].team`.`branch`, 'String')) LIKE LOWER(?), dynamicElement(`body_v2.education[].awards[].participated`.`team`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{"%Civil%", "%Civil%", "%Civil%", "%Civil%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// IN on a single-type Int64 array terminal: any element of scores IN {90, 95}
|
|
name: "Terminal Int64 array IN operator",
|
|
filter: "body.education[].scores IN [90, 95]",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) IN (?, ?), dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{float64(90), float64(95), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// NOT IN on a single-type Int64 array terminal: no element of scores IN {90, 95}.
|
|
// Negative operator → NOT wraps outer arrayExists; no path-index guard added.
|
|
name: "Terminal Int64 array NOT IN operator",
|
|
filter: "body.education[].scores NOT IN [90, 95]",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) IN (?, ?), dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{float64(90), float64(95), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// BETWEEN on a single-type Int64 array terminal: any element of scores BETWEEN 80 AND 95.
|
|
// BETWEEN is an IsArrayOperator → value arrives as []any{80,95} → c.valueType.IsArray=true.
|
|
name: "Terminal Int64 array BETWEEN operator",
|
|
filter: "body.education[].scores BETWEEN 80 AND 95",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'education'))",
|
|
Args: []any{float64(80), float64(95), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// NOT BETWEEN on a single-type Int64 array terminal: no element BETWEEN 80 AND 95.
|
|
// Negative → NOT wraps outer arrayExists; no path-index guard.
|
|
name: "Terminal Int64 array NOT BETWEEN operator",
|
|
filter: "body.education[].scores NOT BETWEEN 80 AND 95",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) BETWEEN ? AND ?, dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{float64(80), float64(95), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Nested path hasAny filter",
|
|
filter: "hasAny(education[].awards[].participated[].members, ['Piyush', 'Tushar'])",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> x IN (?, ?), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"Piyush", "Tushar", "Piyush", "Tushar", "Piyush", "Tushar", "Piyush", "Tushar", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "dynamic_array_element_compare_CONTAINS",
|
|
filter: "interests[].entities[].product_codes Contains 1",
|
|
expected: TestExpected{
|
|
WhereClause: "((arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> (arrayExists(x -> LOWER(toString(x)) LIKE LOWER(?), arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))) OR arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)')))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))) AND has(JSONAllPaths(body_v2), 'interests'))",
|
|
Args: []any{"%1%", float64(1), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "dynamic_array_element_compare_HAS_STRING",
|
|
filter: "has(interests[].entities[].product_codes, '2002')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{int64(2002), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "dynamic_array_element_compare_NOT_EQUAL",
|
|
filter: "interests[].entities[].product_codes != '1'",
|
|
expected: TestExpected{
|
|
WhereClause: "(NOT arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{int64(1), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "dynamic_array_element_compare_HAS_INT",
|
|
filter: "has(interests[].entities[].product_codes, 1001)",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.interests`-> arrayExists(`body_v2.interests[].entities`-> arrayExists(x -> accurateCastOrNull(x, 'Float64') = ?, arrayFilter(x->(dynamicType(x) IN ('String', 'Int64', 'Float64', 'Bool')), dynamicElement(`body_v2.interests[].entities`.`product_codes`, 'Array(Dynamic)'))), dynamicElement(`body_v2.interests`.`entities`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), dynamicElement(body_v2.`interests`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{float64(1001), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── scalar leaf reached through an array ───
|
|
{
|
|
name: "Nested primitive leaf has",
|
|
filter: "has(body.education[].name, 'IIT')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"IIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Nested primitive leaf hasAny",
|
|
filter: "hasAny(body.education[].name, ['IIT', 'MIT'])",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') IN (?, ?), false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"IIT", "MIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// contains-all: single-value hasAll over a nested leaf collapses to has.
|
|
name: "Nested primitive leaf hasAll single collapses to has",
|
|
filter: "hasAll(body.education[].name, 'IIT')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"IIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// contains-all: "some element is IIT AND some element is MIT" (AND of per-value has).
|
|
name: "Nested primitive leaf hasAll multi (contains-all)",
|
|
filter: "hasAll(body.education[].name, ['IIT', 'MIT'])",
|
|
expected: TestExpected{
|
|
WhereClause: "(arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) AND arrayExists(`body_v2.education`-> ifNull(dynamicElement(`body_v2.education`.`name`, 'String') = ?, false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))",
|
|
Args: []any{"IIT", "MIT", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── numeric literal against an Int64 array is collision-handled ───
|
|
{
|
|
name: "Nested Int64 array has collision",
|
|
filter: "has(body.education[].scores, 90)",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> arrayExists(x -> toFloat64(x) = ?, dynamicElement(`body_v2.education`.`scores`, 'Array(Nullable(Int64))')), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{float64(90), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── hasAny folds multiple scalar arguments into one value set ─────
|
|
{
|
|
name: "hasAny folds multiple scalar args",
|
|
filter: "hasAny(body.user.permissions, 'read', 'write')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(x -> x IN (?, ?), dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'))",
|
|
Args: []any{"read", "write", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── hasToken over JSON string fields (nested leaf, top-level array, nested array) ──
|
|
{
|
|
name: "hasToken nested string leaf",
|
|
filter: "hasToken(body.education[].name, 'harvard')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> ifNull(hasToken(LOWER(dynamicElement(`body_v2.education`.`name`, 'String')), LOWER(?)), false), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"harvard", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "hasToken top-level string array",
|
|
filter: "hasToken(body.user.permissions, 'admin')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'))",
|
|
Args: []any{"admin", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "hasToken nested string array",
|
|
filter: "hasToken(body.education[].awards[].participated[].members, 'piyush')",
|
|
expected: TestExpected{
|
|
WhereClause: "arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> (arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards[].participated`-> arrayExists(x -> hasToken(LOWER(x), LOWER(?)), dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))')), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))",
|
|
Args: []any{"piyush", "piyush", "piyush", "piyush", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── hasToken over the message field: bare body and explicit body.message are
|
|
// equivalent, both target the body.message column directly (no dynamicElement) ──
|
|
{
|
|
name: "hasToken bare body",
|
|
filter: "hasToken(body, 'production')",
|
|
expected: TestExpected{
|
|
WhereClause: "hasToken(LOWER(body.message), LOWER(?))",
|
|
Args: []any{"production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
Warnings: []string{bodySearchDefaultWarning},
|
|
},
|
|
},
|
|
{
|
|
name: "hasToken explicit body.message",
|
|
filter: "hasToken(body.message, 'production')",
|
|
expected: TestExpected{
|
|
WhereClause: "hasToken(LOWER(body.message), LOWER(?))",
|
|
Args: []any{"production", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── scalar (non-array) leaf: treated as a single-element set ─────────────
|
|
{
|
|
name: "Scalar leaf has",
|
|
filter: "has(body.user.name, 'alice')",
|
|
expected: TestExpected{
|
|
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false)",
|
|
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Scalar leaf hasAny",
|
|
filter: "hasAny(body.user.name, ['alice', 'bob'])",
|
|
expected: TestExpected{
|
|
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') IN (?, ?), false)",
|
|
Args: []any{"alice", "bob", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// contains-all: single-value hasAll over a plain scalar collapses to has.
|
|
name: "Scalar leaf hasAll single collapses to has",
|
|
filter: "hasAll(body.user.name, 'alice')",
|
|
expected: TestExpected{
|
|
WhereClause: "ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false)",
|
|
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
// contains-all over a one-element set: the scalar must equal every value, so
|
|
// distinct values never match.
|
|
name: "Scalar leaf hasAll multi (contains-all)",
|
|
filter: "hasAll(body.user.name, ['alice', 'bob'])",
|
|
expected: TestExpected{
|
|
WhereClause: "(ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false) AND ifNull(dynamicElement(body_v2.`user.name`, 'String') = ?, false))",
|
|
Args: []any{"alice", "bob", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, qbtypes.RequestTypeRaw,
|
|
qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: c.filter},
|
|
Limit: 10,
|
|
}, nil)
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.GetQuery(), q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestJSONStmtBuilder_IndexedPaths(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, true)
|
|
cases := []struct {
|
|
name string
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected TestExpected
|
|
expectedErr error
|
|
}{
|
|
// ── indexed equal: emits assumeNotNull(dynamicElement) = ? ───────────
|
|
{
|
|
name: "Indexed String Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.name = 'alice'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "((lower(assumeNotNull(dynamicElement(body_v2.user.name, 'String'))) = ?) AND has(JSONAllPaths(body_v2), 'user.name'))",
|
|
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Indexed Int64 Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.address.zip = 110001"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "((toFloat64(assumeNotNull(dynamicElement(body_v2.user.address.zip, 'Int64'))) = ?) AND has(JSONAllPaths(body_v2), 'user.address.zip'))",
|
|
Args: []any{float64(110001), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── indexed exists: index is skipped; emits plain IS NOT NULL ───────────
|
|
// EXISTS/NOT EXISTS bypass the index because indexed columns store a default
|
|
// empty value for absent fields, making != "" unreliable for existence checks
|
|
// (a field with value "" would be incorrectly excluded).
|
|
{
|
|
name: "Indexed String Exists",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.name Exists"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "(dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL)",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── indexed not-equal: assumeNotNull wrapping + no path index ─────────
|
|
{
|
|
name: "Indexed String NotEqual",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.name != 'alice'"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "(lower(assumeNotNull(dynamicElement(body_v2.user.name, 'String'))) <> ?)",
|
|
Args: []any{"alice", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
|
|
// ── special-character indexed paths ───────────────────────────────────
|
|
{
|
|
name: "Indexed hyphen key Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.http-status = 200"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "(((toFloat64(assumeNotNull(dynamicElement(body_v2.`http-status`, 'Int64'))) = ?) AND has(JSONAllPaths(body_v2), 'http-status')) OR ((toFloat64OrNull(lower(assumeNotNull(dynamicElement(body_v2.`http-status`, 'String')))) = ?) AND has(JSONAllPaths(body_v2), 'http-status')))",
|
|
Args: []any{float64(200), float64(200), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "Indexed nested hyphen key Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.response.time-taken > 1.5"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "((toFloat64(assumeNotNull(dynamicElement(body_v2.response.`time-taken`, 'Float64'))) > ?) AND has(JSONAllPaths(body_v2), 'response.time-taken'))",
|
|
Args: []any{float64(1.5), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
// ── Bool is NOT IndexSupported — index branch must be skipped ─────────
|
|
// user.active is Bool; even with indexed builder it must not use the index.
|
|
{
|
|
name: "Bool skips index — Equal",
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
Filter: &qbtypes.Filter{Expression: "body.user.active = true"},
|
|
Limit: 10,
|
|
},
|
|
expected: TestExpected{
|
|
WhereClause: "((dynamicElement(body_v2.`user.active`, 'Bool') = ?) AND has(JSONAllPaths(body_v2), 'user.active'))",
|
|
Args: []any{true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, qbtypes.RequestTypeRaw, c.query, nil)
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.GetQuery(), q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestJSONStmtBuilder_SelectField(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "select_x_education[].awards[].participated[].members",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
SelectFields: []telemetrytypes.TelemetryFieldKey{
|
|
{
|
|
Name: "education[].awards[].participated[].members",
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "user.name exists",
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `__SELECT_KEY_0_education[].awards[].participated[].members` FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "select_x_education[].awards[].type",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
SelectFields: []telemetrytypes.TelemetryFieldKey{
|
|
{
|
|
Name: "education[].awards[].type",
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "user.name exists",
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `__SELECT_KEY_0_education[].awards[].type` FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "select_x_user.name",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
SelectFields: []telemetrytypes.TelemetryFieldKey{
|
|
{
|
|
Name: "user.name",
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, multiIf((dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.name`, 'String'), NULL) AS `__SELECT_KEY_0_user.name` FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestJSONStmtBuilder_OrderBy(t *testing.T) {
|
|
statementBuilder, _ := buildJSONTestStatementBuilder(t, false)
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "order_by_education[].awards[].participated[].members",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
Order: []qbtypes.OrderBy{
|
|
{
|
|
Key: qbtypes.OrderByKey{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "education[].awards[].participated[].members",
|
|
},
|
|
},
|
|
Direction: qbtypes.OrderDirectionAsc,
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "user.name exists",
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) asc LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
{
|
|
name: "order_by_user.name",
|
|
requestType: qbtypes.RequestTypeRaw,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
Limit: 10,
|
|
Order: []qbtypes.OrderBy{
|
|
{
|
|
Key: qbtypes.OrderByKey{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "user.name",
|
|
},
|
|
},
|
|
Direction: qbtypes.OrderDirectionAsc,
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY multiIf((dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.name`, 'String'), NULL) asc LIMIT ?",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) {
|
|
statementBuilder, metadataStore := buildJSONTestStatementBuilder(t, false)
|
|
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
|
keysMap := logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
|
for _, keys := range keysMap {
|
|
for _, key := range keys {
|
|
metadataStore.SetKey(key)
|
|
}
|
|
}
|
|
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErrContains string
|
|
}{
|
|
{
|
|
name: "resource_aggregation_and_group_by_with_json_enabled",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
|
Signal: telemetrytypes.SignalLogs,
|
|
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "region",
|
|
},
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "user.name exists",
|
|
},
|
|
Aggregations: []qbtypes.LogAggregation{
|
|
{
|
|
Expression: "count_distinct(service.name)",
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`region` IS NOT NULL, resource.`region`::String, NULL)) AS `__GROUP_BY_KEY_0_region`, countDistinct(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) OR mapContains(attributes_string, 'user.name')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, `__GROUP_BY_KEY_0_region`",
|
|
Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
|
Warnings: []string{"Key `user.name` is ambiguous, found 2 different combinations of field context / data type: [name=user.name,context=body,datatype=string name=user.name,context=attribute,datatype=string]."},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
if c.expectedErrContains != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErrContains)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetrytypestest.MockMetadataStore {
|
|
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
|
mockMetadataStore.SetStaticFields(logstelemetryschema.IntrinsicFields)
|
|
types, _ := telemetrytypes.TestJSONTypeSet()
|
|
for path, fieldDataTypes := range types {
|
|
for _, fdt := range fieldDataTypes {
|
|
key := &telemetrytypes.TelemetryFieldKey{
|
|
Name: path,
|
|
Signal: telemetrytypes.SignalLogs,
|
|
FieldContext: telemetrytypes.FieldContextBody,
|
|
FieldDataType: fdt,
|
|
}
|
|
if addIndexes {
|
|
jsonType := telemetrytypes.MappingFieldDataTypeToJSONDataType[fdt]
|
|
idx := slices.IndexFunc(telemetrytypes.TestIndexedPaths, func(entry telemetrytypes.TestIndexedPathEntry) bool {
|
|
return entry.Path == path && entry.Type == jsonType
|
|
})
|
|
if idx >= 0 {
|
|
key.Indexes = append(key.Indexes, telemetrytypes.TelemetryFieldKeySkipIndex{
|
|
Name: path,
|
|
FieldContext: telemetrytypes.FieldContextBody,
|
|
FieldDataType: fdt,
|
|
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
|
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(logstelemetryschema.LogsV2BodyV2Column, path, jsonType.StringValue()),
|
|
})
|
|
}
|
|
}
|
|
|
|
err := key.SetJSONAccessPlan(telemetrytypes.JSONColumnMetadata{
|
|
BaseColumn: logstelemetryschema.LogsV2BodyV2Column,
|
|
PromotedColumn: logstelemetryschema.LogsV2BodyPromotedColumn,
|
|
}, types)
|
|
require.NoError(t, err)
|
|
|
|
mockMetadataStore.SetKey(key)
|
|
}
|
|
}
|
|
|
|
return mockMetadataStore
|
|
}
|
|
|
|
func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStatementBuilder, *telemetrytypestest.MockMetadataStore) {
|
|
t.Helper()
|
|
|
|
mockMetadataStore := buildTestTelemetryMetadataStore(t, addIndexes)
|
|
fl := flaggertest.WithUseJSONBody(t, true)
|
|
fm := logstelemetryschema.NewFieldMapper(fl)
|
|
cb := logstelemetryschema.NewConditionBuilder(fm, fl)
|
|
|
|
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
|
|
|
statementBuilder := NewLogQueryStatementBuilder(
|
|
instrumentationtest.New().ToProviderSettings(),
|
|
mockMetadataStore,
|
|
fm,
|
|
cb,
|
|
aggExprRewriter,
|
|
logstelemetryschema.DefaultFullTextColumn,
|
|
fl,
|
|
nil,
|
|
statementbuilder.Config{SkipResourceFingerprint: statementbuilder.SkipResourceFingerprint{Enabled: false, Threshold: 100000}},
|
|
)
|
|
|
|
return statementBuilder, mockMetadataStore
|
|
}
|