mirror of
https://github.com/SigNoz/signoz.git
synced 2026-09-19 09:50:41 +01:00
9c886be120
The semantic convention family as first call citizen revealed that the
current state of the query builder needs a bit refactoring for long term
maintenance.
The `FieldMapper` and `ConditionBuilder` are now one abstraction
`Storage`.
A storage now answers
- what the compiler cannot know i.e one read per field key (the bare
SQL, the membership present or absent, what an absent row reads, and
whether the read keeps its type or filters only).
- the fallback for a key metadata does not report
- its traits
- and one Condition compilation part.
And we introduce a new type to use in the system, `Resolved`
```
// Resolved is what resolution produces for one key: its meanings, and how
// they came to be. It is the only thing the compilers receive. Compile it
// with the operator and value it was resolved with.
type Resolved struct {
Key *telemetrytypes.TelemetryFieldKey
Fields []*telemetrytypes.LogicalField
// FromFallback: the fields came from the storage's fallback, not from
// metadata matches.
FromFallback bool
// Ambiguous: the matches held several interpretations.
Ambiguous bool
// Skipped: the storage contributes nothing for this key.
Skipped bool
Warnings []string
}
```
The prepared SQL has no changes, where it changed, it specifically made
the expression better by removing the redundant part.
- The prepared SQL remains identical with this refactoring
- No changes to integration tests
Assisted-by: Claude Fable 5.1
254 lines
11 KiB
Go
254 lines
11 KiB
Go
package meterstatementbuilder
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
|
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
|
"github.com/SigNoz/signoz/pkg/statementbuilder/metricsstatementbuilder"
|
|
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
|
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
|
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
|
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
|
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
|
"github.com/SigNoz/signoz/pkg/valuer"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestStatementBuilder(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
requestType qbtypes.RequestType
|
|
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]
|
|
expected qbtypes.Statement
|
|
expectedErr error
|
|
}{
|
|
{
|
|
name: "test_cumulative_rate_sum",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
|
|
Signal: telemetrytypes.SignalMetrics,
|
|
StepInterval: qbtypes.Step{Duration: 24 * time.Hour},
|
|
Aggregations: []qbtypes.MetricAggregation{
|
|
{
|
|
MetricName: "signoz_calls_total",
|
|
Type: metrictypes.SumType,
|
|
Temporality: metrictypes.Cumulative,
|
|
TimeAggregation: metrictypes.TimeAggregationRate,
|
|
SpaceAggregation: metrictypes.SpaceAggregationSum,
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "service.name = 'cartservice'",
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "service.name",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, max(value) AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
|
|
Args: []any{"signoz_calls_total", uint64(1747785600000), uint64(1747983420000), "cartservice", "cumulative", 0},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "test_delta_rate_sum",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
|
|
Signal: telemetrytypes.SignalMetrics,
|
|
StepInterval: qbtypes.Step{Duration: 24 * time.Hour},
|
|
Aggregations: []qbtypes.MetricAggregation{
|
|
{
|
|
MetricName: "signoz_calls_total",
|
|
Type: metrictypes.SumType,
|
|
Temporality: metrictypes.Delta,
|
|
TimeAggregation: metrictypes.TimeAggregationRate,
|
|
SpaceAggregation: metrictypes.SpaceAggregationSum,
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "service.name = 'cartservice'",
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "service.name",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, sum(value)/86400 AS value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
|
|
Args: []any{"signoz_calls_total", uint64(1747872000000), uint64(1747983420000), "cartservice", "delta"},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "test_delta_rate_avg",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
|
|
Signal: telemetrytypes.SignalMetrics,
|
|
StepInterval: qbtypes.Step{Duration: 24 * time.Hour},
|
|
Aggregations: []qbtypes.MetricAggregation{
|
|
{
|
|
MetricName: "signoz_calls_total",
|
|
Type: metrictypes.SumType,
|
|
Temporality: metrictypes.Delta,
|
|
TimeAggregation: metrictypes.TimeAggregationRate,
|
|
SpaceAggregation: metrictypes.SpaceAggregationAvg,
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "service.name = 'cartservice'",
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "service.name",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, sum(value)/86400 AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'service.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
|
|
Args: []any{"signoz_calls_total", uint64(1747872000000), uint64(1747983420000), "cartservice", "delta", 0},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
{
|
|
name: "test_gauge_avg_sum",
|
|
requestType: qbtypes.RequestTypeTimeSeries,
|
|
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
|
|
Signal: telemetrytypes.SignalMetrics,
|
|
StepInterval: qbtypes.Step{Duration: 24 * time.Hour},
|
|
Aggregations: []qbtypes.MetricAggregation{
|
|
{
|
|
MetricName: "system.memory.usage",
|
|
Type: metrictypes.GaugeType,
|
|
Temporality: metrictypes.Unspecified,
|
|
TimeAggregation: metrictypes.TimeAggregationAvg,
|
|
SpaceAggregation: metrictypes.SpaceAggregationSum,
|
|
},
|
|
},
|
|
Filter: &qbtypes.Filter{
|
|
Expression: "host.name = 'big-data-node-1'",
|
|
},
|
|
Limit: 10,
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{
|
|
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
|
Name: "host.name",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
expected: qbtypes.Statement{
|
|
Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(86400)) AS ts, JSONExtractString(labels, 'host.name') AS `__GROUP_BY_KEY_0_host.name`, avg(value) AS per_series_value FROM signoz_meter.distributed_samples AS points WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? AND JSONExtractString(labels, 'host.name') = ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_host.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_host.name`, ts",
|
|
Args: []any{"system.memory.usage", uint64(1747872000000), uint64(1747983420000), "big-data-node-1", "unspecified", 0},
|
|
},
|
|
expectedErr: nil,
|
|
},
|
|
}
|
|
|
|
storage := metricstelemetryschema.NewStorage()
|
|
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
|
keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")
|
|
if err != nil {
|
|
t.Fatalf("failed to load field keys: %v", err)
|
|
}
|
|
mockMetadataStore.KeysMap = keys
|
|
|
|
flagger := flaggertest.New(t)
|
|
|
|
metricStmtBuilder := metricsstatementbuilder.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, storage, flagger)
|
|
|
|
statementBuilder := NewMeterQueryStatementBuilder(
|
|
instrumentationtest.New().ToProviderSettings(),
|
|
mockMetadataStore,
|
|
storage,
|
|
metricStmtBuilder,
|
|
)
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
|
|
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
|
|
|
if c.expectedErr != nil {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), c.expectedErr.Error())
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, c.expected.Query, q.Query)
|
|
require.Equal(t, c.expected.Args, q.Args)
|
|
require.Equal(t, c.expected.Warnings, q.Warnings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGroupByAliasAvoidsColumnCollision(t *testing.T) {
|
|
storage := metricstelemetryschema.NewStorage()
|
|
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
|
keys, err := telemetrytypestest.LoadFieldKeysFromJSON("testdata/keys_map.json")
|
|
require.NoError(t, err)
|
|
mockMetadataStore.KeysMap = keys
|
|
|
|
flagger := flaggertest.New(t)
|
|
|
|
statementBuilder := NewMeterQueryStatementBuilder(
|
|
instrumentationtest.New().ToProviderSettings(),
|
|
mockMetadataStore,
|
|
storage,
|
|
metricsstatementbuilder.NewMetricQueryStatementBuilder(instrumentationtest.New().ToProviderSettings(), mockMetadataStore, storage, flagger),
|
|
)
|
|
|
|
for _, groupBy := range []string{"ts", "value", "fingerprint", "service.name"} {
|
|
t.Run(groupBy, func(t *testing.T) {
|
|
stmt, err := statementBuilder.Build(
|
|
context.Background(),
|
|
valuer.UUID{},
|
|
1747947419000,
|
|
1747983448000,
|
|
qbtypes.RequestTypeTimeSeries,
|
|
qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
|
|
Signal: telemetrytypes.SignalMetrics,
|
|
StepInterval: qbtypes.Step{Duration: 24 * time.Hour},
|
|
Aggregations: []qbtypes.MetricAggregation{
|
|
{
|
|
MetricName: "signoz_calls_total",
|
|
Type: metrictypes.SumType,
|
|
Temporality: metrictypes.Cumulative,
|
|
TimeAggregation: metrictypes.TimeAggregationRate,
|
|
SpaceAggregation: metrictypes.SpaceAggregationSum,
|
|
},
|
|
},
|
|
GroupBy: []qbtypes.GroupByKey{
|
|
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: groupBy}},
|
|
},
|
|
},
|
|
nil,
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
assert.Contains(t, stmt.Query, fmt.Sprintf("`__GROUP_BY_KEY_0_%s`", groupBy))
|
|
assert.NotContains(t, stmt.Query, fmt.Sprintf("`%s`", groupBy),
|
|
"the group-by column must not be selected under the label's own name")
|
|
assert.Equal(t, 1, strings.Count(stmt.Query, " AS ts,"),
|
|
"the step bucket is the only column named ts")
|
|
})
|
|
}
|
|
}
|