mirror of
https://github.com/SigNoz/signoz.git
synced 2026-07-07 15:10:37 +01:00
Compare commits
5 Commits
issue-5535
...
tvats-quer
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d5f30ab8f | ||
|
|
04935be5bf | ||
|
|
27e71b9f79 | ||
|
|
ead26b15c7 | ||
|
|
a07a69ddb5 |
8
.github/workflows/integrationci.yaml
vendored
8
.github/workflows/integrationci.yaml
vendored
@@ -48,7 +48,11 @@ jobs:
|
||||
- logspipelines
|
||||
- passwordauthn
|
||||
- preference
|
||||
- querier
|
||||
- querierlogs
|
||||
- queriertraces
|
||||
- queriermetrics
|
||||
- querierscalar
|
||||
- queriercommon
|
||||
- rawexportdata
|
||||
- role
|
||||
- rootuser
|
||||
@@ -56,8 +60,6 @@ jobs:
|
||||
- querier_json_body
|
||||
- querier_skip_resource_fingerprint
|
||||
- ttl
|
||||
- clickhousecluster
|
||||
- metricreduction
|
||||
sqlstore-provider:
|
||||
- postgres
|
||||
- sqlite
|
||||
|
||||
@@ -39,48 +39,48 @@ func TestReducedStatementBuilder(t *testing.T) {
|
||||
name: "gauge_sum_latest",
|
||||
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationLatest, metrictypes.SpaceAggregationSum),
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, anyLast(last) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, argMax(value, unix_milli) AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum_last`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, anyLast(last) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, argMax(value, unix_milli) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gauge_avg_avg",
|
||||
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum_last`, points.computed_at) AS value, argMax(`count_series`, points.computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gauge_min_min",
|
||||
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMin, metrictypes.SpaceAggregationMin),
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(min) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(value) AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`min`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(min) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(value) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`min`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gauge_max_max",
|
||||
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMax, metrictypes.SpaceAggregationMax),
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(value) AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`max`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(value) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`max`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "counter_sum_rate",
|
||||
query: reducedQuery("test.metric.sum", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationRate, metrictypes.SpaceAggregationSum),
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) / 300 AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric.sum", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), 0, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), false, "test.metric.sum", uint64(1746999600000), uint64(1747172760000)},
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) / 300 AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric.sum", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), 0, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), "test.metric.sum", uint64(1746999600000), uint64(1747172760000), false},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "counter_avg_increase",
|
||||
query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationIncrease, metrictypes.SpaceAggregationAvg),
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum`, points.computed_at) AS value, argMax(`count_series`, points.computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0, "test.metric", uint64(1746999600000), uint64(1747172760000), false, "test.metric", uint64(1746999600000), uint64(1747172760000)},
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0, "test.metric", uint64(1746999600000), uint64(1747172760000), "test.metric", uint64(1746999600000), uint64(1747172760000), false},
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -103,16 +103,16 @@ func TestReducedStatementBuilder(t *testing.T) {
|
||||
name: "histogram_p99",
|
||||
query: reducedQuery("test.metric.bucket", metrictypes.HistogramType, metrictypes.Cumulative, metrictypes.TimeAggregationUnspecified, metrictypes.SpaceAggregationPercentile99),
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(value) / 300 AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, `le`, argMax(`sum`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli, `le`) GROUP BY fingerprint, ts, `le`), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), false, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000)},
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(value) / 300 AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts, `le`), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), false},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "summary_avg",
|
||||
query: reducedQuery("test.metric", metrictypes.SummaryType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum_last`, points.computed_at) AS value, argMax(`count_series`, points.computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
|
||||
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
|
||||
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -337,28 +337,20 @@ func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
}
|
||||
|
||||
// dedup recomputed buckets: latest computed_at wins per (series, 60s bucket)
|
||||
// TODO(srikanthccv): add _5m/_30m tables similar to samples_v4
|
||||
// and wrie them up in querier before GA
|
||||
// TODO(srikanthccv): FINAL clause for the reduced table.
|
||||
dedup := sqlbuilder.NewSelectBuilder()
|
||||
dedup.Select("points.reduced_fingerprint AS fingerprint", "points.unix_milli AS unix_milli")
|
||||
for _, g := range query.GroupBy {
|
||||
dedup.SelectMore(fmt.Sprintf("`%s`", g.Name))
|
||||
}
|
||||
dedup.SelectMore(fmt.Sprintf("argMax(%s, points.computed_at) AS value", value))
|
||||
dedup.Select("reduced_fingerprint AS fingerprint", "unix_milli")
|
||||
dedup.SelectMore(fmt.Sprintf("argMax(%s, computed_at) AS value", value))
|
||||
if weight != "" {
|
||||
dedup.SelectMore(fmt.Sprintf("argMax(%s, points.computed_at) AS weight", weight))
|
||||
dedup.SelectMore(fmt.Sprintf("argMax(%s, computed_at) AS weight", weight))
|
||||
}
|
||||
dedup.From(fmt.Sprintf("%s.%s AS points", DBName, WhichReducedSamplesTableToUse(agg.Type)))
|
||||
dedup.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
dedup.From(fmt.Sprintf("%s.%s", DBName, WhichReducedSamplesTableToUse(agg.Type)))
|
||||
dedup.Where(
|
||||
dedup.In("metric_name", agg.MetricName),
|
||||
dedup.GTE("unix_milli", start),
|
||||
dedup.LT("unix_milli", end),
|
||||
)
|
||||
dedup.GroupBy("fingerprint", "unix_milli")
|
||||
dedup.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
|
||||
dedupQuery, dedupArgs := dedup.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)
|
||||
dedup.GroupBy("reduced_fingerprint", "unix_milli")
|
||||
dedupQuery, dedupArgs := dedup.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("fingerprint")
|
||||
@@ -372,11 +364,13 @@ func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
// denominator is reduced with avg
|
||||
sb.SelectMore("avg(weight) AS per_series_weight")
|
||||
}
|
||||
sb.From(fmt.Sprintf("(%s)", dedupQuery))
|
||||
sb.From(fmt.Sprintf("(%s) AS points", dedupQuery))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.GroupBy("fingerprint", "ts")
|
||||
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
|
||||
|
||||
q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, dedupArgs...)
|
||||
initArgs := append(append([]any{}, dedupArgs...), timeSeriesCTEArgs...)
|
||||
q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, initArgs...)
|
||||
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, true
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ pytest_plugins = [
|
||||
"fixtures.postgres",
|
||||
"fixtures.sql",
|
||||
"fixtures.sqlite",
|
||||
"fixtures.keeper",
|
||||
"fixtures.zookeeper",
|
||||
"fixtures.signoz",
|
||||
"fixtures.audit",
|
||||
"fixtures.logs",
|
||||
@@ -80,6 +80,12 @@ def pytest_addoption(parser: pytest.Parser):
|
||||
default="25.5.6",
|
||||
help="clickhouse version",
|
||||
)
|
||||
parser.addoption(
|
||||
"--zookeeper-version",
|
||||
action="store",
|
||||
default="3.7.1",
|
||||
help="zookeeper version",
|
||||
)
|
||||
parser.addoption(
|
||||
"--schema-migrator-version",
|
||||
action="store",
|
||||
|
||||
425
tests/fixtures/clickhouse.py
vendored
425
tests/fixtures/clickhouse.py
vendored
@@ -2,7 +2,6 @@ import os
|
||||
from collections.abc import Callable, Generator
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import clickhouse_connect
|
||||
import clickhouse_connect.driver
|
||||
@@ -18,88 +17,30 @@ from fixtures.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
CLICKHOUSE_USERNAME = "signoz"
|
||||
CLICKHOUSE_PASSWORD = "password"
|
||||
|
||||
CUSTOM_FUNCTION_CONFIG = """
|
||||
<functions>
|
||||
<function>
|
||||
<type>executable</type>
|
||||
<name>histogramQuantile</name>
|
||||
<return_type>Float64</return_type>
|
||||
<argument>
|
||||
<type>Array(Float64)</type>
|
||||
<name>buckets</name>
|
||||
</argument>
|
||||
<argument>
|
||||
<type>Array(Float64)</type>
|
||||
<name>counts</name>
|
||||
</argument>
|
||||
<argument>
|
||||
<type>Float64</type>
|
||||
<name>quantile</name>
|
||||
</argument>
|
||||
<format>CSV</format>
|
||||
<command>./histogramQuantile</command>
|
||||
</function>
|
||||
</functions>
|
||||
"""
|
||||
|
||||
# Distributed inserts to a remote shard are async by default. We force
|
||||
# sycn at the profile level for deterministic tests.
|
||||
CLUSTER_USERS_CONFIG = """
|
||||
<clickhouse>
|
||||
<profiles>
|
||||
<default>
|
||||
<insert_distributed_sync>1</insert_distributed_sync>
|
||||
</default>
|
||||
</profiles>
|
||||
</clickhouse>
|
||||
"""
|
||||
|
||||
|
||||
def render_remote_servers(shard_hosts: list[tuple[str, int]], secret: str | None = None) -> str:
|
||||
"""Render the <remote_servers> block for a cluster named `cluster` with one
|
||||
single-replica shard per (host, port).
|
||||
@pytest.fixture(name="clickhouse", scope="package")
|
||||
def clickhouse(
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
zookeeper: types.TestContainerDocker,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.TestContainerClickhouse:
|
||||
"""
|
||||
Package-scoped fixture for Clickhouse TestContainer.
|
||||
"""
|
||||
shards = "".join(
|
||||
f"""
|
||||
<shard>
|
||||
<replica>
|
||||
<host>{host}</host>
|
||||
<port>{port}</port>
|
||||
</replica>
|
||||
</shard>"""
|
||||
for host, port in shard_hosts
|
||||
)
|
||||
|
||||
# Multi-node clusters need `secret` because distributed queries otherwise
|
||||
# authenticate as the `default` user, which the docker entrypoint restricts
|
||||
# to localhost when a custom user is configured.
|
||||
secret_block = (
|
||||
f"""
|
||||
<secret>{secret}</secret>"""
|
||||
if secret
|
||||
else ""
|
||||
)
|
||||
def create() -> types.TestContainerClickhouse:
|
||||
version = request.config.getoption("--clickhouse-version")
|
||||
|
||||
return f"""
|
||||
<remote_servers>
|
||||
<cluster>{secret_block}{shards}
|
||||
</cluster>
|
||||
</remote_servers>"""
|
||||
container = ClickHouseContainer(
|
||||
image=f"clickhouse/clickhouse-server:{version}",
|
||||
port=9000,
|
||||
username="signoz",
|
||||
password="password",
|
||||
)
|
||||
|
||||
|
||||
def render_node_config(
|
||||
keeper_address: str,
|
||||
keeper_port: int,
|
||||
shard: str,
|
||||
remote_servers: str,
|
||||
distributed_ddl_path: str = "/clickhouse/task_queue/ddl",
|
||||
) -> str:
|
||||
# <zookeeper> is ClickHouse's config section name for any coordination
|
||||
# service, including ClickHouse Keeper.
|
||||
return f"""
|
||||
cluster_config = f"""
|
||||
<clickhouse>
|
||||
<logger>
|
||||
<level>information</level>
|
||||
@@ -114,23 +55,33 @@ def render_node_config(
|
||||
</logger>
|
||||
|
||||
<macros>
|
||||
<shard>{shard}</shard>
|
||||
<shard>01</shard>
|
||||
<replica>01</replica>
|
||||
</macros>
|
||||
|
||||
<zookeeper>
|
||||
<node>
|
||||
<host>{keeper_address}</host>
|
||||
<port>{keeper_port}</port>
|
||||
<host>{zookeeper.container_configs["2181"].address}</host>
|
||||
<port>{zookeeper.container_configs["2181"].port}</port>
|
||||
</node>
|
||||
</zookeeper>
|
||||
{remote_servers}
|
||||
|
||||
<remote_servers>
|
||||
<cluster>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>127.0.0.1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</cluster>
|
||||
</remote_servers>
|
||||
|
||||
<user_defined_executable_functions_config>*function.xml</user_defined_executable_functions_config>
|
||||
<user_scripts_path>/var/lib/clickhouse/user_scripts/</user_scripts_path>
|
||||
|
||||
<distributed_ddl>
|
||||
<path>{distributed_ddl_path}</path>
|
||||
<path>/clickhouse/task_queue/ddl</path>
|
||||
<profile>default</profile>
|
||||
</distributed_ddl>
|
||||
|
||||
@@ -171,66 +122,38 @@ def render_node_config(
|
||||
</clickhouse>
|
||||
"""
|
||||
|
||||
custom_function_config = """
|
||||
<functions>
|
||||
<function>
|
||||
<type>executable</type>
|
||||
<name>histogramQuantile</name>
|
||||
<return_type>Float64</return_type>
|
||||
<argument>
|
||||
<type>Array(Float64)</type>
|
||||
<name>buckets</name>
|
||||
</argument>
|
||||
<argument>
|
||||
<type>Array(Float64)</type>
|
||||
<name>counts</name>
|
||||
</argument>
|
||||
<argument>
|
||||
<type>Float64</type>
|
||||
<name>quantile</name>
|
||||
</argument>
|
||||
<format>CSV</format>
|
||||
<command>./histogramQuantile</command>
|
||||
</function>
|
||||
</functions>
|
||||
"""
|
||||
|
||||
def install_histogram_quantile(container: ClickHouseContainer) -> None:
|
||||
wrapped = container.get_wrapped_container()
|
||||
exit_code, output = wrapped.exec_run(
|
||||
[
|
||||
"bash",
|
||||
"-c",
|
||||
(
|
||||
'version="v0.0.1" && '
|
||||
'node_os=$(uname -s | tr "[:upper:]" "[:lower:]") && '
|
||||
"node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && "
|
||||
"cd /tmp && "
|
||||
'wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" && '
|
||||
"tar -xzf histogram-quantile.tar.gz && "
|
||||
"mkdir -p /var/lib/clickhouse/user_scripts && "
|
||||
"mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile && "
|
||||
"chmod +x /var/lib/clickhouse/user_scripts/histogramQuantile"
|
||||
),
|
||||
],
|
||||
)
|
||||
if exit_code != 0:
|
||||
raise RuntimeError(f"Failed to install histogramQuantile binary: {output.decode()}")
|
||||
|
||||
|
||||
def create_clickhouse( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
keeper: types.TestContainerDocker,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
cache_key: str = "clickhouse",
|
||||
version: str | None = None,
|
||||
) -> types.TestContainerClickhouse:
|
||||
coordinator = next(iter(keeper.container_configs.values()))
|
||||
|
||||
def create() -> types.TestContainerClickhouse:
|
||||
clickhouse_version = version or request.config.getoption("--clickhouse-version")
|
||||
|
||||
container = ClickHouseContainer(
|
||||
image=f"clickhouse/clickhouse-server:{clickhouse_version}",
|
||||
port=9000,
|
||||
username=CLICKHOUSE_USERNAME,
|
||||
password=CLICKHOUSE_PASSWORD,
|
||||
)
|
||||
|
||||
cluster_config = render_node_config(
|
||||
keeper_address=coordinator.address,
|
||||
keeper_port=coordinator.port,
|
||||
shard="01",
|
||||
remote_servers=render_remote_servers([("127.0.0.1", 9000)]),
|
||||
)
|
||||
|
||||
tmp_dir = tmpfs(cache_key)
|
||||
tmp_dir = tmpfs("clickhouse")
|
||||
cluster_config_file_path = os.path.join(tmp_dir, "cluster.xml")
|
||||
with open(cluster_config_file_path, "w", encoding="utf-8") as f:
|
||||
f.write(cluster_config)
|
||||
|
||||
custom_function_file_path = os.path.join(tmp_dir, "custom-function.xml")
|
||||
with open(custom_function_file_path, "w", encoding="utf-8") as f:
|
||||
f.write(CUSTOM_FUNCTION_CONFIG)
|
||||
f.write(custom_function_config)
|
||||
|
||||
container.with_volume_mapping(cluster_config_file_path, "/etc/clickhouse-server/config.d/cluster.xml")
|
||||
container.with_volume_mapping(
|
||||
@@ -240,7 +163,27 @@ def create_clickhouse( # pylint: disable=too-many-arguments,too-many-positional
|
||||
container.with_network(network)
|
||||
container.start()
|
||||
|
||||
install_histogram_quantile(container)
|
||||
# Download and install the histogramQuantile binary
|
||||
wrapped = container.get_wrapped_container()
|
||||
exit_code, output = wrapped.exec_run(
|
||||
[
|
||||
"bash",
|
||||
"-c",
|
||||
(
|
||||
'version="v0.0.1" && '
|
||||
'node_os=$(uname -s | tr "[:upper:]" "[:lower:]") && '
|
||||
"node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && "
|
||||
"cd /tmp && "
|
||||
'wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" && '
|
||||
"tar -xzf histogram-quantile.tar.gz && "
|
||||
"mkdir -p /var/lib/clickhouse/user_scripts && "
|
||||
"mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile && "
|
||||
"chmod +x /var/lib/clickhouse/user_scripts/histogramQuantile"
|
||||
),
|
||||
],
|
||||
)
|
||||
if exit_code != 0:
|
||||
raise RuntimeError(f"Failed to install histogramQuantile binary: {output.decode()}")
|
||||
|
||||
connection = clickhouse_connect.get_client(
|
||||
user=container.username,
|
||||
@@ -310,7 +253,7 @@ def create_clickhouse( # pylint: disable=too-many-arguments,too-many-positional
|
||||
return reuse.wrap(
|
||||
request,
|
||||
pytestconfig,
|
||||
cache_key,
|
||||
"clickhouse",
|
||||
empty=lambda: types.TestContainerSQL(
|
||||
container=types.TestContainerDocker(id="", host_configs={}, container_configs={}),
|
||||
conn=None,
|
||||
@@ -322,212 +265,6 @@ def create_clickhouse( # pylint: disable=too-many-arguments,too-many-positional
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="clickhouse", scope="package")
|
||||
def clickhouse(
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
keeper: types.TestContainerDocker,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.TestContainerClickhouse:
|
||||
"""
|
||||
Package-scoped fixture for Clickhouse TestContainer.
|
||||
"""
|
||||
return create_clickhouse(
|
||||
tmpfs=tmpfs,
|
||||
network=network,
|
||||
keeper=keeper,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="clickhouse_node_conns", scope="function")
|
||||
def clickhouse_node_conns(
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
) -> Generator[list[clickhouse_connect.driver.client.Client], Any]:
|
||||
"""Per-node clients (index 0 = the initiator) for asserting shard-local
|
||||
state via the local, non-distributed tables. Empty for single-node
|
||||
fixtures, which don't populate `nodes`."""
|
||||
conns = [
|
||||
clickhouse_connect.get_client(
|
||||
user=clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME"],
|
||||
password=clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD"],
|
||||
host=node.host_configs["8123"].address,
|
||||
port=node.host_configs["8123"].port,
|
||||
)
|
||||
for node in clickhouse.nodes
|
||||
]
|
||||
yield conns
|
||||
for conn in conns:
|
||||
conn.close()
|
||||
|
||||
|
||||
def create_clickhouse_cluster( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
keeper: types.TestContainerDocker,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
cache_key: str = "clickhouse_cluster",
|
||||
shards: int = 2,
|
||||
version: str | None = None,
|
||||
) -> types.TestContainerClickhouse:
|
||||
"""
|
||||
To some extent, taken inspiration from how ClickHouse's own integration
|
||||
harness composes real clusters: deterministic hostnames
|
||||
(network aliases), per-node shard macros, and a shared cluster definition
|
||||
named `cluster`.
|
||||
|
||||
`conn`/`env` point at node 1 i.e the initiator every query-service query and
|
||||
migration goes through. Per-node containers are exposed via `nodes` so
|
||||
tests can assert shard-local state.
|
||||
"""
|
||||
coordinator = next(iter(keeper.container_configs.values()))
|
||||
|
||||
def create() -> types.TestContainerClickhouse:
|
||||
clickhouse_version = version or request.config.getoption("--clickhouse-version")
|
||||
|
||||
# Unique aliases per creation: docker allows duplicate network aliases
|
||||
# (DNS round-robin), so a stale cluster must never share names with a
|
||||
# fresh one.
|
||||
suffix = uuid4().hex[:6]
|
||||
aliases = [f"signoz-ch-{suffix}-{i:02d}" for i in range(1, shards + 1)]
|
||||
remote_servers = render_remote_servers([(alias, 9000) for alias in aliases], secret=cache_key)
|
||||
# Own DDL queue path: the keeper instance may be shared with other
|
||||
# environments under --reuse; its DDL queue stays separate.
|
||||
distributed_ddl_path = f"/clickhouse/{cache_key}-{suffix}/task_queue/ddl"
|
||||
|
||||
nodes: list[types.TestContainerDocker] = []
|
||||
started: list[ClickHouseContainer] = []
|
||||
try:
|
||||
for i, alias in enumerate(aliases, start=1):
|
||||
node_config = render_node_config(
|
||||
keeper_address=coordinator.address,
|
||||
keeper_port=coordinator.port,
|
||||
shard=f"{i:02d}",
|
||||
remote_servers=remote_servers,
|
||||
distributed_ddl_path=distributed_ddl_path,
|
||||
)
|
||||
|
||||
tmp_dir = tmpfs(f"clickhouse-{suffix}-{i:02d}")
|
||||
cluster_config_file_path = os.path.join(tmp_dir, "cluster.xml")
|
||||
with open(cluster_config_file_path, "w", encoding="utf-8") as f:
|
||||
f.write(node_config)
|
||||
custom_function_file_path = os.path.join(tmp_dir, "custom-function.xml")
|
||||
with open(custom_function_file_path, "w", encoding="utf-8") as f:
|
||||
f.write(CUSTOM_FUNCTION_CONFIG)
|
||||
users_config_file_path = os.path.join(tmp_dir, "users.xml")
|
||||
with open(users_config_file_path, "w", encoding="utf-8") as f:
|
||||
f.write(CLUSTER_USERS_CONFIG)
|
||||
|
||||
container = ClickHouseContainer(
|
||||
image=f"clickhouse/clickhouse-server:{clickhouse_version}",
|
||||
port=9000,
|
||||
username=CLICKHOUSE_USERNAME,
|
||||
password=CLICKHOUSE_PASSWORD,
|
||||
)
|
||||
container.with_volume_mapping(cluster_config_file_path, "/etc/clickhouse-server/config.d/cluster.xml")
|
||||
container.with_volume_mapping(custom_function_file_path, "/etc/clickhouse-server/custom-function.xml")
|
||||
container.with_volume_mapping(users_config_file_path, "/etc/clickhouse-server/users.d/integration-cluster.xml")
|
||||
container.with_network(network)
|
||||
container.with_network_aliases(alias)
|
||||
container.start()
|
||||
started.append(container)
|
||||
|
||||
install_histogram_quantile(container)
|
||||
|
||||
nodes.append(
|
||||
types.TestContainerDocker(
|
||||
id=container.get_wrapped_container().id,
|
||||
host_configs={
|
||||
"9000": types.TestContainerUrlConfig(
|
||||
"tcp",
|
||||
container.get_container_host_ip(),
|
||||
container.get_exposed_port(9000),
|
||||
),
|
||||
"8123": types.TestContainerUrlConfig(
|
||||
"tcp",
|
||||
container.get_container_host_ip(),
|
||||
container.get_exposed_port(8123),
|
||||
),
|
||||
},
|
||||
container_configs={
|
||||
"9000": types.TestContainerUrlConfig("tcp", alias, 9000),
|
||||
"8123": types.TestContainerUrlConfig("tcp", alias, 8123),
|
||||
},
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
for container in started:
|
||||
container.stop()
|
||||
raise
|
||||
|
||||
connection = clickhouse_connect.get_client(
|
||||
user=CLICKHOUSE_USERNAME,
|
||||
password=CLICKHOUSE_PASSWORD,
|
||||
host=nodes[0].host_configs["8123"].address,
|
||||
port=nodes[0].host_configs["8123"].port,
|
||||
)
|
||||
|
||||
return types.TestContainerClickhouse(
|
||||
container=nodes[0],
|
||||
conn=connection,
|
||||
env={
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN": f"tcp://{CLICKHOUSE_USERNAME}:{CLICKHOUSE_PASSWORD}@{aliases[0]}:{9000}",
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME": CLICKHOUSE_USERNAME,
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD": CLICKHOUSE_PASSWORD,
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER": "cluster",
|
||||
},
|
||||
nodes=nodes,
|
||||
)
|
||||
|
||||
def delete(resource: types.TestContainerClickhouse) -> None:
|
||||
client = docker.from_env()
|
||||
for node in resource.nodes or [resource.container]:
|
||||
try:
|
||||
client.containers.get(container_id=node.id).stop()
|
||||
client.containers.get(container_id=node.id).remove(v=True)
|
||||
except docker.errors.NotFound:
|
||||
logger.info(
|
||||
"Skipping removal of Clickhouse cluster node, node(%s) not found. Maybe it was manually removed?",
|
||||
{"id": node.id},
|
||||
)
|
||||
|
||||
def restore(cache: dict) -> types.TestContainerClickhouse:
|
||||
nodes = [types.TestContainerDocker.from_cache(node) for node in cache["nodes"]]
|
||||
env = cache["env"]
|
||||
host_config = nodes[0].host_configs["8123"]
|
||||
|
||||
conn = clickhouse_connect.get_client(
|
||||
user=env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME"],
|
||||
password=env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD"],
|
||||
host=host_config.address,
|
||||
port=host_config.port,
|
||||
)
|
||||
|
||||
return types.TestContainerClickhouse(
|
||||
container=nodes[0],
|
||||
conn=conn,
|
||||
env=env,
|
||||
nodes=nodes,
|
||||
)
|
||||
|
||||
return reuse.wrap(
|
||||
request,
|
||||
pytestconfig,
|
||||
cache_key,
|
||||
empty=lambda: types.TestContainerClickhouse(
|
||||
container=types.TestContainerDocker(id="", host_configs={}, container_configs={}),
|
||||
conn=None,
|
||||
env={},
|
||||
),
|
||||
create=create,
|
||||
delete=delete,
|
||||
restore=restore,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="check_query_log")
|
||||
def check_query_log(
|
||||
signoz: types.SigNoz,
|
||||
|
||||
121
tests/fixtures/keeper.py
vendored
121
tests/fixtures/keeper.py
vendored
@@ -1,121 +0,0 @@
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
|
||||
import docker
|
||||
import docker.errors
|
||||
import pytest
|
||||
from testcontainers.core.container import DockerContainer, Network
|
||||
|
||||
from fixtures import reuse, types
|
||||
from fixtures.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
KEEPER_CONFIG = """
|
||||
<clickhouse>
|
||||
<listen_host>0.0.0.0</listen_host>
|
||||
<keeper_server>
|
||||
<tcp_port>9181</tcp_port>
|
||||
<server_id>1</server_id>
|
||||
<log_storage_path>/var/lib/clickhouse-keeper/coordination/log</log_storage_path>
|
||||
<snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path>
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
||||
<session_timeout_ms>30000</session_timeout_ms>
|
||||
<raft_logs_level>warning</raft_logs_level>
|
||||
</coordination_settings>
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>localhost</hostname>
|
||||
<port>9234</port>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
</clickhouse>
|
||||
"""
|
||||
|
||||
|
||||
def create_clickhouse_keeper(
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
cache_key: str = "clickhousekeeper",
|
||||
version: str | None = None,
|
||||
) -> types.TestContainerDocker:
|
||||
|
||||
def create() -> types.TestContainerDocker:
|
||||
keeper_version = version or request.config.getoption("--clickhouse-version")
|
||||
|
||||
tmp_dir = tmpfs(cache_key)
|
||||
keeper_config_file_path = os.path.join(tmp_dir, "keeper_config.xml")
|
||||
with open(keeper_config_file_path, "w", encoding="utf-8") as f:
|
||||
f.write(KEEPER_CONFIG)
|
||||
|
||||
container = DockerContainer(image=f"clickhouse/clickhouse-keeper:{keeper_version}")
|
||||
container.with_volume_mapping(keeper_config_file_path, "/etc/clickhouse-keeper/keeper_config.xml")
|
||||
container.with_exposed_ports(9181)
|
||||
container.with_network(network=network)
|
||||
|
||||
container.start()
|
||||
return types.TestContainerDocker(
|
||||
id=container.get_wrapped_container().id,
|
||||
host_configs={
|
||||
"9181": types.TestContainerUrlConfig(
|
||||
scheme="tcp",
|
||||
address=container.get_container_host_ip(),
|
||||
port=container.get_exposed_port(9181),
|
||||
)
|
||||
},
|
||||
container_configs={
|
||||
"9181": types.TestContainerUrlConfig(
|
||||
scheme="tcp",
|
||||
address=container.get_wrapped_container().name,
|
||||
port=9181,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
def delete(container: types.TestContainerDocker):
|
||||
client = docker.from_env()
|
||||
try:
|
||||
client.containers.get(container_id=container.id).stop()
|
||||
client.containers.get(container_id=container.id).remove(v=True)
|
||||
except docker.errors.NotFound:
|
||||
logger.info(
|
||||
"Skipping removal of ClickHouse Keeper, Keeper(%s) not found. Maybe it was manually removed?",
|
||||
{"id": container.id},
|
||||
)
|
||||
|
||||
def restore(cache: dict) -> types.TestContainerDocker:
|
||||
return types.TestContainerDocker.from_cache(cache)
|
||||
|
||||
return reuse.wrap(
|
||||
request,
|
||||
pytestconfig,
|
||||
cache_key,
|
||||
lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
|
||||
create,
|
||||
delete,
|
||||
restore,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="keeper", scope="package")
|
||||
def keeper(
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.TestContainerDocker:
|
||||
"""
|
||||
Package-scoped fixture for ClickHouse Keeper TestContainer.
|
||||
"""
|
||||
return create_clickhouse_keeper(
|
||||
tmpfs=tmpfs,
|
||||
network=network,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
)
|
||||
83
tests/fixtures/metricreduction.py
vendored
83
tests/fixtures/metricreduction.py
vendored
@@ -1,83 +0,0 @@
|
||||
import datetime
|
||||
from collections.abc import Sequence
|
||||
|
||||
import clickhouse_connect.driver.client
|
||||
|
||||
from fixtures.metrics import MetricsBufferSample, MetricsBufferTimeSeries
|
||||
|
||||
|
||||
def local_series_counts(
|
||||
node_conns: list[clickhouse_connect.driver.client.Client],
|
||||
table: str,
|
||||
metric_name: str,
|
||||
) -> list[int]:
|
||||
"""Distinct series per node via the LOCAL (non-distributed) table."""
|
||||
return [
|
||||
int(
|
||||
conn.query(
|
||||
f"SELECT count(DISTINCT fingerprint) FROM signoz_metrics.{table} WHERE metric_name = %(metric_name)s",
|
||||
parameters={"metric_name": metric_name},
|
||||
).result_rows[0][0]
|
||||
)
|
||||
for conn in node_conns
|
||||
]
|
||||
|
||||
|
||||
def assert_spans_shards(
|
||||
node_conns: list[clickhouse_connect.driver.client.Client],
|
||||
table: str,
|
||||
metric_name: str,
|
||||
total: int,
|
||||
) -> None:
|
||||
"""Guard for distributed tests: a green run on a cluster proves nothing
|
||||
unless the seeded series actually landed on more than one shard."""
|
||||
counts = local_series_counts(node_conns, table, metric_name)
|
||||
assert sum(counts) == total, f"expected {total} series in {table} across shards, got {counts}"
|
||||
assert min(counts) > 0, f"seeded series in {table} all landed on one shard: {counts}"
|
||||
|
||||
|
||||
def build_recent_gauge_data(
|
||||
metric_name: str,
|
||||
base_epoch: int,
|
||||
services: Sequence[str],
|
||||
pods_per_service: int,
|
||||
minutes: int,
|
||||
value: float = 1.0,
|
||||
) -> tuple[list[MetricsBufferTimeSeries], list[MetricsBufferSample]]:
|
||||
"""Collector-shaped buffer rows for a gauge under a reduction rule that
|
||||
keeps `service`: per raw series a raw series row (is_reduced=false, full
|
||||
labels, reduced_fingerprint -> group) plus the group's reduced series row
|
||||
(is_reduced=true, kept labels), and one raw sample per series per minute
|
||||
carrying both fingerprints. Returns (time_series, samples) for
|
||||
insert_buffer_metrics."""
|
||||
reduced_series = {
|
||||
service: MetricsBufferTimeSeries(
|
||||
metric_name=metric_name,
|
||||
labels={"service": service},
|
||||
timestamp=datetime.datetime.fromtimestamp(base_epoch, tz=datetime.UTC),
|
||||
is_reduced=True,
|
||||
)
|
||||
for service in services
|
||||
}
|
||||
raw_series = [
|
||||
MetricsBufferTimeSeries(
|
||||
metric_name=metric_name,
|
||||
labels={"service": service, "pod": f"pod-{service}-{i}"},
|
||||
timestamp=datetime.datetime.fromtimestamp(base_epoch, tz=datetime.UTC),
|
||||
reduced_fingerprint=reduced_series[service].fingerprint,
|
||||
)
|
||||
for service in services
|
||||
for i in range(pods_per_service)
|
||||
]
|
||||
samples = [
|
||||
MetricsBufferSample(
|
||||
metric_name=metric_name,
|
||||
fingerprint=ts.fingerprint,
|
||||
timestamp=datetime.datetime.fromtimestamp(base_epoch + minute * 60, tz=datetime.UTC),
|
||||
value=value,
|
||||
reduced_fingerprint=ts.reduced_fingerprint,
|
||||
)
|
||||
for ts in raw_series
|
||||
for minute in range(minutes)
|
||||
]
|
||||
return raw_series + list(reduced_series.values()), samples
|
||||
426
tests/fixtures/metrics.py
vendored
426
tests/fixtures/metrics.py
vendored
@@ -11,14 +11,6 @@ import pytest
|
||||
from fixtures import types
|
||||
from fixtures.time import parse_timestamp
|
||||
|
||||
_REDUCED_METRICS_TABLES_TO_TRUNCATE = [
|
||||
"time_series_v4_reduced",
|
||||
"samples_v4_reduced_last_60s",
|
||||
"samples_v4_reduced_sum_60s",
|
||||
"time_series_v4_buffer",
|
||||
"samples_v4_buffer",
|
||||
]
|
||||
|
||||
|
||||
class MetricsTimeSeries(ABC):
|
||||
"""Represents a row in the time_series_v4 table."""
|
||||
@@ -422,267 +414,6 @@ class Metrics(ABC):
|
||||
return metrics
|
||||
|
||||
|
||||
class MetricsReducedTimeSeries(ABC):
|
||||
"""Represents a row in the time_series_v4_reduced table i.e what
|
||||
the time_series_v4_reduced_mv materializes for a metric under a
|
||||
reduction rule. One row per kept-label group. `fingerprint` holds the
|
||||
reduced fingerprint and `labels` contains only the kept labels.
|
||||
|
||||
The fingerprint recipe (md5, like MetricsTimeSeries) does not match the
|
||||
collector's real hash; it only needs to be consistent with the
|
||||
reduced_fingerprint used in the reduced samples rows.
|
||||
"""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments
|
||||
self,
|
||||
metric_name: str,
|
||||
kept_labels: dict[str, str],
|
||||
timestamp: datetime.datetime,
|
||||
temporality: str = "Unspecified",
|
||||
description: str = "",
|
||||
unit: str = "",
|
||||
type_: str = "Gauge",
|
||||
is_monotonic: bool = False,
|
||||
env: str = "default",
|
||||
) -> None:
|
||||
kept_labels = dict(kept_labels)
|
||||
kept_labels["__name__"] = metric_name
|
||||
self.env = env
|
||||
# mirror time_series_v4_reduced_mv: monotonic cumulative counters are
|
||||
# reduced as deltas
|
||||
if temporality == "Cumulative" and is_monotonic:
|
||||
temporality = "Delta"
|
||||
self.temporality = temporality
|
||||
self.metric_name = metric_name
|
||||
self.description = description
|
||||
self.unit = unit
|
||||
self.type = type_
|
||||
self.is_monotonic = is_monotonic
|
||||
self.labels = json.dumps(kept_labels, separators=(",", ":"))
|
||||
self.attrs = kept_labels
|
||||
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
|
||||
self.normalized = False
|
||||
|
||||
fingerprint_str = metric_name + self.labels
|
||||
self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
|
||||
|
||||
def to_row(self) -> list:
|
||||
return [
|
||||
self.env,
|
||||
self.temporality,
|
||||
self.metric_name,
|
||||
self.description,
|
||||
self.unit,
|
||||
self.type,
|
||||
self.is_monotonic,
|
||||
self.fingerprint,
|
||||
self.unix_milli,
|
||||
self.labels,
|
||||
self.attrs,
|
||||
{},
|
||||
{},
|
||||
self.normalized,
|
||||
]
|
||||
|
||||
|
||||
class MetricsReducedSampleLast60s(ABC):
|
||||
"""Represents a row in the samples_v4_reduced_last_60s table. One 60s
|
||||
bucket per reduced group, as the samples_v4_reduced_last_60s_mv refresh
|
||||
would emit it (gauges and non-monotonic cumulative sums)."""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments
|
||||
self,
|
||||
metric_name: str,
|
||||
reduced_fingerprint: np.uint64,
|
||||
timestamp: datetime.datetime,
|
||||
sum_last: float,
|
||||
min_value: float,
|
||||
max_value: float,
|
||||
sum_values: float,
|
||||
count_series: int,
|
||||
count_samples: int,
|
||||
temporality: str = "Unspecified",
|
||||
env: str = "default",
|
||||
computed_at: datetime.datetime | None = None,
|
||||
) -> None:
|
||||
self.env = env
|
||||
self.temporality = temporality
|
||||
self.metric_name = metric_name
|
||||
self.reduced_fingerprint = reduced_fingerprint
|
||||
# buckets are 60s-aligned: intDiv(unix_milli, 60000) * 60000
|
||||
self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
|
||||
self.sum_last = np.float64(sum_last)
|
||||
self.min = np.float64(min_value)
|
||||
self.max = np.float64(max_value)
|
||||
self.sum_values = np.float64(sum_values)
|
||||
self.count_series = np.uint64(count_series)
|
||||
self.count_samples = np.uint64(count_samples)
|
||||
# the refresh stamps now(); default to shortly after the bucket closes
|
||||
if computed_at is None:
|
||||
computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
|
||||
self.computed_at = computed_at
|
||||
|
||||
def to_row(self) -> list:
|
||||
return [
|
||||
self.env,
|
||||
self.temporality,
|
||||
self.metric_name,
|
||||
self.reduced_fingerprint,
|
||||
self.unix_milli,
|
||||
self.sum_last,
|
||||
self.min,
|
||||
self.max,
|
||||
self.sum_values,
|
||||
self.count_series,
|
||||
self.count_samples,
|
||||
self.computed_at,
|
||||
]
|
||||
|
||||
|
||||
class MetricsReducedSampleSum60s(ABC):
|
||||
"""Represents a row in the samples_v4_reduced_sum_60s table. One 60s
|
||||
bucket per reduced group for delta counters and histograms."""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments
|
||||
self,
|
||||
metric_name: str,
|
||||
reduced_fingerprint: np.uint64,
|
||||
timestamp: datetime.datetime,
|
||||
sum_value: float,
|
||||
count_series: int,
|
||||
count_samples: int,
|
||||
temporality: str = "Delta",
|
||||
env: str = "default",
|
||||
computed_at: datetime.datetime | None = None,
|
||||
) -> None:
|
||||
self.env = env
|
||||
self.temporality = temporality
|
||||
self.metric_name = metric_name
|
||||
self.reduced_fingerprint = reduced_fingerprint
|
||||
self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
|
||||
self.sum = np.float64(sum_value)
|
||||
self.count_series = np.uint64(count_series)
|
||||
self.count_samples = np.uint64(count_samples)
|
||||
if computed_at is None:
|
||||
computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
|
||||
self.computed_at = computed_at
|
||||
|
||||
def to_row(self) -> list:
|
||||
return [
|
||||
self.env,
|
||||
self.temporality,
|
||||
self.metric_name,
|
||||
self.reduced_fingerprint,
|
||||
self.unix_milli,
|
||||
self.sum,
|
||||
self.count_series,
|
||||
self.count_samples,
|
||||
self.computed_at,
|
||||
]
|
||||
|
||||
|
||||
class MetricsBufferTimeSeries(ABC):
|
||||
"""Represents a row in the time_series_v4_buffer table. This is the collector's
|
||||
universal landing target under cardinality control. For a ruled metric the
|
||||
collector writes two rows per series: the raw one (is_reduced=false, full
|
||||
labels, reduced_fingerprint pointing at its group) and the group's reduced
|
||||
one (is_reduced=true, kept labels, fingerprint = reduced fingerprint)."""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments
|
||||
self,
|
||||
metric_name: str,
|
||||
labels: dict[str, str],
|
||||
timestamp: datetime.datetime,
|
||||
reduced_fingerprint: np.uint64 | int = 0,
|
||||
is_reduced: bool = False,
|
||||
temporality: str = "Unspecified",
|
||||
description: str = "",
|
||||
unit: str = "",
|
||||
type_: str = "Gauge",
|
||||
is_monotonic: bool = False,
|
||||
env: str = "default",
|
||||
) -> None:
|
||||
labels = dict(labels)
|
||||
labels["__name__"] = metric_name
|
||||
self.env = env
|
||||
self.temporality = temporality
|
||||
self.metric_name = metric_name
|
||||
self.description = description
|
||||
self.unit = unit
|
||||
self.type = type_
|
||||
self.is_monotonic = is_monotonic
|
||||
self.reduced_fingerprint = np.uint64(reduced_fingerprint)
|
||||
self.is_reduced = is_reduced
|
||||
self.labels = json.dumps(labels, separators=(",", ":"))
|
||||
self.attrs = labels
|
||||
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
|
||||
self.normalized = False
|
||||
|
||||
fingerprint_str = metric_name + self.labels
|
||||
self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
|
||||
|
||||
def to_row(self) -> list:
|
||||
return [
|
||||
self.env,
|
||||
self.temporality,
|
||||
self.metric_name,
|
||||
self.description,
|
||||
self.unit,
|
||||
self.type,
|
||||
self.is_monotonic,
|
||||
self.fingerprint,
|
||||
self.reduced_fingerprint,
|
||||
self.is_reduced,
|
||||
self.unix_milli,
|
||||
self.labels,
|
||||
self.attrs,
|
||||
{},
|
||||
{},
|
||||
self.normalized,
|
||||
]
|
||||
|
||||
|
||||
class MetricsBufferSample(ABC):
|
||||
"""Represents a row in the samples_v4_buffer table. Ruled samples carry
|
||||
the raw fingerprint plus the group's reduced_fingerprint; unruled samples
|
||||
have reduced_fingerprint = 0."""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments
|
||||
self,
|
||||
metric_name: str,
|
||||
fingerprint: np.uint64,
|
||||
timestamp: datetime.datetime,
|
||||
value: float,
|
||||
reduced_fingerprint: np.uint64 | int = 0,
|
||||
is_monotonic: bool = False,
|
||||
temporality: str = "Unspecified",
|
||||
env: str = "default",
|
||||
flags: int = 0,
|
||||
) -> None:
|
||||
self.env = env
|
||||
self.temporality = temporality
|
||||
self.metric_name = metric_name
|
||||
self.fingerprint = fingerprint
|
||||
self.reduced_fingerprint = np.uint64(reduced_fingerprint)
|
||||
self.is_monotonic = is_monotonic
|
||||
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
|
||||
self.value = np.float64(value)
|
||||
self.flags = np.uint32(flags)
|
||||
|
||||
def to_row(self) -> list:
|
||||
return [
|
||||
self.env,
|
||||
self.temporality,
|
||||
self.metric_name,
|
||||
self.fingerprint,
|
||||
self.reduced_fingerprint,
|
||||
self.is_monotonic,
|
||||
self.unix_milli,
|
||||
self.value,
|
||||
self.flags,
|
||||
]
|
||||
|
||||
|
||||
def insert_metrics_to_clickhouse(conn, metrics: list[Metrics]) -> None:
|
||||
"""
|
||||
Insert metrics into ClickHouse tables.
|
||||
@@ -845,163 +576,6 @@ def insert_metrics(
|
||||
)
|
||||
|
||||
|
||||
def insert_reduced_metrics_to_clickhouse(
|
||||
conn,
|
||||
time_series: list[MetricsReducedTimeSeries],
|
||||
last_samples: list[MetricsReducedSampleLast60s] | None = None,
|
||||
sum_samples: list[MetricsReducedSampleSum60s] | None = None,
|
||||
) -> None:
|
||||
"""Insert reduced series into distributed_time_series_v4_reduced and 60s
|
||||
buckets into the reduced samples tables. These tables exist only when
|
||||
the schema migrator version includes the metrics cardinality-control
|
||||
migration."""
|
||||
if time_series:
|
||||
conn.insert(
|
||||
database="signoz_metrics",
|
||||
table="distributed_time_series_v4_reduced",
|
||||
column_names=[
|
||||
"env",
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"description",
|
||||
"unit",
|
||||
"type",
|
||||
"is_monotonic",
|
||||
"fingerprint",
|
||||
"unix_milli",
|
||||
"labels",
|
||||
"attrs",
|
||||
"scope_attrs",
|
||||
"resource_attrs",
|
||||
"__normalized",
|
||||
],
|
||||
data=[ts.to_row() for ts in time_series],
|
||||
)
|
||||
|
||||
if last_samples:
|
||||
conn.insert(
|
||||
database="signoz_metrics",
|
||||
table="distributed_samples_v4_reduced_last_60s",
|
||||
column_names=[
|
||||
"env",
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"reduced_fingerprint",
|
||||
"unix_milli",
|
||||
"sum_last",
|
||||
"min",
|
||||
"max",
|
||||
"sum_values",
|
||||
"count_series",
|
||||
"count_samples",
|
||||
"computed_at",
|
||||
],
|
||||
data=[sample.to_row() for sample in last_samples],
|
||||
)
|
||||
|
||||
if sum_samples:
|
||||
conn.insert(
|
||||
database="signoz_metrics",
|
||||
table="distributed_samples_v4_reduced_sum_60s",
|
||||
column_names=[
|
||||
"env",
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"reduced_fingerprint",
|
||||
"unix_milli",
|
||||
"sum",
|
||||
"count_series",
|
||||
"count_samples",
|
||||
"computed_at",
|
||||
],
|
||||
data=[sample.to_row() for sample in sum_samples],
|
||||
)
|
||||
|
||||
|
||||
def insert_buffer_metrics_to_clickhouse(
|
||||
conn,
|
||||
time_series: list[MetricsBufferTimeSeries],
|
||||
samples: list[MetricsBufferSample],
|
||||
) -> None:
|
||||
if time_series:
|
||||
conn.insert(
|
||||
database="signoz_metrics",
|
||||
table="distributed_time_series_v4_buffer",
|
||||
column_names=[
|
||||
"env",
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"description",
|
||||
"unit",
|
||||
"type",
|
||||
"is_monotonic",
|
||||
"fingerprint",
|
||||
"reduced_fingerprint",
|
||||
"is_reduced",
|
||||
"unix_milli",
|
||||
"labels",
|
||||
"attrs",
|
||||
"scope_attrs",
|
||||
"resource_attrs",
|
||||
"__normalized",
|
||||
],
|
||||
data=[ts.to_row() for ts in time_series],
|
||||
)
|
||||
|
||||
if samples:
|
||||
conn.insert(
|
||||
database="signoz_metrics",
|
||||
table="distributed_samples_v4_buffer",
|
||||
column_names=[
|
||||
"env",
|
||||
"temporality",
|
||||
"metric_name",
|
||||
"fingerprint",
|
||||
"reduced_fingerprint",
|
||||
"is_monotonic",
|
||||
"unix_milli",
|
||||
"value",
|
||||
"flags",
|
||||
],
|
||||
data=[sample.to_row() for sample in samples],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="insert_reduced_metrics", scope="function")
|
||||
def insert_reduced_metrics(
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
) -> Generator[Callable[..., None], Any]:
|
||||
def _insert_reduced_metrics(
|
||||
time_series: list[MetricsReducedTimeSeries],
|
||||
last_samples: list[MetricsReducedSampleLast60s] | None = None,
|
||||
sum_samples: list[MetricsReducedSampleSum60s] | None = None,
|
||||
) -> None:
|
||||
insert_reduced_metrics_to_clickhouse(clickhouse.conn, time_series, last_samples, sum_samples)
|
||||
|
||||
yield _insert_reduced_metrics
|
||||
|
||||
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
|
||||
for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
|
||||
clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
|
||||
|
||||
|
||||
@pytest.fixture(name="insert_buffer_metrics", scope="function")
|
||||
def insert_buffer_metrics(
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
) -> Generator[Callable[..., None], Any]:
|
||||
def _insert_buffer_metrics(
|
||||
time_series: list[MetricsBufferTimeSeries],
|
||||
samples: list[MetricsBufferSample],
|
||||
) -> None:
|
||||
insert_buffer_metrics_to_clickhouse(clickhouse.conn, time_series, samples)
|
||||
|
||||
yield _insert_buffer_metrics
|
||||
|
||||
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
|
||||
for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
|
||||
clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
|
||||
|
||||
|
||||
@pytest.fixture(name="remove_metrics_ttl_and_storage_settings", scope="function")
|
||||
def remove_metrics_ttl_and_storage_settings(signoz: types.SigNoz):
|
||||
"""
|
||||
|
||||
13
tests/fixtures/migrator.py
vendored
13
tests/fixtures/migrator.py
vendored
@@ -8,30 +8,27 @@ from fixtures.logger import setup_logger
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
def create_migrator( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
def create_migrator(
|
||||
network: Network,
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
cache_key: str = "migrator",
|
||||
env_overrides: dict | None = None,
|
||||
version: str | None = None,
|
||||
) -> types.Operation:
|
||||
"""
|
||||
Factory function for running schema migrations.
|
||||
Accepts optional env_overrides to customize the migrator environment, and
|
||||
an optional version to pin a schema-migrator release different from the
|
||||
--schema-migrator-version option.
|
||||
Accepts optional env_overrides to customize the migrator environment.
|
||||
"""
|
||||
|
||||
def create() -> None:
|
||||
migrator_version = version or request.config.getoption("--schema-migrator-version")
|
||||
version = request.config.getoption("--schema-migrator-version")
|
||||
client = docker.from_env()
|
||||
|
||||
environment = dict(env_overrides) if env_overrides else {}
|
||||
|
||||
container = client.containers.run(
|
||||
image=f"signoz/signoz-schema-migrator:{migrator_version}",
|
||||
image=f"signoz/signoz-schema-migrator:{version}",
|
||||
command=f"sync --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN']}",
|
||||
detach=True,
|
||||
auto_remove=False,
|
||||
@@ -50,7 +47,7 @@ def create_migrator( # pylint: disable=too-many-arguments,too-many-positional-a
|
||||
container.remove()
|
||||
|
||||
container = client.containers.run(
|
||||
image=f"signoz/signoz-schema-migrator:{migrator_version}",
|
||||
image=f"signoz/signoz-schema-migrator:{version}",
|
||||
command=f"async --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN']}",
|
||||
detach=True,
|
||||
auto_remove=False,
|
||||
|
||||
51
tests/fixtures/querier.py
vendored
51
tests/fixtures/querier.py
vendored
@@ -189,35 +189,6 @@ def make_query_request(
|
||||
)
|
||||
|
||||
|
||||
def aligned_epoch(ago: timedelta, step_seconds: int = DEFAULT_STEP_INTERVAL) -> int:
|
||||
"""Epoch seconds for `now - ago`, floored to a step boundary so seeded
|
||||
points land exactly on the query's toStartOfInterval buckets."""
|
||||
return (int((datetime.now(tz=UTC) - ago).timestamp()) // step_seconds) * step_seconds
|
||||
|
||||
|
||||
def query_metric_values( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
metric_name: str,
|
||||
start_epoch: int,
|
||||
end_epoch: int,
|
||||
time_agg: str,
|
||||
space_agg: str,
|
||||
step_interval: int = DEFAULT_STEP_INTERVAL,
|
||||
) -> list[dict]:
|
||||
"""Run a single metrics builder query over [start_epoch, end_epoch) in
|
||||
epoch seconds and return its series values sorted by timestamp."""
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=start_epoch * 1000,
|
||||
end_ms=end_epoch * 1000,
|
||||
queries=[build_builder_query("A", metric_name, time_agg, space_agg, step_interval=step_interval)],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
return sorted(get_series_values(response.json(), "A"), key=lambda v: v["timestamp"])
|
||||
|
||||
|
||||
def build_builder_query(
|
||||
name: str,
|
||||
metric_name: str,
|
||||
@@ -979,3 +950,25 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def make_scalar_query_request(
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
now: datetime,
|
||||
queries: list[dict],
|
||||
lookback_minutes: int = 5,
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((now - timedelta(minutes=lookback_minutes)).timestamp() * 1000),
|
||||
"end": int(now.timestamp() * 1000),
|
||||
"requestType": "scalar",
|
||||
"compositeQuery": {"queries": queries},
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
7
tests/fixtures/types.py
vendored
7
tests/fixtures/types.py
vendored
@@ -1,4 +1,4 @@
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
from urllib.parse import urljoin
|
||||
|
||||
@@ -84,16 +84,11 @@ class TestContainerClickhouse:
|
||||
container: TestContainerDocker
|
||||
conn: clickhouse_connect.driver.client.Client
|
||||
env: dict[str, str]
|
||||
# Per-node containers when running a multi-node cluster. Empty for the
|
||||
# default single-node setup; nodes[0] is the node `conn`/`env` point at
|
||||
# (the initiator every query goes through).
|
||||
nodes: list[TestContainerDocker] = field(default_factory=list)
|
||||
|
||||
def __cache__(self) -> dict:
|
||||
return {
|
||||
"container": self.container.__cache__(),
|
||||
"env": self.env,
|
||||
"nodes": [node.__cache__() for node in self.nodes],
|
||||
}
|
||||
|
||||
def __log__(self) -> str:
|
||||
|
||||
67
tests/fixtures/zookeeper.py
vendored
Normal file
67
tests/fixtures/zookeeper.py
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
import docker
|
||||
import docker.errors
|
||||
import pytest
|
||||
from testcontainers.core.container import DockerContainer, Network
|
||||
|
||||
from fixtures import reuse, types
|
||||
from fixtures.logger import setup_logger
|
||||
|
||||
logger = setup_logger(__name__)
|
||||
|
||||
|
||||
@pytest.fixture(name="zookeeper", scope="package")
|
||||
def zookeeper(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
|
||||
"""
|
||||
Package-scoped fixture for Zookeeper TestContainer.
|
||||
"""
|
||||
|
||||
def create() -> types.TestContainerDocker:
|
||||
version = request.config.getoption("--zookeeper-version")
|
||||
|
||||
container = DockerContainer(image=f"signoz/zookeeper:{version}")
|
||||
container.with_env("ALLOW_ANONYMOUS_LOGIN", "yes")
|
||||
container.with_exposed_ports(2181)
|
||||
container.with_network(network=network)
|
||||
|
||||
container.start()
|
||||
return types.TestContainerDocker(
|
||||
id=container.get_wrapped_container().id,
|
||||
host_configs={
|
||||
"2181": types.TestContainerUrlConfig(
|
||||
scheme="tcp",
|
||||
address=container.get_container_host_ip(),
|
||||
port=container.get_exposed_port(2181),
|
||||
)
|
||||
},
|
||||
container_configs={
|
||||
"2181": types.TestContainerUrlConfig(
|
||||
scheme="tcp",
|
||||
address=container.get_wrapped_container().name,
|
||||
port=2181,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
def delete(container: types.TestContainerDocker):
|
||||
client = docker.from_env()
|
||||
try:
|
||||
client.containers.get(container_id=container.id).stop()
|
||||
client.containers.get(container_id=container.id).remove(v=True)
|
||||
except docker.errors.NotFound:
|
||||
logger.info(
|
||||
"Skipping removal of Zookeeper, Zookeeper(%s) not found. Maybe it was manually removed?",
|
||||
{"id": container.id},
|
||||
)
|
||||
|
||||
def restore(cache: dict) -> types.TestContainerDocker:
|
||||
return types.TestContainerDocker.from_cache(cache)
|
||||
|
||||
return reuse.wrap(
|
||||
request,
|
||||
pytestconfig,
|
||||
"zookeeper",
|
||||
lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
|
||||
create,
|
||||
delete,
|
||||
restore,
|
||||
)
|
||||
@@ -1,52 +0,0 @@
|
||||
import clickhouse_connect.driver.client
|
||||
|
||||
from fixtures import types
|
||||
|
||||
TOTAL_ROWS = 64
|
||||
|
||||
|
||||
def test_topology(
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
|
||||
) -> None:
|
||||
aliases = {node.container_configs["9000"].address for node in clickhouse.nodes}
|
||||
|
||||
# Every node sees the same 2-shard cluster definition and identifies
|
||||
# exactly itself as the local replica
|
||||
|
||||
for i, conn in enumerate(clickhouse_node_conns, start=1):
|
||||
rows = conn.query("SELECT shard_num, host_name, is_local FROM system.clusters WHERE cluster = 'cluster' ORDER BY shard_num").result_rows
|
||||
assert [row[0] for row in rows] == [1, 2], f"node {i}: expected 2 shards, got {rows}"
|
||||
assert {row[1] for row in rows} == aliases, f"node {i}: cluster hosts {rows} != node aliases {aliases}"
|
||||
local = [row[0] for row in rows if row[2]]
|
||||
assert local == [i], f"node {i}: expected to be local for shard {i} only, got {local}"
|
||||
|
||||
|
||||
def test_replicated_distributed_round_trip(
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
|
||||
) -> None:
|
||||
# ON CLUSTER DDL reaches both nodes, Replicated engines register with the
|
||||
# keeper via per-node macros, and a sharded Distributed insert scatters rows
|
||||
# across shards while the distributed read returns the union.
|
||||
conn = clickhouse.conn
|
||||
try:
|
||||
conn.query("CREATE DATABASE IF NOT EXISTS it_cluster ON CLUSTER 'cluster'")
|
||||
conn.query("CREATE TABLE it_cluster.events ON CLUSTER 'cluster' (id UInt64, payload String) ENGINE = ReplicatedMergeTree ORDER BY id")
|
||||
conn.query("CREATE TABLE it_cluster.distributed_events ON CLUSTER 'cluster' AS it_cluster.events ENGINE = Distributed('cluster', 'it_cluster', 'events', cityHash64(id))")
|
||||
|
||||
conn.insert(
|
||||
database="it_cluster",
|
||||
table="distributed_events",
|
||||
column_names=["id", "payload"],
|
||||
data=[[i, f"payload-{i:03d}"] for i in range(TOTAL_ROWS)],
|
||||
)
|
||||
|
||||
distributed_count = int(conn.query("SELECT count() FROM it_cluster.distributed_events").result_rows[0][0])
|
||||
assert distributed_count == TOTAL_ROWS
|
||||
|
||||
local_counts = [int(node_conn.query("SELECT count() FROM it_cluster.events").result_rows[0][0]) for node_conn in clickhouse_node_conns]
|
||||
assert sum(local_counts) == TOTAL_ROWS, f"local counts {local_counts} do not add up to {TOTAL_ROWS}"
|
||||
assert min(local_counts) > 0, f"all rows landed on one shard: {local_counts}"
|
||||
finally:
|
||||
conn.query("DROP DATABASE IF EXISTS it_cluster ON CLUSTER 'cluster' SYNC")
|
||||
@@ -1,48 +0,0 @@
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from testcontainers.core.container import Network
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.clickhouse import create_clickhouse_cluster
|
||||
from fixtures.keeper import create_clickhouse_keeper
|
||||
|
||||
CLICKHOUSE_VERSION = "25.12.5"
|
||||
|
||||
|
||||
@pytest.fixture(name="keeper", scope="package")
|
||||
def keeper_cluster(
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.TestContainerDocker:
|
||||
return create_clickhouse_keeper(
|
||||
tmpfs=tmpfs,
|
||||
network=network,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
cache_key="keeper_cluster",
|
||||
version=CLICKHOUSE_VERSION,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="clickhouse", scope="package")
|
||||
def clickhouse_cluster(
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
keeper: types.TestContainerDocker,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.TestContainerClickhouse:
|
||||
return create_clickhouse_cluster(
|
||||
tmpfs=tmpfs,
|
||||
network=network,
|
||||
keeper=keeper,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
cache_key="clickhouse_cluster",
|
||||
shards=2,
|
||||
version=CLICKHOUSE_VERSION,
|
||||
)
|
||||
@@ -1,203 +0,0 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import clickhouse_connect.driver.client
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.metricreduction import assert_spans_shards
|
||||
from fixtures.metrics import (
|
||||
Metrics,
|
||||
MetricsReducedSampleLast60s,
|
||||
MetricsReducedTimeSeries,
|
||||
)
|
||||
from fixtures.querier import aligned_epoch, query_metric_values
|
||||
|
||||
|
||||
def test_query_spanning_rule_activation_combines_raw_and_reduced_data(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
insert_reduced_metrics: Callable[..., None],
|
||||
clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
|
||||
) -> None:
|
||||
"""Before a reduction rule activates, data lives in the raw tables; after,
|
||||
only the reduced tables have data. A single query spanning the activation
|
||||
time must return one continuous series with no gap and no double counting:
|
||||
32 raw series at 2.0 collapse into 16 groups whose per-minute total is
|
||||
4.0, so the summed value stays 320 per step on both sides. Enough series
|
||||
are seeded that both shards hold data (checked below), so correct totals
|
||||
also prove the queries read every shard."""
|
||||
metric_name = "test_reduction_activation_boundary"
|
||||
base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
|
||||
services = [f"svc-{i:02d}" for i in range(16)]
|
||||
|
||||
# first 30 minutes: raw data (2 pods per service, one sample per minute)
|
||||
insert_metrics(
|
||||
[
|
||||
Metrics(
|
||||
metric_name=metric_name,
|
||||
labels={"service": service, "pod": f"{service}-pod-{pod}"},
|
||||
timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
|
||||
value=2.0,
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
)
|
||||
for service in services
|
||||
for pod in range(2)
|
||||
for minute in range(30)
|
||||
]
|
||||
)
|
||||
|
||||
# next 30 minutes: reduced data only (one row per service per minute)
|
||||
time_series = [
|
||||
MetricsReducedTimeSeries(
|
||||
metric_name=metric_name,
|
||||
kept_labels={"service": service},
|
||||
timestamp=datetime.fromtimestamp(base_epoch + 30 * 60, tz=UTC),
|
||||
)
|
||||
for service in services
|
||||
]
|
||||
insert_reduced_metrics(
|
||||
time_series,
|
||||
[
|
||||
MetricsReducedSampleLast60s(
|
||||
metric_name=metric_name,
|
||||
reduced_fingerprint=ts.fingerprint,
|
||||
timestamp=datetime.fromtimestamp(base_epoch + (30 + minute) * 60, tz=UTC),
|
||||
sum_last=4.0,
|
||||
min_value=2.0,
|
||||
max_value=2.0,
|
||||
sum_values=4.0,
|
||||
count_series=2,
|
||||
count_samples=2,
|
||||
)
|
||||
for ts in time_series
|
||||
for minute in range(30)
|
||||
],
|
||||
)
|
||||
|
||||
assert_spans_shards(clickhouse_node_conns, "time_series_v4", metric_name, total=len(services) * 2)
|
||||
assert_spans_shards(clickhouse_node_conns, "time_series_v4_reduced", metric_name, total=len(services))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 3600, "sum", "sum", step_interval=300)
|
||||
|
||||
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(12)]
|
||||
assert [v["value"] for v in values] == [320.0] * 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"space_agg, expected",
|
||||
[
|
||||
("sum", 12.0), # sum_last: 4 + 8
|
||||
("avg", 3.0), # sum(sum_last) / sum(count_series): 12 / 4
|
||||
("min", 1.0), # min(min)
|
||||
("max", 6.0), # max(max)
|
||||
],
|
||||
)
|
||||
def test_aggregations_across_series(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_reduced_metrics: Callable[..., None],
|
||||
space_agg: str,
|
||||
expected: float,
|
||||
) -> None:
|
||||
"""Aggregating across series reads the pre-aggregated reduced columns:
|
||||
sum/avg from sum_last with the count_series weight, min/max from the
|
||||
min/max columns."""
|
||||
metric_name = f"test_reduction_across_series_{space_agg}"
|
||||
base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
|
||||
|
||||
groups = [
|
||||
# (service, sum_last, min, max, count_series)
|
||||
("a", 4.0, 1.0, 3.0, 2),
|
||||
("b", 8.0, 2.0, 6.0, 2),
|
||||
]
|
||||
time_series = {
|
||||
service: MetricsReducedTimeSeries(
|
||||
metric_name=metric_name,
|
||||
kept_labels={"service": service},
|
||||
timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
|
||||
)
|
||||
for service, _, _, _, _ in groups
|
||||
}
|
||||
insert_reduced_metrics(
|
||||
list(time_series.values()),
|
||||
[
|
||||
MetricsReducedSampleLast60s(
|
||||
metric_name=metric_name,
|
||||
reduced_fingerprint=time_series[service].fingerprint,
|
||||
timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
|
||||
sum_last=sum_last,
|
||||
min_value=min_value,
|
||||
max_value=max_value,
|
||||
sum_values=sum_last,
|
||||
count_series=count_series,
|
||||
count_samples=count_series,
|
||||
)
|
||||
for service, sum_last, min_value, max_value, count_series in groups
|
||||
for minute in range(20)
|
||||
],
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 20 * 60, "avg", space_agg, step_interval=300)
|
||||
|
||||
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
|
||||
assert [v["value"] for v in values] == [expected] * 4
|
||||
|
||||
|
||||
def test_recomputed_minutes_use_only_the_newest_values(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_reduced_metrics: Callable[..., None],
|
||||
) -> None:
|
||||
"""The collector rewrites recent minutes on every refresh, so the same
|
||||
minute exists multiple times with increasing computed_at. Queries must
|
||||
count each minute once, using its newest version: write the same minutes
|
||||
twice with different values and only the second write may show up."""
|
||||
metric_name = "test_reduction_recompute"
|
||||
base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
|
||||
|
||||
time_series = [
|
||||
MetricsReducedTimeSeries(
|
||||
metric_name=metric_name,
|
||||
kept_labels={"service": service},
|
||||
timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
|
||||
)
|
||||
for service in ("a", "b")
|
||||
]
|
||||
|
||||
def minute_rows(sum_last: float, computed_at_offset_seconds: int) -> list[MetricsReducedSampleLast60s]:
|
||||
return [
|
||||
MetricsReducedSampleLast60s(
|
||||
metric_name=metric_name,
|
||||
reduced_fingerprint=ts.fingerprint,
|
||||
timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
|
||||
sum_last=sum_last,
|
||||
min_value=sum_last,
|
||||
max_value=sum_last,
|
||||
sum_values=sum_last,
|
||||
count_series=1,
|
||||
count_samples=1,
|
||||
computed_at=datetime.fromtimestamp(base_epoch + minute * 60 + computed_at_offset_seconds, tz=UTC),
|
||||
)
|
||||
for ts in time_series
|
||||
for minute in range(10)
|
||||
]
|
||||
|
||||
# first write says 1.0; a later rewrite of the same minutes says 5.0
|
||||
insert_reduced_metrics(time_series, minute_rows(sum_last=1.0, computed_at_offset_seconds=120))
|
||||
insert_reduced_metrics(time_series, minute_rows(sum_last=5.0, computed_at_offset_seconds=180))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 10 * 60, "sum", "sum", step_interval=300)
|
||||
|
||||
# 2 groups x 5 minutes x 5.0 per step; the 1.0 rows must not contribute
|
||||
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(2)]
|
||||
assert [v["value"] for v in values] == [50.0] * 2
|
||||
@@ -1,70 +0,0 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.metrics import (
|
||||
MetricsReducedSampleSum60s,
|
||||
MetricsReducedTimeSeries,
|
||||
)
|
||||
from fixtures.querier import aligned_epoch, query_metric_values
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"time_agg, expected",
|
||||
[
|
||||
# 2 groups x 5 minutes x 30.0 per 300s step
|
||||
("rate", 1.0), # 300 / 300s
|
||||
("increase", 300.0),
|
||||
],
|
||||
)
|
||||
def test_counter_rate_and_increase(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_reduced_metrics: Callable[..., None],
|
||||
time_agg: str,
|
||||
expected: float,
|
||||
) -> None:
|
||||
metric_name = f"test_reduction_counter_{time_agg}"
|
||||
base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
|
||||
|
||||
# monotonic cumulative counter: MetricsReducedTimeSeries mirrors the
|
||||
# collector's temporality rewrite to Delta
|
||||
time_series = [
|
||||
MetricsReducedTimeSeries(
|
||||
metric_name=metric_name,
|
||||
kept_labels={"service": service},
|
||||
timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
|
||||
temporality="Cumulative",
|
||||
type_="Sum",
|
||||
is_monotonic=True,
|
||||
)
|
||||
for service in ("a", "b")
|
||||
]
|
||||
assert all(ts.temporality == "Delta" for ts in time_series)
|
||||
|
||||
insert_reduced_metrics(
|
||||
time_series,
|
||||
sum_samples=[
|
||||
MetricsReducedSampleSum60s(
|
||||
metric_name=metric_name,
|
||||
reduced_fingerprint=ts.fingerprint,
|
||||
timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
|
||||
sum_value=30.0,
|
||||
count_series=2,
|
||||
count_samples=2,
|
||||
temporality="Delta",
|
||||
)
|
||||
for ts in time_series
|
||||
for minute in range(20)
|
||||
],
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 20 * 60, time_agg, "sum", step_interval=300)
|
||||
|
||||
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
|
||||
assert [v["value"] for v in values] == [expected] * 4
|
||||
@@ -1,70 +0,0 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.metricreduction import build_recent_gauge_data
|
||||
from fixtures.querier import (
|
||||
aligned_epoch,
|
||||
build_builder_query,
|
||||
get_all_series,
|
||||
index_series_by_label,
|
||||
make_query_request,
|
||||
query_metric_values,
|
||||
)
|
||||
|
||||
SERVICES = ("a", "b")
|
||||
PODS_PER_SERVICE = 2
|
||||
MINUTES = 20
|
||||
|
||||
|
||||
def test_recent_queries_return_full_resolution_totals(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_buffer_metrics: Callable[..., None],
|
||||
) -> None:
|
||||
metric_name = "test_reduction_recent_totals"
|
||||
# samples span [now-25m, now-5m); the query window sits inside the last 24h
|
||||
base_epoch = aligned_epoch(timedelta(minutes=25), step_seconds=300)
|
||||
insert_buffer_metrics(*build_recent_gauge_data(metric_name, base_epoch, SERVICES, PODS_PER_SERVICE, MINUTES))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + MINUTES * 60, "sum", "sum", step_interval=300)
|
||||
|
||||
# 4 raw series x 5 samples x 1.0 per step: full raw resolution, and the
|
||||
# reduced series rows must not be counted (their fingerprints match no
|
||||
# samples, and the time-series lookup filters them out)
|
||||
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
|
||||
assert [v["value"] for v in values] == [float(len(SERVICES) * PODS_PER_SERVICE * 5)] * 4
|
||||
|
||||
|
||||
def test_recent_queries_group_by_full_labels(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_buffer_metrics: Callable[..., None],
|
||||
) -> None:
|
||||
"""Group-by resolves against the raw buffer series rows (full labels), so
|
||||
grouping by the kept label still sees every raw series underneath."""
|
||||
metric_name = "test_reduction_recent_groupby"
|
||||
base_epoch = aligned_epoch(timedelta(minutes=25), step_seconds=300)
|
||||
insert_buffer_metrics(*build_recent_gauge_data(metric_name, base_epoch, SERVICES, PODS_PER_SERVICE, MINUTES))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=base_epoch * 1000,
|
||||
end_ms=(base_epoch + MINUTES * 60) * 1000,
|
||||
queries=[build_builder_query("A", metric_name, "sum", "sum", step_interval=300, group_by=["service"])],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
|
||||
series_by_service = index_series_by_label(get_all_series(response.json(), "A"), "service")
|
||||
assert set(series_by_service.keys()) == set(SERVICES)
|
||||
for service in SERVICES:
|
||||
values = sorted(series_by_service[service]["values"], key=lambda v: v["timestamp"])
|
||||
# 2 pods x 5 samples x 1.0 per step
|
||||
assert [v["value"] for v in values] == [float(PODS_PER_SERVICE * 5)] * 4
|
||||
@@ -1,99 +0,0 @@
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from testcontainers.core.container import Network
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import register_admin
|
||||
from fixtures.clickhouse import create_clickhouse_cluster
|
||||
from fixtures.keeper import create_clickhouse_keeper
|
||||
from fixtures.migrator import create_migrator
|
||||
from fixtures.signoz import create_signoz
|
||||
|
||||
SCHEMA_MIGRATOR_VERSION = "v0.144.6-rc.2"
|
||||
CLICKHOUSE_VERSION = "25.12.5"
|
||||
|
||||
|
||||
@pytest.fixture(name="keeper", scope="package")
|
||||
def keeper_metricreduction(
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.TestContainerDocker:
|
||||
return create_clickhouse_keeper(
|
||||
tmpfs=tmpfs,
|
||||
network=network,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
cache_key="keeper_metricreduction",
|
||||
version=CLICKHOUSE_VERSION,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="clickhouse", scope="package")
|
||||
def clickhouse_metricreduction(
|
||||
tmpfs: Generator[types.LegacyPath, Any],
|
||||
network: Network,
|
||||
keeper: types.TestContainerDocker,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.TestContainerClickhouse:
|
||||
return create_clickhouse_cluster(
|
||||
tmpfs=tmpfs,
|
||||
network=network,
|
||||
keeper=keeper,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
cache_key="clickhouse_metricreduction",
|
||||
shards=2,
|
||||
version=CLICKHOUSE_VERSION,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="migrator", scope="package")
|
||||
def migrator_metricreduction(
|
||||
network: Network,
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.Operation:
|
||||
return create_migrator(
|
||||
network=network,
|
||||
clickhouse=clickhouse,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
cache_key="migrator_metricreduction",
|
||||
version=SCHEMA_MIGRATOR_VERSION,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="signoz", scope="package")
|
||||
def signoz_metricreduction( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
network: Network,
|
||||
zeus: types.TestContainerDocker,
|
||||
gateway: types.TestContainerDocker,
|
||||
sqlstore: types.TestContainerSQL,
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.SigNoz:
|
||||
return create_signoz(
|
||||
network=network,
|
||||
zeus=zeus,
|
||||
gateway=gateway,
|
||||
sqlstore=sqlstore,
|
||||
clickhouse=clickhouse,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
cache_key="signoz_metricreduction",
|
||||
env_overrides={
|
||||
"SIGNOZ_FLAGGER_CONFIG_BOOLEAN_ENABLE__METRICS__REDUCTION": True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="create_user_admin", scope="package")
|
||||
def create_user_admin_metricreduction(signoz: types.SigNoz, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.Operation:
|
||||
return register_admin(signoz, request, pytestconfig, cache_key="create_user_admin_metricreduction")
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,928 +0,0 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import querier, types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.metrics import Metrics
|
||||
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
|
||||
|
||||
log_or_trace_service_counts = {
|
||||
"service-a": 5,
|
||||
"service-b": 3,
|
||||
"service-c": 7,
|
||||
"service-d": 1,
|
||||
}
|
||||
|
||||
metric_values_for_test = {
|
||||
"service-a": 50.0,
|
||||
"service-b": 30.0,
|
||||
"service-c": 70.0,
|
||||
"service-d": 10.0,
|
||||
}
|
||||
|
||||
|
||||
def generate_logs_with_counts(
|
||||
now: datetime,
|
||||
service_counts: dict[str, int],
|
||||
) -> list[Logs]:
|
||||
logs = []
|
||||
for service, count in service_counts.items():
|
||||
for i in range(count):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": service},
|
||||
body=f"{service} log {i}",
|
||||
)
|
||||
)
|
||||
return logs
|
||||
|
||||
|
||||
def generate_traces_with_counts(
|
||||
now: datetime,
|
||||
service_counts: dict[str, int],
|
||||
) -> list[Traces]:
|
||||
traces = []
|
||||
for service, count in service_counts.items():
|
||||
for i in range(count):
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
span_id = TraceIdGenerator.span_id()
|
||||
traces.append(
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
trace_id=trace_id,
|
||||
span_id=span_id,
|
||||
resources={"service.name": service},
|
||||
name=f"{service} span {i}",
|
||||
)
|
||||
)
|
||||
return traces
|
||||
|
||||
|
||||
def generate_metrics_with_values(
|
||||
now: datetime,
|
||||
service_values: dict[str, float],
|
||||
) -> list[Metrics]:
|
||||
metrics = []
|
||||
for service, value in service_values.items():
|
||||
metrics.append(
|
||||
Metrics(
|
||||
metric_name="test.metric",
|
||||
labels={"service.name": service},
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=value,
|
||||
)
|
||||
)
|
||||
return metrics
|
||||
|
||||
|
||||
def make_scalar_query_request(
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
now: datetime,
|
||||
queries: list[dict],
|
||||
lookback_minutes: int = 5,
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((now - timedelta(minutes=lookback_minutes)).timestamp() * 1000),
|
||||
"end": int(now.timestamp() * 1000),
|
||||
"requestType": "scalar",
|
||||
"compositeQuery": {"queries": queries},
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def build_logs_query(
|
||||
name: str = "A",
|
||||
aggregations: list[str] | None = None,
|
||||
group_by: list[str] | None = None,
|
||||
order_by: list[tuple[str, str]] | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict:
|
||||
if aggregations is None:
|
||||
aggregations = ["count()"]
|
||||
|
||||
aggs = [querier.build_logs_aggregation(expr) for expr in aggregations]
|
||||
gb = [querier.build_group_by_field(f, "string", "resource") for f in group_by] if group_by else None
|
||||
order = [querier.build_order_by(name, direction) for name, direction in order_by] if order_by else None
|
||||
|
||||
return querier.build_scalar_query(
|
||||
name=name,
|
||||
signal="logs",
|
||||
aggregations=aggs,
|
||||
group_by=gb,
|
||||
order=order,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
def build_traces_query(
|
||||
name: str = "A",
|
||||
aggregations: list[str] | None = None,
|
||||
group_by: list[str] | None = None,
|
||||
order_by: list[tuple[str, str]] | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict:
|
||||
if aggregations is None:
|
||||
aggregations = ["count()"]
|
||||
|
||||
aggs = [querier.build_logs_aggregation(expr) for expr in aggregations]
|
||||
gb = [querier.build_group_by_field(f, "string", "resource") for f in group_by] if group_by else None
|
||||
order = [querier.build_order_by(name, direction) for name, direction in order_by] if order_by else None
|
||||
|
||||
return querier.build_scalar_query(
|
||||
name=name,
|
||||
signal="traces",
|
||||
aggregations=aggs,
|
||||
group_by=gb,
|
||||
order=order,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
def build_metrics_query(
|
||||
name: str = "A",
|
||||
metric_name: str = "test.metric",
|
||||
time_aggregation: str = "latest",
|
||||
space_aggregation: str = "sum",
|
||||
group_by: list[str] | None = None,
|
||||
order_by: list[tuple[str, str]] | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict:
|
||||
aggs = [querier.build_metrics_aggregation(metric_name, time_aggregation, space_aggregation, "unspecified")]
|
||||
gb = [querier.build_group_by_field(f, "string", "attribute") for f in group_by] if group_by else None
|
||||
order = [querier.build_order_by(name, direction) for name, direction in order_by] if order_by else None
|
||||
|
||||
return querier.build_scalar_query(
|
||||
name=name,
|
||||
signal="metrics",
|
||||
aggregations=aggs,
|
||||
group_by=gb,
|
||||
order=order,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_no_order(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3), ("service-d", 1)],
|
||||
"Logs no order - default desc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("count()", "asc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-b", 3), ("service-a", 5), ("service-c", 7)],
|
||||
"Logs order by agg asc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_agg_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("count()", "desc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3), ("service-d", 1)],
|
||||
"Logs order by agg desc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_grouping_key_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("service.name", "asc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 5), ("service-b", 3), ("service-c", 7), ("service-d", 1)],
|
||||
"Logs order by grouping key asc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_grouping_key_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("service.name", "desc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-c", 7), ("service-b", 3), ("service-a", 5)],
|
||||
"Logs order by grouping key desc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_multiple_aggs_order_by_first_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_logs_query(
|
||||
group_by=["service.name"],
|
||||
aggregations=["count()", "count_distinct(body)"],
|
||||
order_by=[("count()", "asc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_column_order(data, 0, ["service-d", "service-b", "service-a", "service-c"], "First column")
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_multiple_aggs_order_by_second_agg_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_logs_query(
|
||||
group_by=["service.name"],
|
||||
aggregations=["count()", "count_distinct(body)"],
|
||||
order_by=[("count_distinct(body)", "desc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
# count_distinct(body) should equal count() since each log has unique body
|
||||
querier.assert_scalar_column_order(data, 0, ["service-c", "service-a", "service-b", "service-d"], "First column")
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_agg_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("count()", "asc")], limit=2)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-b", 3)],
|
||||
"Logs order by agg asc with limit 2",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_agg_desc_limit_3(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("count()", "desc")], limit=3)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3)],
|
||||
"Logs order by agg desc with limit 3",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_order_by_grouping_key_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("service.name", "asc")], limit=2)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 5), ("service-b", 3)],
|
||||
"Logs order by grouping key asc with limit 2",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_no_order(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3), ("service-d", 1)],
|
||||
"Traces no order - default desc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("count()", "asc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-b", 3), ("service-a", 5), ("service-c", 7)],
|
||||
"Traces order by agg asc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_agg_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("count()", "desc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3), ("service-d", 1)],
|
||||
"Traces order by agg desc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_grouping_key_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("service.name", "asc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 5), ("service-b", 3), ("service-c", 7), ("service-d", 1)],
|
||||
"Traces order by grouping key asc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_grouping_key_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("service.name", "desc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-c", 7), ("service-b", 3), ("service-a", 5)],
|
||||
"Traces order by grouping key desc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_multiple_aggs_order_by_first_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_traces_query(
|
||||
group_by=["service.name"],
|
||||
aggregations=["count()", "count_distinct(trace_id)"],
|
||||
order_by=[("count()", "asc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_column_order(data, 0, ["service-d", "service-b", "service-a", "service-c"], "First column")
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_agg_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("count()", "asc")], limit=2)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-b", 3)],
|
||||
"Traces order by agg asc with limit 2",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_agg_desc_limit_3(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("count()", "desc")], limit=3)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3)],
|
||||
"Traces order by agg desc with limit 3",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_order_by_grouping_key_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("service.name", "asc")], limit=2)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 5), ("service-b", 3)],
|
||||
"Traces order by grouping key asc with limit 2",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_no_order(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_metrics_query(group_by=["service.name"])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[
|
||||
("service-c", 70.0),
|
||||
("service-a", 50.0),
|
||||
("service-b", 30.0),
|
||||
("service-d", 10.0),
|
||||
],
|
||||
"Metrics no order - default desc",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_order_by_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("sum(test.metric)", "asc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[
|
||||
("service-d", 10.0),
|
||||
("service-b", 30.0),
|
||||
("service-a", 50.0),
|
||||
("service-c", 70.0),
|
||||
],
|
||||
"Metrics order by agg asc",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_order_by_grouping_key_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("service.name", "asc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[
|
||||
("service-a", 50.0),
|
||||
("service-b", 30.0),
|
||||
("service-c", 70.0),
|
||||
("service-d", 10.0),
|
||||
],
|
||||
"Metrics order by grouping key asc",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_order_by_agg_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("sum(test.metric)", "asc")],
|
||||
limit=2,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 10.0), ("service-b", 30.0)],
|
||||
"Metrics order by agg asc with limit 2",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_order_by_agg_desc_limit_3(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("sum(test.metric)", "desc")],
|
||||
limit=3,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 70.0), ("service-a", 50.0), ("service-b", 30.0)],
|
||||
"Metrics order by agg desc with limit 3",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_order_by_grouping_key_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("service.name", "asc")],
|
||||
limit=2,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 50.0), ("service-b", 30.0)],
|
||||
"Metrics order by grouping key asc with limit 2",
|
||||
)
|
||||
@@ -0,0 +1,114 @@
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import get_rows, make_query_request
|
||||
|
||||
# Positive coverage for the body-JSON array functions hasAny / hasAll in JSON-body
|
||||
# mode (BODY_JSON_QUERY_ENABLED=true). Here body_v2 arrays resolve to real ClickHouse
|
||||
# Arrays via dynamicElement, so these succeed — unlike legacy body mode, where
|
||||
# hasAny/hasAll xfail (see querierlogs/09_json_body_functions.py).
|
||||
# export_json_types registers the body paths + array element types (tags -> []string,
|
||||
# ids -> []int64) so the builder resolves body.tags/body.ids as arrays.
|
||||
|
||||
|
||||
def test_logs_json_body_has_any_string(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
export_json_types: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""hasAny over a []string body array: matches logs sharing ANY listed value."""
|
||||
now = datetime.now(tz=UTC)
|
||||
logs = [
|
||||
Logs(timestamp=now - timedelta(seconds=3), resources={"service.name": "app-service"}, body_v2=json.dumps({"tags": ["production", "api", "critical"]}), body_promoted="", severity_text="INFO"),
|
||||
Logs(timestamp=now - timedelta(seconds=2), resources={"service.name": "app-service"}, body_v2=json.dumps({"tags": ["staging", "api", "test"]}), body_promoted="", severity_text="INFO"),
|
||||
Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": "app-service"}, body_v2=json.dumps({"tags": ["production", "web", "important"]}), body_promoted="", severity_text="INFO"),
|
||||
]
|
||||
export_json_types(logs)
|
||||
insert_logs(logs)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
# log1 has "critical", log2 has "test", log3 has neither -> 2 matches
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(seconds=10)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[{"type": "builder_query", "spec": {"name": "A", "signal": "logs", "disabled": False, "limit": 100, "offset": 0, "filter": {"expression": "hasAny(body.tags, ['critical', 'test'])"}, "order": [{"key": {"name": "timestamp"}, "direction": "desc"}], "aggregations": [{"expression": "count()"}]}}],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
rows = get_rows(response)
|
||||
assert len(rows) == 2
|
||||
assert all(("critical" in row["data"]["body"]["tags"]) or ("test" in row["data"]["body"]["tags"]) for row in rows)
|
||||
|
||||
|
||||
def test_logs_json_body_has_all_string(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
export_json_types: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""hasAll over a []string body array: matches only logs having ALL listed values."""
|
||||
now = datetime.now(tz=UTC)
|
||||
logs = [
|
||||
Logs(timestamp=now - timedelta(seconds=2), resources={"service.name": "app-service"}, body_v2=json.dumps({"tags": ["production", "api", "critical"]}), body_promoted="", severity_text="INFO"),
|
||||
Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": "app-service"}, body_v2=json.dumps({"tags": ["production", "web", "important"]}), body_promoted="", severity_text="INFO"),
|
||||
]
|
||||
export_json_types(logs)
|
||||
insert_logs(logs)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
# only the second log has both "production" AND "web"
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(seconds=10)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[{"type": "builder_query", "spec": {"name": "A", "signal": "logs", "disabled": False, "limit": 100, "offset": 0, "filter": {"expression": "hasAll(body.tags, ['production', 'web'])"}, "order": [{"key": {"name": "timestamp"}, "direction": "desc"}], "aggregations": [{"expression": "count()"}]}}],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
rows = get_rows(response)
|
||||
assert len(rows) == 1
|
||||
tags = rows[0]["data"]["body"]["tags"]
|
||||
assert "production" in tags and "web" in tags
|
||||
|
||||
|
||||
def test_logs_json_body_has_any_number(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
export_json_types: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""hasAny over a []int64 body array."""
|
||||
now = datetime.now(tz=UTC)
|
||||
logs = [
|
||||
Logs(timestamp=now - timedelta(seconds=2), resources={"service.name": "app-service"}, body_v2=json.dumps({"ids": [100, 200, 300]}), body_promoted="", severity_text="INFO"),
|
||||
Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": "app-service"}, body_v2=json.dumps({"ids": [400, 600, 700]}), body_promoted="", severity_text="INFO"),
|
||||
]
|
||||
export_json_types(logs)
|
||||
insert_logs(logs)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
# only the first log has 300 in ids; 999 matches nothing
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(seconds=10)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[{"type": "builder_query", "spec": {"name": "A", "signal": "logs", "disabled": False, "limit": 100, "offset": 0, "filter": {"expression": "hasAny(body.ids, [300, 999])"}, "order": [{"key": {"name": "timestamp"}, "direction": "desc"}], "aggregations": [{"expression": "count()"}]}}],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
rows = get_rows(response)
|
||||
assert len(rows) == 1
|
||||
assert 300 in rows[0]["data"]["body"]["ids"]
|
||||
141
tests/integration/tests/queriercommon/04_filter_operators.py
Normal file
141
tests/integration/tests/queriercommon/04_filter_operators.py
Normal file
@@ -0,0 +1,141 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import get_column_data_from_response, make_query_request
|
||||
|
||||
# Filter-operator coverage that 01_filter_expression.py (NOT semantics) and
|
||||
# 06_json_body.py (CONTAINS) leave out: ILIKE / NOT LIKE / NOT CONTAINS, the
|
||||
# `key:number` data-type-suffix disambiguator on an ambiguous key, and a
|
||||
# truly-unknown key (rejected as a bad request on this HEAD).
|
||||
#
|
||||
# Data model mirrors 01_filter_expression.py::test_not_filter_expression:
|
||||
# alpha-log: resources region="us-east"; attributes status_code=200 (number)
|
||||
# beta-log: resources status_code="500" (string); attributes region=1 (number)
|
||||
# so region/status_code each appear as both a resource and an attribute across the
|
||||
# two logs — the intentional overlap that context prefix + :type disambiguate.
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_bodies",
|
||||
[
|
||||
pytest.param('resource.region ILIKE "%US-EAST%"', {"alpha-log"}, id="ilike_resource"),
|
||||
pytest.param('body ILIKE "%ALPHA%"', {"alpha-log"}, id="ilike_body"),
|
||||
pytest.param('body NOT CONTAINS "alpha"', {"beta-log"}, id="not_contains_body"),
|
||||
pytest.param('NOT body LIKE "%alpha%"', {"beta-log"}, id="not_like_body"),
|
||||
pytest.param("attribute.status_code:number = 200", {"alpha-log"}, id="datatype_suffix_number"),
|
||||
pytest.param('resource.status_code:string = "500"', {"beta-log"}, id="datatype_suffix_string"),
|
||||
],
|
||||
)
|
||||
def test_logs_filter_operators_and_datatype_suffix(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_bodies: set[str],
|
||||
) -> None:
|
||||
"""ILIKE / NOT LIKE / NOT CONTAINS and the key:number|string suffix resolve correctly."""
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
body="alpha-log",
|
||||
resources={"region": "us-east", "env": "production", "hostname": "host-alpha"},
|
||||
attributes={"status_code": 200, "latency_ms": 350, "error_count": 3},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
body="beta-log",
|
||||
resources={"status_code": "500", "latency_ms": "2500", "error_count": "10"},
|
||||
attributes={"region": 1, "env": 2, "hostname": 3},
|
||||
),
|
||||
]
|
||||
)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"filter": {"expression": expression},
|
||||
"order": [
|
||||
{"key": {"name": "timestamp"}, "direction": "desc"},
|
||||
{"key": {"name": "id"}, "direction": "desc"},
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
assert set(get_column_data_from_response(response.json(), "body")) == expected_bodies
|
||||
|
||||
|
||||
def test_logs_filter_key_not_found(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""A filter on a key that exists in no context is rejected (400).
|
||||
|
||||
NOTE: reflects current HEAD behavior. The parked `convert-not-found-to-warning`
|
||||
change will turn this into a 200 with a warning — update this assertion when it lands.
|
||||
"""
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
body="alpha-log",
|
||||
resources={"region": "us-east"},
|
||||
attributes={"status_code": 200},
|
||||
),
|
||||
]
|
||||
)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"filter": {"expression": 'totally.unknown.key = "x"'},
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
500
tests/integration/tests/querierlogs/01_list.py
Normal file
500
tests/integration/tests/querierlogs/01_list.py
Normal file
@@ -0,0 +1,500 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
assert_identical_query_response,
|
||||
make_query_request,
|
||||
)
|
||||
|
||||
|
||||
def test_logs_list(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 2 logs with different attributes
|
||||
|
||||
Tests:
|
||||
1. Query logs for the last 10 seconds and check if the logs are returned in the correct order
|
||||
2. Query values of severity_text attribute from the autocomplete API
|
||||
3. Query values of severity_text attribute from the fields API
|
||||
4. Query values of code.file attribute from the autocomplete API
|
||||
5. Query values of code.file attribute from the fields API
|
||||
6. Query values of code.line attribute from the autocomplete API
|
||||
7. Query values of code.line attribute from the fields API
|
||||
"""
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=datetime.now(tz=UTC) - timedelta(seconds=1),
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "java",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-001",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "001",
|
||||
},
|
||||
attributes={
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"code.file": "/opt/Integration.java",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"code.line": 120,
|
||||
"telemetry.sdk.language": "java",
|
||||
},
|
||||
body="This is a log message, coming from a java application",
|
||||
severity_text="DEBUG",
|
||||
),
|
||||
Logs(
|
||||
timestamp=datetime.now(tz=UTC),
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "go",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-001",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "001",
|
||||
},
|
||||
attributes={
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"code.file": "/opt/integration.go",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"code.line": 120,
|
||||
"metric.domain_id": "d-001",
|
||||
"telemetry.sdk.language": "go",
|
||||
},
|
||||
body="This is a log message, coming from a go application",
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Query Logs for the last 10 seconds and check if the logs are returned in the correct order
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC) - timedelta(seconds=10)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"order": [
|
||||
{"key": {"name": "timestamp"}, "direction": "desc"},
|
||||
{"key": {"name": "id"}, "direction": "desc"},
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
rows = results[0]["rows"]
|
||||
assert len(rows) == 2
|
||||
|
||||
assert rows[0]["data"]["body"] == "This is a log message, coming from a go application"
|
||||
assert rows[0]["data"]["resources_string"] == {
|
||||
"cloud.account.id": "001",
|
||||
"cloud.provider": "integration",
|
||||
"deployment.environment": "production",
|
||||
"host.name": "linux-001",
|
||||
"os.type": "linux",
|
||||
"service.name": "go",
|
||||
}
|
||||
assert rows[0]["data"]["attributes_string"] == {
|
||||
"code.file": "/opt/integration.go",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"metric.domain_id": "d-001",
|
||||
"telemetry.sdk.language": "go",
|
||||
}
|
||||
assert rows[0]["data"]["attributes_number"] == {"code.line": 120}
|
||||
|
||||
assert rows[1]["data"]["body"] == "This is a log message, coming from a java application"
|
||||
assert rows[1]["data"]["resources_string"] == {
|
||||
"cloud.account.id": "001",
|
||||
"cloud.provider": "integration",
|
||||
"deployment.environment": "production",
|
||||
"host.name": "linux-001",
|
||||
"os.type": "linux",
|
||||
"service.name": "java",
|
||||
}
|
||||
assert rows[1]["data"]["attributes_string"] == {
|
||||
"code.file": "/opt/Integration.java",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"telemetry.sdk.language": "java",
|
||||
}
|
||||
assert rows[1]["data"]["attributes_number"] == {"code.line": 120}
|
||||
|
||||
# Query values of severity_text attribute from the autocomplete API
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v3/autocomplete/attribute_values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"aggregateOperator": "noop",
|
||||
"dataSource": "logs",
|
||||
"aggregateAttribute": "",
|
||||
"attributeKey": "severity_text",
|
||||
"searchText": "",
|
||||
"filterAttributeKeyDataType": "string",
|
||||
"tagType": "resource",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["stringAttributeValues"]
|
||||
assert len(values) == 2
|
||||
assert "DEBUG" in values
|
||||
assert "INFO" in values
|
||||
|
||||
# Query values of severity_text attribute from the fields API
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "logs",
|
||||
"name": "severity_text",
|
||||
"searchText": "",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert len(values) == 2
|
||||
assert "DEBUG" in values
|
||||
assert "INFO" in values
|
||||
|
||||
# Query values of code.file attribute from the autocomplete API
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v3/autocomplete/attribute_values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"aggregateOperator": "noop",
|
||||
"dataSource": "logs",
|
||||
"aggregateAttribute": "",
|
||||
"attributeKey": "code.file",
|
||||
"searchText": "",
|
||||
"filterAttributeKeyDataType": "string",
|
||||
"tagType": "tag",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["stringAttributeValues"]
|
||||
assert len(values) == 2
|
||||
assert "/opt/Integration.java" in values
|
||||
assert "/opt/integration.go" in values
|
||||
|
||||
# Query values of code.file attribute from the fields API
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "logs",
|
||||
"name": "code.file",
|
||||
"searchText": "",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert len(values) == 2
|
||||
assert "/opt/Integration.java" in values
|
||||
assert "/opt/integration.go" in values
|
||||
|
||||
# Query values of code.line attribute from the autocomplete API
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v3/autocomplete/attribute_values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"aggregateOperator": "noop",
|
||||
"dataSource": "logs",
|
||||
"aggregateAttribute": "",
|
||||
"attributeKey": "code.line",
|
||||
"searchText": "",
|
||||
"filterAttributeKeyDataType": "float64",
|
||||
"tagType": "tag",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["numberAttributeValues"]
|
||||
assert len(values) == 1
|
||||
assert 120 in values
|
||||
|
||||
# Query values of code.line attribute from the fields API
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "logs",
|
||||
"name": "code.line",
|
||||
"searchText": "",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["numberValues"]
|
||||
assert len(values) == 1
|
||||
assert 120 in values
|
||||
|
||||
# Query keys from the fields API with context specified in the key
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "logs",
|
||||
"searchText": "resource.servic",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert "service.name" in keys
|
||||
assert any(k["fieldContext"] == "resource" for k in keys["service.name"])
|
||||
|
||||
# Do not treat `metric.` as a context prefix for logs
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "logs",
|
||||
"searchText": "metric.do",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert "metric.domain_id" in keys
|
||||
|
||||
# Query values of service.name resource attribute using context-prefixed key
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "logs",
|
||||
"name": "resource.service.name",
|
||||
"searchText": "",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "go" in values
|
||||
assert "java" in values
|
||||
|
||||
# Query values of metric.domain_id (string attribute) and ensure context collision doesn't break it
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "logs",
|
||||
"name": "metric.domain_id",
|
||||
"searchText": "",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "d-001" in values
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"order_by_context,expected_order",
|
||||
####
|
||||
# Tests:
|
||||
# 1. Query logs ordered by attribute.service.name descending
|
||||
# 2. Query logs ordered by resource.service.name descending
|
||||
# 3. Query logs ordered by service.name descending
|
||||
###
|
||||
[
|
||||
pytest.param("attribute", ["log-002", "log-001", "log-004", "log-003"]),
|
||||
pytest.param("resource", ["log-003", "log-004", "log-001", "log-002"]),
|
||||
pytest.param("", ["log-002", "log-001", "log-003", "log-004"]),
|
||||
],
|
||||
)
|
||||
def test_logs_list_with_order_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
order_by_context: str,
|
||||
expected_order: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 3 logs with service.name in attributes and resources
|
||||
"""
|
||||
|
||||
attribute_resource_pair = [
|
||||
[{"id": "log-001", "service.name": "c"}, {}],
|
||||
[{"id": "log-002", "service.name": "d"}, {}],
|
||||
[{"id": "log-003"}, {"service.name": "b"}],
|
||||
[{"id": "log-004"}, {"service.name": "a"}],
|
||||
]
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=datetime.now(tz=UTC) - timedelta(seconds=3),
|
||||
attributes=attribute_resource_pair[i][0],
|
||||
resources=attribute_resource_pair[i][1],
|
||||
body="Log with DEBUG severity",
|
||||
severity_text="DEBUG",
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
query = {
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"order": [
|
||||
{
|
||||
"key": {
|
||||
"name": "service.name",
|
||||
"fieldContext": order_by_context,
|
||||
},
|
||||
"direction": "desc",
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
query_with_inline_context = {
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"order": [
|
||||
{
|
||||
"key": {
|
||||
"name": f"{order_by_context + '.' if order_by_context else ''}service.name",
|
||||
},
|
||||
"direction": "desc",
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=1)).timestamp() * 1000),
|
||||
end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[query],
|
||||
)
|
||||
|
||||
# Verify that both queries return the same results with specifying context with key name
|
||||
response_with_inline_context = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=1)).timestamp() * 1000),
|
||||
end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[query_with_inline_context],
|
||||
)
|
||||
|
||||
assert_identical_query_response(response, response_with_inline_context)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
rows = results[0]["rows"]
|
||||
ids = [row["data"]["attributes_string"].get("id", "") for row in rows]
|
||||
|
||||
assert ids == expected_order
|
||||
788
tests/integration/tests/querierlogs/02_aggregation.py
Normal file
788
tests/integration/tests/querierlogs/02_aggregation.py
Normal file
@@ -0,0 +1,788 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
assert_identical_query_response,
|
||||
make_query_request,
|
||||
)
|
||||
|
||||
|
||||
def test_logs_time_series_count(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 17 logs with service.name attribute set to "java" and severity_text attribute set to "DEBUG", 23 logs with service.name attribute set to "erlang" and severity_text attribute set to "ERROR", 29 logs with service.name attribute set to "go" and severity_text attribute set to "WARNING".
|
||||
All logs have incrementing code.line attribute, modulo 2 for host.name and cloud.account.id.
|
||||
|
||||
Tests:
|
||||
1. count() of all logs for the last 5 minutes
|
||||
2. count() of all logs where code.line = 7 for last 5 minutes
|
||||
3. count() of all logs where service.name = "erlang" OR cloud.account.id = "000" for last 5 minutes
|
||||
4. count() of all logs grouped by host.name for the last 5 minutes
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = []
|
||||
|
||||
for i in range(17):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(microseconds=i + 1), # These logs will be grouped in the now - 1 minute bucket
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "java",
|
||||
"os.type": "linux",
|
||||
"host.name": f"linux-00{i % 2}",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": f"00{i % 2}",
|
||||
},
|
||||
attributes={
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"code.file": "/opt/Integration.java",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"code.line": i + 1,
|
||||
"telemetry.sdk.language": "java",
|
||||
},
|
||||
body=f"This is a log message, number {i + 1} coming from a java application",
|
||||
severity_text="DEBUG",
|
||||
)
|
||||
)
|
||||
for i in range(23):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=1) - timedelta(microseconds=i + 1), # These logs will be grouped in the now - 2 minute bucket
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "erlang",
|
||||
"os.type": "linux",
|
||||
"host.name": f"linux-00{i % 2}",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": f"00{i % 2}",
|
||||
},
|
||||
attributes={
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"code.file": "/opt/Integration.erlang",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"code.line": i + 1,
|
||||
"telemetry.sdk.language": "erlang",
|
||||
},
|
||||
body=f"This is a log message, number {i + 1} coming from a erlang application",
|
||||
severity_text="ERROR",
|
||||
)
|
||||
)
|
||||
for i in range(29):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=2) - timedelta(microseconds=i + 1), # These logs will be grouped in the now - 3 minute bucket
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "go",
|
||||
"os.type": "linux",
|
||||
"host.name": f"linux-00{i % 2}",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": f"00{i % 2}",
|
||||
},
|
||||
attributes={
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"code.file": "/opt/Integration.go",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"code.line": i + 1,
|
||||
"telemetry.sdk.language": "go",
|
||||
},
|
||||
body=f"This is a log message, number {i + 1} coming from a go application",
|
||||
severity_text="WARNING",
|
||||
)
|
||||
)
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# count() of all logs for the last 5 minutes
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 1
|
||||
|
||||
values = series[0]["values"]
|
||||
assert len(values) == 3
|
||||
|
||||
# Care about the order of the values
|
||||
assert [
|
||||
i
|
||||
for i in values
|
||||
if i
|
||||
not in [
|
||||
{
|
||||
"timestamp": int((now - timedelta(minutes=3)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 29,
|
||||
},
|
||||
{
|
||||
"timestamp": int((now - timedelta(minutes=2)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 23,
|
||||
},
|
||||
{
|
||||
"timestamp": int((now - timedelta(minutes=1)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 17,
|
||||
},
|
||||
]
|
||||
] == []
|
||||
|
||||
# count() of all logs where code.line = 7 for last 5 minutes
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": "code.line = 7"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 1
|
||||
|
||||
values = series[0]["values"]
|
||||
assert len(values) == 3
|
||||
|
||||
# Care about the order of the values
|
||||
assert [
|
||||
i
|
||||
for i in values
|
||||
if i
|
||||
not in [
|
||||
{
|
||||
"timestamp": int((now - timedelta(minutes=3)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 1,
|
||||
},
|
||||
{
|
||||
"timestamp": int((now - timedelta(minutes=2)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 1,
|
||||
},
|
||||
{
|
||||
"timestamp": int((now - timedelta(minutes=1)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 1,
|
||||
},
|
||||
]
|
||||
] == []
|
||||
|
||||
# count() of all logs where service.name = "erlang" OR cloud.account.id = "000" for last 5 minutes
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": "service.name = 'erlang' OR cloud.account.id = '000'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 1
|
||||
|
||||
values = series[0]["values"]
|
||||
assert len(values) == 3
|
||||
|
||||
# Do not care about the order of the values
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=3)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 15,
|
||||
} in values
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=2)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 23,
|
||||
} in values
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=1)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 9,
|
||||
} in values
|
||||
|
||||
# count() of all logs grouped by host.name for the last 5 minutes
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "host.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"order": [{"key": {"name": "host.name"}, "direction": "desc"}],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
response_with_inline_context = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
request_type="time_series",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "resource.host.name:string",
|
||||
}
|
||||
],
|
||||
"order": [{"key": {"name": "host.name"}, "direction": "desc"}],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert_identical_query_response(response, response_with_inline_context)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2
|
||||
|
||||
# Care about the order of the values
|
||||
assert series[0]["labels"] == [
|
||||
{
|
||||
"key": {
|
||||
"name": "host.name",
|
||||
},
|
||||
"value": "linux-001",
|
||||
}
|
||||
]
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=3)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 14,
|
||||
} in series[0]["values"]
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=2)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 11,
|
||||
} in series[0]["values"]
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=1)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 8,
|
||||
} in series[0]["values"]
|
||||
|
||||
assert series[1]["labels"] == [
|
||||
{
|
||||
"key": {
|
||||
"name": "host.name",
|
||||
},
|
||||
"value": "linux-000",
|
||||
}
|
||||
]
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=3)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 15,
|
||||
} in series[1]["values"]
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=2)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 12,
|
||||
} in series[1]["values"]
|
||||
assert {
|
||||
"timestamp": int((now - timedelta(minutes=1)).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"value": 9,
|
||||
} in series[1]["values"]
|
||||
|
||||
|
||||
def test_datatype_collision(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert logs with data type collision scenarios to test DataTypeCollisionHandledFieldName function
|
||||
|
||||
Tests:
|
||||
1. severity_number comparison with string value
|
||||
2. http.status_code with mixed string/number values
|
||||
3. response.time with string values in numeric field
|
||||
4. Edge cases: empty strings
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = []
|
||||
|
||||
# Logs with string values in numeric fields
|
||||
severity_levels = ["DEBUG", "INFO", "WARN"]
|
||||
for i in range(3):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(microseconds=i + 1),
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "java",
|
||||
"os.type": "linux",
|
||||
"host.name": f"linux-00{i % 2}",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": f"00{i % 2}",
|
||||
},
|
||||
attributes={
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"code.file": "/opt/Integration.java",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"code.line": i + 1,
|
||||
"telemetry.sdk.language": "java",
|
||||
"http.status_code": "200", # String value
|
||||
"response.time": "123.45", # String value
|
||||
},
|
||||
body=f"Test log {i + 1} with string values",
|
||||
severity_text=severity_levels[i], # DEBUG(5-8), INFO(9-12), WARN(13-16)
|
||||
)
|
||||
)
|
||||
|
||||
# Logs with numeric values in string fields
|
||||
severity_levels_2 = ["ERROR", "FATAL", "TRACE", "DEBUG"]
|
||||
for i in range(4):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(microseconds=i + 10),
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "go",
|
||||
"os.type": "linux",
|
||||
"host.name": f"linux-00{i % 2}",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": f"00{i % 2}",
|
||||
},
|
||||
attributes={
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"code.file": "/opt/integration.go",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"code.line": i + 1,
|
||||
"telemetry.sdk.language": "go",
|
||||
"http.status_code": 404, # Numeric value
|
||||
"response.time": 456.78, # Numeric value
|
||||
},
|
||||
body=f"Test log {i + 4} with numeric values",
|
||||
severity_text=severity_levels_2[i], # ERROR(17-20), FATAL(21-24), TRACE(1-4), DEBUG(5-8)
|
||||
)
|
||||
)
|
||||
|
||||
# Edge case: empty string and zero value
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(microseconds=20),
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "python",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-002",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "002",
|
||||
},
|
||||
attributes={
|
||||
"log.iostream": "stdout",
|
||||
"logtag": "F",
|
||||
"code.file": "/opt/integration.py",
|
||||
"code.function": "com.example.Integration.process",
|
||||
"code.line": 1,
|
||||
"telemetry.sdk.language": "python",
|
||||
"http.status_code": "", # Empty string
|
||||
"response.time": 0, # Zero value
|
||||
},
|
||||
body="Edge case test log",
|
||||
severity_text="ERROR",
|
||||
)
|
||||
)
|
||||
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# count() of all logs for the where severity_number > '7'
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "scalar",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": "severity_number > '7'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
count = results[0]["data"][0][0]
|
||||
assert count == 5
|
||||
|
||||
# count() of all logs for the where severity_number > '7.0'
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "scalar",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": "severity_number > '7.0'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
count = results[0]["data"][0][0]
|
||||
assert count == 5
|
||||
|
||||
# Test 2: severity_number comparison with string value
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "scalar",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": "severity_number = '13'"}, # String comparison with numeric field
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
count = results[0]["data"][0][0]
|
||||
# WARN severity maps to 13-16 range, so should find 1 log with severity_number = 13
|
||||
assert count == 1
|
||||
|
||||
# Test 3: http.status_code with numeric value (query contains number, actual value is string "200")
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "scalar",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": "http.status_code = 200"}, # Numeric comparison with string field
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
count = results[0]["data"][0][0]
|
||||
# Should return 3 logs with http.status_code = "200" (first 3 logs have string value "200")
|
||||
assert count == 3
|
||||
|
||||
# Test 4: http.status_code with string value (query contains string, actual value is numeric 404)
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "scalar",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": "http.status_code = '404'"}, # String comparison with numeric field
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
count = results[0]["data"][0][0]
|
||||
# Should return 4 logs with http.status_code = 404 (next 4 logs have numeric value 404)
|
||||
assert count == 4
|
||||
|
||||
# Test 5: Edge case - empty string comparison
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).replace(second=0, microsecond=0).timestamp() * 1000),
|
||||
"requestType": "scalar",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": "http.status_code = ''"}, # Empty string comparison
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": True, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
count = results[0]["data"][0][0]
|
||||
# Should return 1 log with empty http.status_code (edge case log)
|
||||
assert count == 1
|
||||
849
tests/integration/tests/querierlogs/03_fill.py
Normal file
849
tests/integration/tests/querierlogs/03_fill.py
Normal file
@@ -0,0 +1,849 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
assert_minutely_bucket_values,
|
||||
find_named_result,
|
||||
index_series_by_label,
|
||||
)
|
||||
|
||||
|
||||
def test_logs_fill_gaps(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillGaps for logs without groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
resources={"service.name": "test-service"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Log at minute 3",
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=1),
|
||||
resources={"service.name": "test-service"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Log at minute 1",
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": True},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) >= 1
|
||||
|
||||
values = series[0]["values"]
|
||||
# Logs are exactly at minute -3 and minute -1, so counts should be 1 there and 0 everywhere else
|
||||
ts_min_1 = int((now - timedelta(minutes=1)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
assert_minutely_bucket_values(
|
||||
values,
|
||||
now,
|
||||
expected_by_ts={ts_min_1: 1, ts_min_3: 1},
|
||||
context="logs/fillGaps",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_fill_gaps_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillGaps for logs with groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
resources={"service.name": "service-a"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Log from service A",
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
resources={"service.name": "service-b"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Log from service B",
|
||||
severity_text="ERROR",
|
||||
),
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": True},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2, "Expected 2 series for 2 service groups"
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
series_by_service = index_series_by_label(series, "service.name")
|
||||
|
||||
assert set(series_by_service.keys()) == {"service-a", "service-b"}
|
||||
|
||||
# service-a has one log at minute -3, service-b at minute -2
|
||||
expectations = {
|
||||
"service-a": {ts_min_3: 1.0},
|
||||
"service-b": {ts_min_2: 1.0},
|
||||
}
|
||||
|
||||
for service_name, s in series_by_service.items():
|
||||
assert_minutely_bucket_values(
|
||||
s["values"],
|
||||
now,
|
||||
expected_by_ts=expectations[service_name],
|
||||
context=f"logs/fillGaps/{service_name}",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_fill_gaps_formula(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillGaps for logs with formula.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
resources={"service.name": "test"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Test log",
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
resources={"service.name": "another-test"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Another test log",
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"filter": {"expression": "service.name = 'test'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "B",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"filter": {"expression": "service.name = 'another-test'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_formula",
|
||||
"spec": {
|
||||
"name": "F1",
|
||||
"expression": "A + B",
|
||||
"disabled": False,
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": True},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
f1 = find_named_result(results, "F1")
|
||||
assert f1 is not None, "Expected formula result named F1"
|
||||
|
||||
aggregations = f1.get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) >= 1
|
||||
|
||||
assert_minutely_bucket_values(
|
||||
series[0]["values"],
|
||||
now,
|
||||
expected_by_ts={ts_min_3: 1, ts_min_2: 1},
|
||||
context="logs/fillGaps/F1",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_fill_gaps_formula_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillGaps for logs with formula and groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
resources={"service.name": "group1"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Test log",
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
resources={"service.name": "group2"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Test log 2",
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "B",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_formula",
|
||||
"spec": {
|
||||
"name": "F1",
|
||||
"expression": "A + B",
|
||||
"disabled": False,
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": True},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
f1 = find_named_result(results, "F1")
|
||||
assert f1 is not None, "Expected formula result named F1"
|
||||
|
||||
aggregations = f1.get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2
|
||||
|
||||
series_by_service = index_series_by_label(series, "service.name")
|
||||
|
||||
assert set(series_by_service.keys()) == {"group1", "group2"}
|
||||
|
||||
expectations = {
|
||||
"group1": {ts_min_3: 2.0},
|
||||
"group2": {ts_min_2: 2.0},
|
||||
}
|
||||
|
||||
for service_name, s in series_by_service.items():
|
||||
assert_minutely_bucket_values(
|
||||
s["values"],
|
||||
now,
|
||||
expected_by_ts=expectations[service_name],
|
||||
context=f"logs/fillGaps/F1/{service_name}",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_fill_zero(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillZero function for logs without groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
resources={"service.name": "test"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Test log",
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
"functions": [{"name": "fillZero"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0].get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) >= 1
|
||||
values = series[0]["values"]
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
assert_minutely_bucket_values(
|
||||
values,
|
||||
now,
|
||||
expected_by_ts={ts_min_3: 1},
|
||||
context="logs/fillZero",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_fill_zero_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillZero function for logs with groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
resources={"service.name": "service-a"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Log A",
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
resources={"service.name": "service-b"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Log B",
|
||||
severity_text="ERROR",
|
||||
),
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
"functions": [{"name": "fillZero"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2, "Expected 2 series for 2 service groups"
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
series_by_service = index_series_by_label(series, "service.name")
|
||||
|
||||
assert set(series_by_service.keys()) == {"service-a", "service-b"}
|
||||
expectations = {
|
||||
"service-a": {ts_min_3: 1.0},
|
||||
"service-b": {ts_min_2: 1.0},
|
||||
}
|
||||
|
||||
for service_name, s in series_by_service.items():
|
||||
assert_minutely_bucket_values(
|
||||
s["values"],
|
||||
now,
|
||||
expected_by_ts=expectations[service_name],
|
||||
context=f"logs/fillZero/{service_name}",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_fill_zero_formula(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillZero function for logs with formula.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
resources={"service.name": "test"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Test log",
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
resources={"service.name": "another-test"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Another log",
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"filter": {"expression": "service.name = 'test'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "B",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"filter": {"expression": "service.name = 'another-test'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_formula",
|
||||
"spec": {
|
||||
"name": "F1",
|
||||
"expression": "A + B",
|
||||
"disabled": False,
|
||||
"functions": [{"name": "fillZero"}],
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
f1 = find_named_result(results, "F1")
|
||||
assert f1 is not None, "Expected formula result named F1"
|
||||
|
||||
aggregations = f1.get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) >= 1
|
||||
values = series[0]["values"]
|
||||
|
||||
assert_minutely_bucket_values(
|
||||
values,
|
||||
now,
|
||||
expected_by_ts={ts_min_3: 1, ts_min_2: 1},
|
||||
context="logs/fillZero/F1",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_fill_zero_formula_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillZero function for logs with formula and groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
resources={"service.name": "group1"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Test log",
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
resources={"service.name": "group2"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body="Test log 2",
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "B",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_formula",
|
||||
"spec": {
|
||||
"name": "F1",
|
||||
"expression": "A + B",
|
||||
"disabled": False,
|
||||
"functions": [{"name": "fillZero"}],
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
f1 = find_named_result(results, "F1")
|
||||
assert f1 is not None, "Expected formula result named F1"
|
||||
|
||||
aggregations = f1.get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2
|
||||
|
||||
series_by_service = index_series_by_label(series, "service.name")
|
||||
|
||||
assert set(series_by_service.keys()) == {"group1", "group2"}
|
||||
|
||||
expectations = {
|
||||
"group1": {ts_min_3: 2.0},
|
||||
"group2": {ts_min_2: 2.0},
|
||||
}
|
||||
|
||||
for service_name, s in series_by_service.items():
|
||||
assert_minutely_bucket_values(
|
||||
s["values"],
|
||||
now,
|
||||
expected_by_ts=expectations[service_name],
|
||||
context=f"logs/fillZero/F1/{service_name}",
|
||||
)
|
||||
193
tests/integration/tests/querierlogs/04_formula.py
Normal file
193
tests/integration/tests/querierlogs/04_formula.py
Normal file
@@ -0,0 +1,193 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
build_formula_query,
|
||||
build_group_by_field,
|
||||
build_logs_aggregation,
|
||||
build_order_by,
|
||||
build_scalar_query,
|
||||
find_named_result,
|
||||
make_query_request,
|
||||
)
|
||||
|
||||
|
||||
def test_logs_formula_orderby_and_limit(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test that formula results are correctly ordered and limited when
|
||||
order and limit are applied on the formula.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
logs: list[Logs] = []
|
||||
# For service-i (i in 0..9): insert (10 - i) ERROR logs and 2 INFO logs.
|
||||
# A counts ERROR, B counts INFO, so A/B = (10 - i) / 2.
|
||||
# service-0 ratio = 5.0 (highest), service-9 ratio = 0.5 (lowest).
|
||||
for i in range(10):
|
||||
for j in range(10 - i):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=j + 1),
|
||||
resources={"service.name": f"service-{i}"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body=f"Error log {i}-{j}",
|
||||
severity_text="ERROR",
|
||||
)
|
||||
)
|
||||
for k in range(2):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=k + 1),
|
||||
resources={"service.name": f"service-{i}"},
|
||||
attributes={"code.file": "test.py"},
|
||||
body=f"Info log {i}-{k}",
|
||||
severity_text="INFO",
|
||||
)
|
||||
)
|
||||
# Extra INFO-only services that appear in B but not in A. The formula
|
||||
for name in ("service-info-only-1", "service-info-only-2"):
|
||||
for k in range(2):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(minutes=k + 1),
|
||||
resources={"service.name": name},
|
||||
attributes={"code.file": "test.py"},
|
||||
body=f"Info log {name}-{k}",
|
||||
severity_text="INFO",
|
||||
)
|
||||
)
|
||||
|
||||
# Logs look like this (columns = minutes before `now`; query range is
|
||||
# (now - 15m, now], so the `now` column is the exclusive upper bound and
|
||||
# no log lands there). E = ERROR, I = INFO, X = both at that minute.
|
||||
#
|
||||
# t-10 t-9 t-8 t-7 t-6 t-5 t-4 t-3 t-2 t-1 |now | A B A/B
|
||||
# service-0: E E E E E E E E X X | | 10 2 5.0
|
||||
# service-1: . E E E E E E E X X | | 9 2 4.5
|
||||
# service-2: . . E E E E E E X X | | 8 2 4.0
|
||||
# service-3: . . . E E E E E X X | | 7 2 3.5
|
||||
# service-4: . . . . E E E E X X | | 6 2 3.0
|
||||
# service-5: . . . . . E E E X X | | 5 2 2.5
|
||||
# service-6: . . . . . . E E X X | | 4 2 2.0
|
||||
# service-7: . . . . . . . E X X | | 3 2 1.5
|
||||
# service-8: . . . . . . . . X X | | 2 2 1.0
|
||||
# service-9: . . . . . . . . I X | | 1 2 0.5
|
||||
# info-only-1: . . . . . . . . I I | | 0* 2 0.0
|
||||
# info-only-2: . . . . . . . . I I | | 0* 2 0.0
|
||||
#
|
||||
# * A is missing for the info-only services; because A is count(), the
|
||||
# formula evaluator defaults missing A to 0, yielding A/B = 0.
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
result = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=15)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="scalar",
|
||||
queries=[
|
||||
build_scalar_query(
|
||||
name="A",
|
||||
signal="logs",
|
||||
aggregations=[build_logs_aggregation("count()")],
|
||||
group_by=[build_group_by_field("service.name")],
|
||||
filter_expression="severity_text = 'ERROR'",
|
||||
disabled=True,
|
||||
),
|
||||
build_scalar_query(
|
||||
name="B",
|
||||
signal="logs",
|
||||
aggregations=[build_logs_aggregation("count()")],
|
||||
group_by=[build_group_by_field("service.name")],
|
||||
filter_expression="severity_text = 'INFO'",
|
||||
disabled=True,
|
||||
),
|
||||
build_formula_query(
|
||||
"F1",
|
||||
"A / B",
|
||||
order=[build_order_by("__result", "desc")],
|
||||
limit=3,
|
||||
),
|
||||
build_formula_query(
|
||||
"F2",
|
||||
"A / B",
|
||||
order=[build_order_by("__result", "desc")],
|
||||
),
|
||||
build_formula_query(
|
||||
"F3",
|
||||
"A / B",
|
||||
order=[build_order_by("__result", "asc")],
|
||||
limit=3,
|
||||
),
|
||||
build_formula_query(
|
||||
"F4",
|
||||
"A / B",
|
||||
order=[build_order_by("__result", "asc")],
|
||||
),
|
||||
],
|
||||
)
|
||||
assert result.status_code == HTTPStatus.OK
|
||||
assert result.json()["status"] == "success"
|
||||
|
||||
results = result.json()["data"]["data"]["results"]
|
||||
|
||||
def extract_services_and_values(query_name: str) -> tuple[list, list]:
|
||||
res = find_named_result(results, query_name)
|
||||
assert res is not None, f"Expected formula result named {query_name}"
|
||||
cols = res["columns"]
|
||||
s_col = next(i for i, c in enumerate(cols) if c["name"] == "service.name")
|
||||
v_col = next(i for i, c in enumerate(cols) if c["name"] == "__result")
|
||||
rows = res["data"]
|
||||
return [row[s_col] for row in rows], [row[v_col] for row in rows]
|
||||
|
||||
# Because A is count(), canDefaultZero["A"] is true; the formula evaluator
|
||||
# defaults A to 0 for services that exist only in B. So the two INFO-only
|
||||
# services appear in the formula result with value 0.0 (extreme bottom in
|
||||
# desc order, extreme top in asc order). Their relative ordering is not
|
||||
# deterministic across separate formula evaluations (tied values).
|
||||
info_only_services = {"service-info-only-1", "service-info-only-2"}
|
||||
|
||||
# F2: desc, no limit -> 12 rows in descending order by value.
|
||||
f2_services, f2_values = extract_services_and_values("F2")
|
||||
assert len(f2_services) == 12, f"F2: expected 12 rows with no limit, got {len(f2_services)}"
|
||||
assert f2_values == [5.0, 4.5, 4.0, 3.5, 3.0, 2.5, 2.0, 1.5, 1.0, 0.5, 0.0, 0.0], f2_values
|
||||
# Top 10 have distinct positive values -> deterministic service ordering.
|
||||
assert f2_services[:10] == [f"service-{i}" for i in range(10)], f2_services[:10]
|
||||
# Tail 2 are the INFO-only services tied at 0.0 (order between them not guaranteed).
|
||||
assert set(f2_services[10:]) == info_only_services, f2_services[10:]
|
||||
|
||||
# F1: desc + limit 3 -> must be exactly the first 3 rows of F2.
|
||||
# Top 3 are not in the tie region, so prefix equality is safe.
|
||||
f1_services, f1_values = extract_services_and_values("F1")
|
||||
assert len(f1_services) == 3, f"F1: expected 3 rows after limit, got {len(f1_services)}"
|
||||
assert f1_services == f2_services[:3], f"F1 services {f1_services} are not the prefix of F2 services {f2_services}"
|
||||
assert f1_values == f2_values[:3], f"F1 values {f1_values} are not the prefix of F2 values {f2_values}"
|
||||
|
||||
# F4: asc, no limit -> 12 rows in ascending order by value.
|
||||
f4_services, f4_values = extract_services_and_values("F4")
|
||||
assert len(f4_services) == 12, f"F4: expected 12 rows with no limit, got {len(f4_services)}"
|
||||
assert f4_values == sorted(f4_values), f"F4 not ascending: {f4_values}"
|
||||
# First 2 are the INFO-only services tied at 0.0 (order between them not guaranteed).
|
||||
assert set(f4_services[:2]) == info_only_services, f4_services[:2]
|
||||
assert f4_values[:2] == [0.0, 0.0], f4_values[:2]
|
||||
# Tail 10 are service-9 down to service-0 by value.
|
||||
assert f4_services[2:] == [f"service-{i}" for i in reversed(range(10))], f4_services[2:]
|
||||
assert f4_values[2:] == [(10 - i) / 2 for i in reversed(range(10))], f4_values[2:]
|
||||
|
||||
# F3: asc + limit 3 -> values must match F4[:3] exactly; service set must
|
||||
# match too. Direct prefix equality on services would be flaky because the
|
||||
# two tied INFO-only entries can swap order between formula evaluations.
|
||||
f3_services, f3_values = extract_services_and_values("F3")
|
||||
assert len(f3_services) == 3, f"F3: expected 3 rows after limit, got {len(f3_services)}"
|
||||
assert f3_values == f4_values[:3], f"F3 values {f3_values} do not match F4[:3] values {f4_values[:3]}"
|
||||
assert set(f3_services) == set(f4_services[:3]), f"F3 services {f3_services} do not match F4[:3] services {f4_services[:3]}"
|
||||
364
tests/integration/tests/querierlogs/05_trace_id_filter.py
Normal file
364
tests/integration/tests/querierlogs/05_trace_id_filter.py
Normal file
@@ -0,0 +1,364 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
make_query_request,
|
||||
)
|
||||
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
|
||||
|
||||
|
||||
def test_logs_list_filter_by_trace_id(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Tests that filtering logs by trace_id uses the trace_summary lookup to
|
||||
narrow the query window before scanning the logs table:
|
||||
1. Returns the matching logs (narrow window, single bucket), including a log
|
||||
flushed shortly after the span ends — kept by the configured padding.
|
||||
2. Does not return duplicate logs when the query window should span multiple
|
||||
exponential buckets (>1 h). The window is clamped to the trace's recorded
|
||||
range widened by the padding, so the post-span log survives the clamp.
|
||||
3. Returns no results when the query window does not contain the trace.
|
||||
4. Logs carrying a trace_id whose trace is NOT in trace_summary (e.g.
|
||||
traces disabled) are still returned — the lookup miss must not
|
||||
short-circuit logs queries.
|
||||
"""
|
||||
target_trace_id = TraceIdGenerator.trace_id()
|
||||
orphan_trace_id = TraceIdGenerator.trace_id()
|
||||
target_root_span_id = TraceIdGenerator.span_id()
|
||||
target_child_span_id = TraceIdGenerator.span_id()
|
||||
orphan_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
common_resources = {
|
||||
"deployment.environment": "production",
|
||||
"service.name": "logs-trace-filter-service",
|
||||
"cloud.provider": "integration",
|
||||
}
|
||||
|
||||
# Populate signoz_traces.distributed_trace_summary by inserting spans for
|
||||
# the target trace_id. trace_summary records min/max of span timestamps
|
||||
# (it ignores span duration), so two spans are inserted to give the trace
|
||||
# a non-trivial recorded window of [now-10s, now-5s].
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_root_span_id,
|
||||
parent_span_id="",
|
||||
name="root-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_child_span_id,
|
||||
parent_span_id=target_root_span_id,
|
||||
name="child-span",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# Insert logs:
|
||||
# - one with the target trace_id, at a timestamp within the trace's
|
||||
# recorded window (now-10s..now-5s, padded ±1s).
|
||||
# - one with the target trace_id flushed ~3s AFTER the span's recorded end
|
||||
# (now-2s). This is outside the ±1s base pad but inside the multi-minute
|
||||
# log_trace_id_window_padding, so it must still be returned.
|
||||
# - one with an orphan trace_id whose trace was never ingested — used to
|
||||
# verify the lookup miss does NOT short-circuit logs queries.
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=7),
|
||||
resources=common_resources,
|
||||
attributes={"http.method": "GET"},
|
||||
body="log inside the target trace window",
|
||||
severity_text="INFO",
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_root_span_id,
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
resources=common_resources,
|
||||
attributes={"http.method": "POST"},
|
||||
body="log flushed after the span ends, within padding window",
|
||||
severity_text="INFO",
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_root_span_id,
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
resources=common_resources,
|
||||
attributes={"http.method": "PUT"},
|
||||
body="log with a trace_id absent from trace_summary",
|
||||
severity_text="INFO",
|
||||
trace_id=orphan_trace_id,
|
||||
span_id=orphan_span_id,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
def _query(start_ms: int, end_ms: int, trace_id: str) -> tuple[list, list[str]]:
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=start_ms,
|
||||
end_ms=end_ms,
|
||||
request_type="raw",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"filter": {"expression": f"trace_id = '{trace_id}'"},
|
||||
"order": [
|
||||
{"key": {"name": "timestamp"}, "direction": "desc"},
|
||||
{"key": {"name": "id"}, "direction": "desc"},
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
rows = response.json()["data"]["data"]["results"][0]["rows"] or []
|
||||
warning = (response.json().get("data") or {}).get("warning") or {}
|
||||
messages = [w.get("message", "") for w in (warning.get("warnings") or [])]
|
||||
return rows, messages
|
||||
|
||||
outside_range_msg = "lies outside the selected time range"
|
||||
|
||||
now_ms = int(now.timestamp() * 1000)
|
||||
|
||||
inside_window_body = "log inside the target trace window"
|
||||
post_span_body = "log flushed after the span ends, within padding window"
|
||||
|
||||
# --- Test 1: narrow window (single bucket, <1 h) ---
|
||||
narrow_start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
narrow_rows, narrow_warnings = _query(narrow_start_ms, now_ms, target_trace_id)
|
||||
|
||||
assert len(narrow_rows) == 2, f"Expected 2 logs for trace_id filter (narrow window), got {len(narrow_rows)}"
|
||||
assert {r["data"]["trace_id"] for r in narrow_rows} == {target_trace_id}
|
||||
narrow_bodies = {r["data"]["body"] for r in narrow_rows}
|
||||
assert inside_window_body in narrow_bodies
|
||||
assert post_span_body in narrow_bodies, "post-span log should be returned within the padding window"
|
||||
assert not any(outside_range_msg in m for m in narrow_warnings), f"Did not expect outside-range warning, got {narrow_warnings}"
|
||||
|
||||
# --- Test 2: wide window (>1 h, clamp to the padded timerange from trace_summary) ---
|
||||
# Should return exactly the two target logs — no duplicates from multi-bucket
|
||||
# scan, and the post-span log survives the clamp only because of the padding.
|
||||
wide_start_ms = int((now - timedelta(hours=12)).timestamp() * 1000)
|
||||
wide_rows, wide_warnings = _query(wide_start_ms, now_ms, target_trace_id)
|
||||
|
||||
assert len(wide_rows) == 2, f"Expected 2 logs for trace_id filter (wide window, multi-bucket), got {len(wide_rows)} — possible duplicate-log regression or padding not applied"
|
||||
assert {r["data"]["trace_id"] for r in wide_rows} == {target_trace_id}
|
||||
wide_bodies = {r["data"]["body"] for r in wide_rows}
|
||||
assert inside_window_body in wide_bodies
|
||||
assert post_span_body in wide_bodies, "post-span log should survive the clamp because of the padding"
|
||||
assert not any(outside_range_msg in m for m in wide_warnings), f"Did not expect outside-range warning, got {wide_warnings}"
|
||||
|
||||
# --- Test 3: window that does not contain the trace returns no results + warning ---
|
||||
past_start_ms = int((now - timedelta(hours=6)).timestamp() * 1000)
|
||||
past_end_ms = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
past_rows, past_warnings = _query(past_start_ms, past_end_ms, target_trace_id)
|
||||
|
||||
assert len(past_rows) == 0, f"Expected 0 logs for trace_id filter outside time window, got {len(past_rows)}"
|
||||
assert any(outside_range_msg in m for m in past_warnings), f"Expected outside-range warning, got warnings={past_warnings}"
|
||||
|
||||
# --- Test 4: trace_id not present in trace_summary still returns logs (no warning) ---
|
||||
orphan_rows, orphan_warnings = _query(narrow_start_ms, now_ms, orphan_trace_id)
|
||||
|
||||
assert len(orphan_rows) == 1, f"Expected 1 log for orphan trace_id (no trace_summary entry), got {len(orphan_rows)} — logs query may have been incorrectly short-circuited"
|
||||
assert orphan_rows[0]["data"]["trace_id"] == orphan_trace_id
|
||||
assert not any(outside_range_msg in m for m in orphan_warnings), f"Did not expect outside-range warning for orphan trace_id, got {orphan_warnings}"
|
||||
|
||||
|
||||
def test_logs_aggregation_filter_by_trace_id(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Tests that the trace_id time-range optimization also applies to
|
||||
non-window-list (time_series / aggregation) logs queries:
|
||||
1. Wide query window containing the trace returns the correct count.
|
||||
2. Query window outside the trace's time range short-circuits to an
|
||||
empty result.
|
||||
3. A trace_id with no row in trace_summary (e.g. traces disabled) still
|
||||
returns the matching logs — the lookup miss must not short-circuit
|
||||
logs aggregation queries.
|
||||
"""
|
||||
target_trace_id = TraceIdGenerator.trace_id()
|
||||
orphan_trace_id = TraceIdGenerator.trace_id()
|
||||
target_root_span_id = TraceIdGenerator.span_id()
|
||||
target_child_span_id = TraceIdGenerator.span_id()
|
||||
orphan_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
common_resources = {
|
||||
"deployment.environment": "production",
|
||||
"service.name": "logs-trace-agg-service",
|
||||
"cloud.provider": "integration",
|
||||
}
|
||||
|
||||
# trace_summary records min/max of span timestamps (it ignores duration),
|
||||
# so insert two spans to give the trace a recorded window wide enough to
|
||||
# comfortably contain the log timestamps below.
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_root_span_id,
|
||||
parent_span_id="",
|
||||
name="root-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_child_span_id,
|
||||
parent_span_id=target_root_span_id,
|
||||
name="child-span",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# Two logs for the target trace_id, both inside the recorded trace window.
|
||||
# One additional log carries an orphan trace_id with no row in
|
||||
# trace_summary — used to verify that the lookup miss does not
|
||||
# short-circuit logs aggregations.
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=7),
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
body="log A inside trace window",
|
||||
severity_text="INFO",
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_root_span_id,
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=6),
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
body="log B inside trace window",
|
||||
severity_text="INFO",
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_root_span_id,
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
body="log with a trace_id absent from trace_summary",
|
||||
severity_text="INFO",
|
||||
trace_id=orphan_trace_id,
|
||||
span_id=orphan_span_id,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
def _count(start_ms: int, end_ms: int, trace_id: str) -> tuple[float, list[str]]:
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=start_ms,
|
||||
end_ms=end_ms,
|
||||
request_type="time_series",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": f"trace_id = '{trace_id}'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
warning = (response.json().get("data") or {}).get("warning") or {}
|
||||
messages = [w.get("message", "") for w in (warning.get("warnings") or [])]
|
||||
aggregations = results[0].get("aggregations") or []
|
||||
if not aggregations:
|
||||
return 0, messages
|
||||
series = aggregations[0].get("series") or []
|
||||
if not series:
|
||||
return 0, messages
|
||||
return sum(v["value"] for v in series[0]["values"]), messages
|
||||
|
||||
outside_range_msg = "lies outside the selected time range"
|
||||
|
||||
now_ms = int(now.timestamp() * 1000)
|
||||
narrow_start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
|
||||
# --- Test 1: wide window (>1 h) containing the trace returns 2 logs ---
|
||||
wide_start_ms = int((now - timedelta(hours=12)).timestamp() * 1000)
|
||||
wide_count, wide_warnings = _count(wide_start_ms, now_ms, target_trace_id)
|
||||
assert wide_count == 2, f"Expected count=2 for trace_id aggregation (wide window), got {wide_count}"
|
||||
assert not any(outside_range_msg in m for m in wide_warnings), f"Did not expect outside-range warning, got {wide_warnings}"
|
||||
|
||||
# --- Test 2: window outside the trace short-circuits to empty + warning ---
|
||||
past_start_ms = int((now - timedelta(hours=6)).timestamp() * 1000)
|
||||
past_end_ms = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
past_count, past_warnings = _count(past_start_ms, past_end_ms, target_trace_id)
|
||||
assert past_count == 0, f"Expected count=0 for trace_id aggregation outside time window, got {past_count}"
|
||||
assert any(outside_range_msg in m for m in past_warnings), f"Expected outside-range warning, got warnings={past_warnings}"
|
||||
|
||||
# --- Test 3: trace_id not present in trace_summary still returns logs (no warning) ---
|
||||
orphan_count, orphan_warnings = _count(narrow_start_ms, now_ms, orphan_trace_id)
|
||||
assert orphan_count == 1, f"Expected count=1 for orphan trace_id aggregation, got {orphan_count} — query may have been incorrectly short-circuited"
|
||||
assert not any(outside_range_msg in m for m in orphan_warnings), f"Did not expect outside-range warning for orphan trace_id, got {orphan_warnings}"
|
||||
276
tests/integration/tests/querierlogs/09_json_body_functions.py
Normal file
276
tests/integration/tests/querierlogs/09_json_body_functions.py
Normal file
@@ -0,0 +1,276 @@
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
|
||||
# Body-JSON array/token functions on the logs body. has() success paths are already
|
||||
# covered by 06_json_body.py::test_logs_json_body_array_membership; this file adds the
|
||||
# sibling functions hasAny / hasAll / hasToken (success) and the function-operator error
|
||||
# paths (has/hasToken on a non-body key, non-string token) which must be rejected.
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason="hasAny/hasAll over a body-JSON array return 500 in legacy body mode (use_json_body off): ClickHouse 'Argument 0 for hasAny must be an array but has type String'. has() handles body arrays here; hasAny/hasAll do not (they work only in JSON-body mode).",
|
||||
strict=False,
|
||||
)
|
||||
def test_logs_json_body_has_any(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""hasAny(body.tags, [...]) matches a log whose array shares ANY value."""
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
resources={"service.name": "app-service"},
|
||||
body=json.dumps({"tags": ["production", "api", "critical"]}),
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
resources={"service.name": "app-service"},
|
||||
body=json.dumps({"tags": ["staging", "api", "test"]}),
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
resources={"service.name": "app-service"},
|
||||
body=json.dumps({"tags": ["production", "web", "important"]}),
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
# log1 has "critical", log2 has "test", log3 has neither -> 2 matches
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((now - timedelta(seconds=10)).timestamp() * 1000),
|
||||
"end": int(now.timestamp() * 1000),
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"filter": {"expression": "hasAny(body.tags, ['critical', 'test'])"},
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
rows = response.json()["data"]["data"]["results"][0]["rows"]
|
||||
assert len(rows) == 2
|
||||
tags_list = [json.loads(row["data"]["body"])["tags"] for row in rows]
|
||||
assert all(("critical" in tags) or ("test" in tags) for tags in tags_list)
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason="hasAny/hasAll over a body-JSON array return 500 in legacy body mode (use_json_body off): ClickHouse 'Argument 0 for hasAll must be an array but has type String'. has() handles body arrays here; hasAny/hasAll do not (they work only in JSON-body mode).",
|
||||
strict=False,
|
||||
)
|
||||
def test_logs_json_body_has_all(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""hasAll(body.tags, [...]) matches only a log whose array has ALL values."""
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
resources={"service.name": "app-service"},
|
||||
body=json.dumps({"tags": ["production", "api", "critical"]}),
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
resources={"service.name": "app-service"},
|
||||
body=json.dumps({"tags": ["production", "web", "important"]}),
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
# only the second log has both "production" AND "web"
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((now - timedelta(seconds=10)).timestamp() * 1000),
|
||||
"end": int(now.timestamp() * 1000),
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"filter": {"expression": "hasAll(body.tags, ['production', 'web'])"},
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
rows = response.json()["data"]["data"]["results"][0]["rows"]
|
||||
assert len(rows) == 1
|
||||
tags = json.loads(rows[0]["data"]["body"])["tags"]
|
||||
assert "production" in tags and "web" in tags
|
||||
|
||||
|
||||
def test_logs_json_body_has_token(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""hasToken(body, 'token') matches logs whose body text contains the token."""
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
resources={"service.name": "app-service"},
|
||||
body=json.dumps({"tags": ["production", "api", "critical"]}),
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
resources={"service.name": "app-service"},
|
||||
body=json.dumps({"tags": ["staging", "api", "test"]}),
|
||||
severity_text="INFO",
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
resources={"service.name": "app-service"},
|
||||
body=json.dumps({"tags": ["production", "web", "important"]}),
|
||||
severity_text="INFO",
|
||||
),
|
||||
]
|
||||
)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
# "production" appears in the first and third log bodies
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((now - timedelta(seconds=10)).timestamp() * 1000),
|
||||
"end": int(now.timestamp() * 1000),
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"filter": {"expression": 'hasToken(body, "production")'},
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
rows = response.json()["data"]["data"]["results"][0]["rows"]
|
||||
assert len(rows) == 2
|
||||
assert all("production" in row["data"]["body"] for row in rows)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression",
|
||||
[
|
||||
pytest.param('has(code.function, "main")', id="has_non_body_key"),
|
||||
pytest.param('hasToken(code.function, "main")', id="hastoken_non_body_key"),
|
||||
pytest.param("hasToken(body, 123)", id="hastoken_non_string_value"),
|
||||
],
|
||||
)
|
||||
def test_logs_json_body_function_errors(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
expression: str,
|
||||
) -> None:
|
||||
"""has/hasToken support only the body JSON field; misuse is rejected (400)."""
|
||||
now = datetime.now(tz=UTC)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((now - timedelta(seconds=10)).timestamp() * 1000),
|
||||
"end": int(now.timestamp() * 1000),
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"filter": {"expression": expression},
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
@@ -8,20 +8,15 @@ import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.fs import get_testdata_file_path
|
||||
from fixtures.metrics import Metrics
|
||||
from fixtures.querier import (
|
||||
assert_minutely_bucket_values,
|
||||
build_builder_query,
|
||||
find_named_result,
|
||||
get_all_warnings,
|
||||
index_series_by_label,
|
||||
make_query_request,
|
||||
)
|
||||
|
||||
FILL_GAPS = "fillGaps"
|
||||
FILL_ZERO = "fillZero"
|
||||
HISTOGRAM_FILE = get_testdata_file_path("histogram_data_1h.jsonl")
|
||||
|
||||
|
||||
def _build_format_options(fill_mode: str) -> dict[str, Any]:
|
||||
@@ -570,371 +565,3 @@ def test_metrics_fill_formula_with_group_by(
|
||||
expected_by_ts=expectations[group],
|
||||
context=f"metrics/{fill_mode}/F1/{group}",
|
||||
)
|
||||
|
||||
|
||||
def test_histogram_p90_returns_warning_outside_data_window(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "test_p90_last_seen_bucket"
|
||||
|
||||
metrics = Metrics.load_from_file(
|
||||
HISTOGRAM_FILE,
|
||||
base_time=now - timedelta(minutes=90),
|
||||
metric_name_override=metric_name,
|
||||
)
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"doesnotreallymatter",
|
||||
"p90",
|
||||
)
|
||||
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_2h, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
start_15m = int((now - timedelta(minutes=15)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_15m, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
data = response.json()
|
||||
warnings = get_all_warnings(data)
|
||||
assert len(warnings) == 1
|
||||
assert warnings[0]["message"].startswith(f"no data found for the metric {metric_name}")
|
||||
|
||||
|
||||
def test_non_existent_metrics_returns_warning(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "whatevergoennnsgoeshere"
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"doesnotreallymatter",
|
||||
"sum",
|
||||
)
|
||||
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_2h, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
data = response.json()
|
||||
warnings = get_all_warnings(data)
|
||||
assert any("whatevergoennnsgoeshere" in w["message"] and "has never been received" in w["message"] for w in warnings), f"expected never-seen metric warning, got: {warnings}"
|
||||
|
||||
|
||||
def test_non_existent_internal_metrics_returns_no_warning(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "signoz_calls_total"
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"doesnotreallymatter",
|
||||
"sum",
|
||||
)
|
||||
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_2h, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
data = response.json()
|
||||
assert get_all_warnings(data) == []
|
||||
|
||||
|
||||
def test_variable_in_filter_returns_no_warning(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
"""
|
||||
A dashboard variable used in a metric filter expression (e.g.
|
||||
`my_tag = $tag`) sits in value position but is lexed as a key token. It
|
||||
must not be mistaken for a missing attribute key and must not produce a
|
||||
"key not found" warning.
|
||||
|
||||
Regression test for https://github.com/SigNoz/engineering-pod/issues/5481
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "test_variable_filter_metric"
|
||||
|
||||
metrics: list[Metrics] = [
|
||||
Metrics(
|
||||
metric_name=metric_name,
|
||||
labels={"my_tag": "service-a"},
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
value=10.0,
|
||||
temporality="Cumulative",
|
||||
),
|
||||
Metrics(
|
||||
metric_name=metric_name,
|
||||
labels={"my_tag": "service-a"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=30.0,
|
||||
temporality="Cumulative",
|
||||
),
|
||||
]
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"increase",
|
||||
"sum",
|
||||
temporality="cumulative",
|
||||
filter_expression="my_tag = $tag",
|
||||
)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms,
|
||||
end_ms,
|
||||
[query],
|
||||
variables={"tag": {"type": "query", "value": "service-a"}},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
# `my_tag` is a real label and `$tag` is a value-position variable, so
|
||||
# neither should be flagged as a missing key on the metric.
|
||||
assert get_all_warnings(data) == [], f"expected no warnings, got: {get_all_warnings(data)}"
|
||||
|
||||
|
||||
# Verify /api/v1/fields/values filters label values by metricNamespace prefix.
|
||||
# Inserts metrics under ns.a and ns.b, then asserts a specific prefix returns
|
||||
# only matching values while a common prefix returns both.
|
||||
def test_metric_namespace_values_filtering(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
metrics: list[Metrics] = [
|
||||
Metrics(
|
||||
metric_name="ns.a.requests_total",
|
||||
labels={"service": "svc-a"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=10.0,
|
||||
),
|
||||
Metrics(
|
||||
metric_name="ns.b.requests_total",
|
||||
labels={"service": "svc-b"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=20.0,
|
||||
),
|
||||
]
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Specific prefix: metricNamespace=ns.a should return only svc-a
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"name": "service",
|
||||
"searchText": "",
|
||||
"metricNamespace": "ns.a",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "svc-a" in values
|
||||
assert "svc-b" not in values
|
||||
|
||||
# Common prefix: metricNamespace=ns should return both
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"name": "service",
|
||||
"searchText": "",
|
||||
"metricNamespace": "ns",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "svc-a" in values
|
||||
assert "svc-b" in values
|
||||
|
||||
|
||||
# Verify /api/v1/fields/values with name=metric_name filters metric names by
|
||||
# metricNamespace prefix. A specific prefix returns only its metric names;
|
||||
# a common prefix returns metric names from all matching namespaces.
|
||||
def test_metric_namespace_metric_name_values_filtering(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
metrics: list[Metrics] = [
|
||||
Metrics(
|
||||
metric_name="ns.a.cpu.utilization",
|
||||
labels={"host": "host-a"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=50.0,
|
||||
),
|
||||
Metrics(
|
||||
metric_name="ns.b.cpu.utilization",
|
||||
labels={"host": "host-b"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=60.0,
|
||||
),
|
||||
]
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Specific prefix: metricNamespace=ns.a should return only ns.a.* metric names
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"name": "metric_name",
|
||||
"searchText": "",
|
||||
"metricNamespace": "ns.a",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "ns.a.cpu.utilization" in values
|
||||
assert "ns.b.cpu.utilization" not in values
|
||||
|
||||
# Common prefix: metricNamespace=ns should return both
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"name": "metric_name",
|
||||
"searchText": "",
|
||||
"metricNamespace": "ns",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "ns.a.cpu.utilization" in values
|
||||
assert "ns.b.cpu.utilization" in values
|
||||
|
||||
|
||||
# Verify /api/v1/fields/keys filters attribute keys by metricNamespace prefix.
|
||||
# Metrics under ns.a and ns.b carry distinct labels; a specific prefix returns
|
||||
# only its keys while a common prefix returns keys from both namespaces.
|
||||
def test_metric_namespace_keys_filtering(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
metrics: list[Metrics] = [
|
||||
Metrics(
|
||||
metric_name="ns.a.cpu.utilization",
|
||||
labels={"a_only_label": "val-a"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=10.0,
|
||||
),
|
||||
Metrics(
|
||||
metric_name="ns.b.cpu.utilization",
|
||||
labels={"b_only_label": "val-b"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=20.0,
|
||||
),
|
||||
]
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Specific prefix: metricNamespace=ns.a should return only a_only_label
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"searchText": "label",
|
||||
"metricNamespace": "ns.a",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert "a_only_label" in keys
|
||||
assert "b_only_label" not in keys
|
||||
|
||||
# Common prefix: metricNamespace=ns should return both keys
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"searchText": "label",
|
||||
"metricNamespace": "ns",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert "a_only_label" in keys
|
||||
assert "b_only_label" in keys
|
||||
175
tests/integration/tests/queriermetrics/02_warnings.py
Normal file
175
tests/integration/tests/queriermetrics/02_warnings.py
Normal file
@@ -0,0 +1,175 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.fs import get_testdata_file_path
|
||||
from fixtures.metrics import Metrics
|
||||
from fixtures.querier import (
|
||||
build_builder_query,
|
||||
get_all_warnings,
|
||||
make_query_request,
|
||||
)
|
||||
|
||||
HISTOGRAM_FILE = get_testdata_file_path("histogram_data_1h.jsonl")
|
||||
|
||||
|
||||
def test_histogram_p90_returns_warning_outside_data_window(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "test_p90_last_seen_bucket"
|
||||
|
||||
metrics = Metrics.load_from_file(
|
||||
HISTOGRAM_FILE,
|
||||
base_time=now - timedelta(minutes=90),
|
||||
metric_name_override=metric_name,
|
||||
)
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"doesnotreallymatter",
|
||||
"p90",
|
||||
)
|
||||
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_2h, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
start_15m = int((now - timedelta(minutes=15)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_15m, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
data = response.json()
|
||||
warnings = get_all_warnings(data)
|
||||
assert len(warnings) == 1
|
||||
assert warnings[0]["message"].startswith(f"no data found for the metric {metric_name}")
|
||||
|
||||
|
||||
def test_non_existent_metrics_returns_warning(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "whatevergoennnsgoeshere"
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"doesnotreallymatter",
|
||||
"sum",
|
||||
)
|
||||
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_2h, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
data = response.json()
|
||||
warnings = get_all_warnings(data)
|
||||
assert any("whatevergoennnsgoeshere" in w["message"] and "has never been received" in w["message"] for w in warnings), f"expected never-seen metric warning, got: {warnings}"
|
||||
|
||||
|
||||
def test_non_existent_internal_metrics_returns_no_warning(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "signoz_calls_total"
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"doesnotreallymatter",
|
||||
"sum",
|
||||
)
|
||||
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_2h, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
data = response.json()
|
||||
assert get_all_warnings(data) == []
|
||||
|
||||
|
||||
def test_variable_in_filter_returns_no_warning(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
"""
|
||||
A dashboard variable used in a metric filter expression (e.g.
|
||||
`my_tag = $tag`) sits in value position but is lexed as a key token. It
|
||||
must not be mistaken for a missing attribute key and must not produce a
|
||||
"key not found" warning.
|
||||
|
||||
Regression test for https://github.com/SigNoz/engineering-pod/issues/5481
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "test_variable_filter_metric"
|
||||
|
||||
metrics: list[Metrics] = [
|
||||
Metrics(
|
||||
metric_name=metric_name,
|
||||
labels={"my_tag": "service-a"},
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
value=10.0,
|
||||
temporality="Cumulative",
|
||||
),
|
||||
Metrics(
|
||||
metric_name=metric_name,
|
||||
labels={"my_tag": "service-a"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=30.0,
|
||||
temporality="Cumulative",
|
||||
),
|
||||
]
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"increase",
|
||||
"sum",
|
||||
temporality="cumulative",
|
||||
filter_expression="my_tag = $tag",
|
||||
)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms,
|
||||
end_ms,
|
||||
[query],
|
||||
variables={"tag": {"type": "query", "value": "service-a"}},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
data = response.json()
|
||||
assert data["status"] == "success"
|
||||
# `my_tag` is a real label and `$tag` is a value-position variable, so
|
||||
# neither should be flagged as a missing key on the metric.
|
||||
assert get_all_warnings(data) == [], f"expected no warnings, got: {get_all_warnings(data)}"
|
||||
217
tests/integration/tests/queriermetrics/03_namespace_metadata.py
Normal file
217
tests/integration/tests/queriermetrics/03_namespace_metadata.py
Normal file
@@ -0,0 +1,217 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.metrics import Metrics
|
||||
|
||||
|
||||
# Verify /api/v1/fields/values filters label values by metricNamespace prefix.
|
||||
# Inserts metrics under ns.a and ns.b, then asserts a specific prefix returns
|
||||
# only matching values while a common prefix returns both.
|
||||
def test_metric_namespace_values_filtering(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
metrics: list[Metrics] = [
|
||||
Metrics(
|
||||
metric_name="ns.a.requests_total",
|
||||
labels={"service": "svc-a"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=10.0,
|
||||
),
|
||||
Metrics(
|
||||
metric_name="ns.b.requests_total",
|
||||
labels={"service": "svc-b"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=20.0,
|
||||
),
|
||||
]
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Specific prefix: metricNamespace=ns.a should return only svc-a
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"name": "service",
|
||||
"searchText": "",
|
||||
"metricNamespace": "ns.a",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "svc-a" in values
|
||||
assert "svc-b" not in values
|
||||
|
||||
# Common prefix: metricNamespace=ns should return both
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"name": "service",
|
||||
"searchText": "",
|
||||
"metricNamespace": "ns",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "svc-a" in values
|
||||
assert "svc-b" in values
|
||||
|
||||
|
||||
# Verify /api/v1/fields/values with name=metric_name filters metric names by
|
||||
# metricNamespace prefix. A specific prefix returns only its metric names;
|
||||
# a common prefix returns metric names from all matching namespaces.
|
||||
def test_metric_namespace_metric_name_values_filtering(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
metrics: list[Metrics] = [
|
||||
Metrics(
|
||||
metric_name="ns.a.cpu.utilization",
|
||||
labels={"host": "host-a"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=50.0,
|
||||
),
|
||||
Metrics(
|
||||
metric_name="ns.b.cpu.utilization",
|
||||
labels={"host": "host-b"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=60.0,
|
||||
),
|
||||
]
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Specific prefix: metricNamespace=ns.a should return only ns.a.* metric names
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"name": "metric_name",
|
||||
"searchText": "",
|
||||
"metricNamespace": "ns.a",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "ns.a.cpu.utilization" in values
|
||||
assert "ns.b.cpu.utilization" not in values
|
||||
|
||||
# Common prefix: metricNamespace=ns should return both
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"name": "metric_name",
|
||||
"searchText": "",
|
||||
"metricNamespace": "ns",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert "ns.a.cpu.utilization" in values
|
||||
assert "ns.b.cpu.utilization" in values
|
||||
|
||||
|
||||
# Verify /api/v1/fields/keys filters attribute keys by metricNamespace prefix.
|
||||
# Metrics under ns.a and ns.b carry distinct labels; a specific prefix returns
|
||||
# only its keys while a common prefix returns keys from both namespaces.
|
||||
def test_metric_namespace_keys_filtering(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
metrics: list[Metrics] = [
|
||||
Metrics(
|
||||
metric_name="ns.a.cpu.utilization",
|
||||
labels={"a_only_label": "val-a"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=10.0,
|
||||
),
|
||||
Metrics(
|
||||
metric_name="ns.b.cpu.utilization",
|
||||
labels={"b_only_label": "val-b"},
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
value=20.0,
|
||||
),
|
||||
]
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Specific prefix: metricNamespace=ns.a should return only a_only_label
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"searchText": "label",
|
||||
"metricNamespace": "ns.a",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert "a_only_label" in keys
|
||||
assert "b_only_label" not in keys
|
||||
|
||||
# Common prefix: metricNamespace=ns should return both keys
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "metrics",
|
||||
"searchText": "label",
|
||||
"metricNamespace": "ns",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert "a_only_label" in keys
|
||||
assert "b_only_label" in keys
|
||||
341
tests/integration/tests/querierscalar/01_logs.py
Normal file
341
tests/integration/tests/querierscalar/01_logs.py
Normal file
@@ -0,0 +1,341 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import querier, types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import make_scalar_query_request
|
||||
|
||||
log_or_trace_service_counts = {
|
||||
"service-a": 5,
|
||||
"service-b": 3,
|
||||
"service-c": 7,
|
||||
"service-d": 1,
|
||||
}
|
||||
|
||||
|
||||
def generate_logs_with_counts(
|
||||
now: datetime,
|
||||
service_counts: dict[str, int],
|
||||
) -> list[Logs]:
|
||||
logs = []
|
||||
for service, count in service_counts.items():
|
||||
for i in range(count):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": service},
|
||||
body=f"{service} log {i}",
|
||||
)
|
||||
)
|
||||
return logs
|
||||
|
||||
|
||||
def build_logs_query(
|
||||
name: str = "A",
|
||||
aggregations: list[str] | None = None,
|
||||
group_by: list[str] | None = None,
|
||||
order_by: list[tuple[str, str]] | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict:
|
||||
if aggregations is None:
|
||||
aggregations = ["count()"]
|
||||
|
||||
aggs = [querier.build_logs_aggregation(expr) for expr in aggregations]
|
||||
gb = [querier.build_group_by_field(f, "string", "resource") for f in group_by] if group_by else None
|
||||
order = [querier.build_order_by(name, direction) for name, direction in order_by] if order_by else None
|
||||
|
||||
return querier.build_scalar_query(
|
||||
name=name,
|
||||
signal="logs",
|
||||
aggregations=aggs,
|
||||
group_by=gb,
|
||||
order=order,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_no_order(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3), ("service-d", 1)],
|
||||
"Logs no order - default desc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("count()", "asc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-b", 3), ("service-a", 5), ("service-c", 7)],
|
||||
"Logs order by agg asc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_agg_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("count()", "desc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3), ("service-d", 1)],
|
||||
"Logs order by agg desc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_grouping_key_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("service.name", "asc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 5), ("service-b", 3), ("service-c", 7), ("service-d", 1)],
|
||||
"Logs order by grouping key asc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_grouping_key_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("service.name", "desc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-c", 7), ("service-b", 3), ("service-a", 5)],
|
||||
"Logs order by grouping key desc",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_multiple_aggs_order_by_first_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_logs_query(
|
||||
group_by=["service.name"],
|
||||
aggregations=["count()", "count_distinct(body)"],
|
||||
order_by=[("count()", "asc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_column_order(data, 0, ["service-d", "service-b", "service-a", "service-c"], "First column")
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_multiple_aggs_order_by_second_agg_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_logs_query(
|
||||
group_by=["service.name"],
|
||||
aggregations=["count()", "count_distinct(body)"],
|
||||
order_by=[("count_distinct(body)", "desc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
# count_distinct(body) should equal count() since each log has unique body
|
||||
querier.assert_scalar_column_order(data, 0, ["service-c", "service-a", "service-b", "service-d"], "First column")
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_agg_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("count()", "asc")], limit=2)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-b", 3)],
|
||||
"Logs order by agg asc with limit 2",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_single_agg_order_by_agg_desc_limit_3(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("count()", "desc")], limit=3)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3)],
|
||||
"Logs order by agg desc with limit 3",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_order_by_grouping_key_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(generate_logs_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_logs_query(group_by=["service.name"], order_by=[("service.name", "asc")], limit=2)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 5), ("service-b", 3)],
|
||||
"Logs order by grouping key asc with limit 2",
|
||||
)
|
||||
316
tests/integration/tests/querierscalar/02_traces.py
Normal file
316
tests/integration/tests/querierscalar/02_traces.py
Normal file
@@ -0,0 +1,316 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import querier, types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import make_scalar_query_request
|
||||
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
|
||||
|
||||
log_or_trace_service_counts = {
|
||||
"service-a": 5,
|
||||
"service-b": 3,
|
||||
"service-c": 7,
|
||||
"service-d": 1,
|
||||
}
|
||||
|
||||
|
||||
def generate_traces_with_counts(
|
||||
now: datetime,
|
||||
service_counts: dict[str, int],
|
||||
) -> list[Traces]:
|
||||
traces = []
|
||||
for service, count in service_counts.items():
|
||||
for i in range(count):
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
span_id = TraceIdGenerator.span_id()
|
||||
traces.append(
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
trace_id=trace_id,
|
||||
span_id=span_id,
|
||||
resources={"service.name": service},
|
||||
name=f"{service} span {i}",
|
||||
)
|
||||
)
|
||||
return traces
|
||||
|
||||
|
||||
def build_traces_query(
|
||||
name: str = "A",
|
||||
aggregations: list[str] | None = None,
|
||||
group_by: list[str] | None = None,
|
||||
order_by: list[tuple[str, str]] | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict:
|
||||
if aggregations is None:
|
||||
aggregations = ["count()"]
|
||||
|
||||
aggs = [querier.build_logs_aggregation(expr) for expr in aggregations]
|
||||
gb = [querier.build_group_by_field(f, "string", "resource") for f in group_by] if group_by else None
|
||||
order = [querier.build_order_by(name, direction) for name, direction in order_by] if order_by else None
|
||||
|
||||
return querier.build_scalar_query(
|
||||
name=name,
|
||||
signal="traces",
|
||||
aggregations=aggs,
|
||||
group_by=gb,
|
||||
order=order,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_no_order(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3), ("service-d", 1)],
|
||||
"Traces no order - default desc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("count()", "asc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-b", 3), ("service-a", 5), ("service-c", 7)],
|
||||
"Traces order by agg asc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_agg_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("count()", "desc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3), ("service-d", 1)],
|
||||
"Traces order by agg desc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_grouping_key_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("service.name", "asc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 5), ("service-b", 3), ("service-c", 7), ("service-d", 1)],
|
||||
"Traces order by grouping key asc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_grouping_key_desc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("service.name", "desc")])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-c", 7), ("service-b", 3), ("service-a", 5)],
|
||||
"Traces order by grouping key desc",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_multiple_aggs_order_by_first_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_traces_query(
|
||||
group_by=["service.name"],
|
||||
aggregations=["count()", "count_distinct(trace_id)"],
|
||||
order_by=[("count()", "asc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_column_order(data, 0, ["service-d", "service-b", "service-a", "service-c"], "First column")
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_agg_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("count()", "asc")], limit=2)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 1), ("service-b", 3)],
|
||||
"Traces order by agg asc with limit 2",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_single_agg_order_by_agg_desc_limit_3(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("count()", "desc")], limit=3)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 7), ("service-a", 5), ("service-b", 3)],
|
||||
"Traces order by agg desc with limit 3",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_scalar_group_by_order_by_grouping_key_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(generate_traces_with_counts(now, log_or_trace_service_counts))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_traces_query(group_by=["service.name"], order_by=[("service.name", "asc")], limit=2)],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 5), ("service-b", 3)],
|
||||
"Traces order by grouping key asc with limit 2",
|
||||
)
|
||||
269
tests/integration/tests/querierscalar/03_metrics.py
Normal file
269
tests/integration/tests/querierscalar/03_metrics.py
Normal file
@@ -0,0 +1,269 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import querier, types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.metrics import Metrics
|
||||
from fixtures.querier import make_scalar_query_request
|
||||
|
||||
metric_values_for_test = {
|
||||
"service-a": 50.0,
|
||||
"service-b": 30.0,
|
||||
"service-c": 70.0,
|
||||
"service-d": 10.0,
|
||||
}
|
||||
|
||||
|
||||
def generate_metrics_with_values(
|
||||
now: datetime,
|
||||
service_values: dict[str, float],
|
||||
) -> list[Metrics]:
|
||||
metrics = []
|
||||
for service, value in service_values.items():
|
||||
metrics.append(
|
||||
Metrics(
|
||||
metric_name="test.metric",
|
||||
labels={"service.name": service},
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=value,
|
||||
)
|
||||
)
|
||||
return metrics
|
||||
|
||||
|
||||
def build_metrics_query(
|
||||
name: str = "A",
|
||||
metric_name: str = "test.metric",
|
||||
time_aggregation: str = "latest",
|
||||
space_aggregation: str = "sum",
|
||||
group_by: list[str] | None = None,
|
||||
order_by: list[tuple[str, str]] | None = None,
|
||||
limit: int | None = None,
|
||||
) -> dict:
|
||||
aggs = [querier.build_metrics_aggregation(metric_name, time_aggregation, space_aggregation, "unspecified")]
|
||||
gb = [querier.build_group_by_field(f, "string", "attribute") for f in group_by] if group_by else None
|
||||
order = [querier.build_order_by(name, direction) for name, direction in order_by] if order_by else None
|
||||
|
||||
return querier.build_scalar_query(
|
||||
name=name,
|
||||
signal="metrics",
|
||||
aggregations=aggs,
|
||||
group_by=gb,
|
||||
order=order,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_no_order(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[build_metrics_query(group_by=["service.name"])],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[
|
||||
("service-c", 70.0),
|
||||
("service-a", 50.0),
|
||||
("service-b", 30.0),
|
||||
("service-d", 10.0),
|
||||
],
|
||||
"Metrics no order - default desc",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_order_by_agg_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("sum(test.metric)", "asc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[
|
||||
("service-d", 10.0),
|
||||
("service-b", 30.0),
|
||||
("service-a", 50.0),
|
||||
("service-c", 70.0),
|
||||
],
|
||||
"Metrics order by agg asc",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_order_by_grouping_key_asc(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("service.name", "asc")],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[
|
||||
("service-a", 50.0),
|
||||
("service-b", 30.0),
|
||||
("service-c", 70.0),
|
||||
("service-d", 10.0),
|
||||
],
|
||||
"Metrics order by grouping key asc",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_order_by_agg_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("sum(test.metric)", "asc")],
|
||||
limit=2,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-d", 10.0), ("service-b", 30.0)],
|
||||
"Metrics order by agg asc with limit 2",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_single_agg_order_by_agg_desc_limit_3(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("sum(test.metric)", "desc")],
|
||||
limit=3,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-c", 70.0), ("service-a", 50.0), ("service-b", 30.0)],
|
||||
"Metrics order by agg desc with limit 3",
|
||||
)
|
||||
|
||||
|
||||
def test_metrics_scalar_group_by_order_by_grouping_key_asc_limit_2(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(generate_metrics_with_values(now, metric_values_for_test))
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
build_metrics_query(
|
||||
group_by=["service.name"],
|
||||
order_by=[("service.name", "asc")],
|
||||
limit=2,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
querier.assert_scalar_result_order(
|
||||
data,
|
||||
[("service-a", 50.0), ("service-b", 30.0)],
|
||||
"Metrics order by grouping key asc with limit 2",
|
||||
)
|
||||
58
tests/integration/tests/querierscalar/04_having.py
Normal file
58
tests/integration/tests/querierscalar/04_having.py
Normal file
@@ -0,0 +1,58 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
assert_scalar_result_order,
|
||||
build_group_by_field,
|
||||
build_logs_aggregation,
|
||||
build_order_by,
|
||||
build_scalar_query,
|
||||
get_scalar_table_data,
|
||||
make_scalar_query_request,
|
||||
)
|
||||
|
||||
# Non-empty HAVING (a post-aggregation filter) — every existing querier test uses
|
||||
# only the empty `{"expression": ""}` HAVING boilerplate.
|
||||
|
||||
|
||||
def test_logs_scalar_group_by_having(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""A scalar group-by with `having count() > 3` returns only the qualifying groups."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
service_counts = {"service-a": 5, "service-b": 3, "service-c": 7, "service-d": 1}
|
||||
logs = []
|
||||
for service, count in service_counts.items():
|
||||
for i in range(count):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": service},
|
||||
body=f"{service} log {i}",
|
||||
)
|
||||
)
|
||||
insert_logs(logs)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
query = build_scalar_query(
|
||||
name="A",
|
||||
signal="logs",
|
||||
aggregations=[build_logs_aggregation("count()")],
|
||||
group_by=[build_group_by_field("service.name", "string", "resource")],
|
||||
order=[build_order_by("count()", "desc")],
|
||||
having_expression="count() > 3",
|
||||
)
|
||||
response = make_scalar_query_request(signoz, token, now, [query])
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
data = get_scalar_table_data(response.json())
|
||||
# only service-c (7) and service-a (5) have count() > 3; service-b (3) and service-d (1) dropped
|
||||
assert_scalar_result_order(data, [("service-c", 7), ("service-a", 5)], "having count() > 3")
|
||||
54
tests/integration/tests/querierscalar/05_reduce_to.py
Normal file
54
tests/integration/tests/querierscalar/05_reduce_to.py
Normal file
@@ -0,0 +1,54 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.metrics import Metrics
|
||||
from fixtures.querier import (
|
||||
assert_scalar_result_order,
|
||||
build_group_by_field,
|
||||
build_metrics_aggregation,
|
||||
build_scalar_query,
|
||||
get_scalar_table_data,
|
||||
make_scalar_query_request,
|
||||
)
|
||||
|
||||
# Metric scalar `reduceTo`: collapse a time series to a single value for a value/table
|
||||
# panel. Metrics scalar group-by is covered by 03_metrics.py, but reduceTo (last/sum/
|
||||
# avg/min/max) is not exercised anywhere.
|
||||
|
||||
|
||||
def test_metrics_scalar_reduce_to_sum(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
"""reduceTo=sum collapses a series' per-step points (10, 20, 30) to 60."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
# same metric_name + labels => one fingerprint/series; three samples in three
|
||||
# distinct 60s step buckets so latest-per-step yields 10, 20, 30.
|
||||
insert_metrics(
|
||||
[
|
||||
Metrics(metric_name="test.reduce.metric", labels={"service.name": "service-a"}, timestamp=now - timedelta(seconds=121), value=10.0, temporality="Unspecified", type_="Gauge", is_monotonic=False),
|
||||
Metrics(metric_name="test.reduce.metric", labels={"service.name": "service-a"}, timestamp=now - timedelta(seconds=61), value=20.0, temporality="Unspecified", type_="Gauge", is_monotonic=False),
|
||||
Metrics(metric_name="test.reduce.metric", labels={"service.name": "service-a"}, timestamp=now - timedelta(seconds=1), value=30.0, temporality="Unspecified", type_="Gauge", is_monotonic=False),
|
||||
]
|
||||
)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
aggregation = build_metrics_aggregation("test.reduce.metric", "latest", "sum", "unspecified")
|
||||
aggregation["reduceTo"] = "sum"
|
||||
query = build_scalar_query(
|
||||
name="A",
|
||||
signal="metrics",
|
||||
aggregations=[aggregation],
|
||||
group_by=[build_group_by_field("service.name", "string", "attribute")],
|
||||
)
|
||||
response = make_scalar_query_request(signoz, token, now, [query])
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
data = get_scalar_table_data(response.json())
|
||||
assert_scalar_result_order(data, [("service-a", 60.0)], "metrics reduceTo=sum")
|
||||
905
tests/integration/tests/queriertraces/01_list.py
Normal file
905
tests/integration/tests/queriertraces/01_list.py
Normal file
@@ -0,0 +1,905 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import (
|
||||
assert_identical_query_response,
|
||||
format_timestamp,
|
||||
generate_traces_with_corrupt_metadata,
|
||||
make_query_request,
|
||||
)
|
||||
from fixtures.traces import (
|
||||
ALL_SELECT_FIELDS,
|
||||
TraceIdGenerator,
|
||||
Traces,
|
||||
TracesEvent,
|
||||
TracesKind,
|
||||
TracesLink,
|
||||
TracesRefType,
|
||||
TracesStatusCode,
|
||||
)
|
||||
|
||||
|
||||
def test_traces_list(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 4 traces with different attributes.
|
||||
http-service: POST /integration -> SELECT, HTTP PATCH
|
||||
topic-service: topic publish
|
||||
|
||||
Tests:
|
||||
1. Query traces for the last 5 minutes and check if the spans are returned in the correct order
|
||||
2. Query root spans for the last 5 minutes and check if the spans are returned in the correct order
|
||||
3. Query values of http.request.method attribute from the autocomplete API
|
||||
4. Query values of http.request.method attribute from the fields API
|
||||
"""
|
||||
http_service_trace_id = TraceIdGenerator.trace_id()
|
||||
http_service_span_id = TraceIdGenerator.span_id()
|
||||
http_service_db_span_id = TraceIdGenerator.span_id()
|
||||
http_service_patch_span_id = TraceIdGenerator.span_id()
|
||||
topic_service_trace_id = TraceIdGenerator.trace_id()
|
||||
topic_service_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
duration=timedelta(seconds=3),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_span_id,
|
||||
parent_span_id="",
|
||||
name="POST /integration",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"net.transport": "IP.TCP",
|
||||
"http.scheme": "http",
|
||||
"http.user_agent": "Integration Test",
|
||||
"http.request.method": "POST",
|
||||
"http.response.status_code": "200",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3.5),
|
||||
duration=timedelta(seconds=0.5),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_db_span_id,
|
||||
parent_span_id=http_service_span_id,
|
||||
name="SELECT",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"db.name": "integration",
|
||||
"db.operation": "SELECT",
|
||||
"db.statement": "SELECT * FROM integration",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_patch_span_id,
|
||||
parent_span_id=http_service_span_id,
|
||||
name="HTTP PATCH",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"http.request.method": "PATCH",
|
||||
"http.status_code": "404",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
duration=timedelta(seconds=4),
|
||||
trace_id=topic_service_trace_id,
|
||||
span_id=topic_service_span_id,
|
||||
parent_span_id="",
|
||||
name="topic publish",
|
||||
kind=TracesKind.SPAN_KIND_PRODUCER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "topic-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-001",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "001",
|
||||
},
|
||||
attributes={
|
||||
"message.type": "SENT",
|
||||
"messaging.operation": "publish",
|
||||
"messaging.message.id": "001",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Query all traces for the past 5 minutes
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
"order": [
|
||||
{"key": {"name": "timestamp"}, "direction": "desc"},
|
||||
],
|
||||
"selectFields": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "span",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "duration_nano",
|
||||
"fieldDataType": "",
|
||||
"fieldContext": "span",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "http_method",
|
||||
"fieldDataType": "",
|
||||
"fieldContext": "span",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "response_status_code",
|
||||
"fieldDataType": "",
|
||||
"fieldContext": "span",
|
||||
"signal": "traces",
|
||||
},
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
# Query results with context appended to key names
|
||||
response_with_inline_context = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
"order": [
|
||||
{"key": {"name": "timestamp"}, "direction": "desc"},
|
||||
],
|
||||
"selectFields": [
|
||||
{
|
||||
"name": "resource.service.name",
|
||||
"fieldDataType": "string",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "span.name:string",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "span.duration_nano",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "span.http_method",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "span.response_status_code",
|
||||
"signal": "traces",
|
||||
},
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert_identical_query_response(response, response_with_inline_context)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
rows = results[0]["rows"]
|
||||
assert len(rows) == 4
|
||||
|
||||
# Care about the order of the rows
|
||||
row_0 = dict(rows[0]["data"])
|
||||
assert row_0.pop("timestamp") is not None
|
||||
assert row_0 == {
|
||||
"duration_nano": 4 * 1e9,
|
||||
"http_method": "",
|
||||
"name": "topic publish",
|
||||
"response_status_code": "",
|
||||
"service.name": "topic-service",
|
||||
"span_id": topic_service_span_id,
|
||||
"trace_id": topic_service_trace_id,
|
||||
}
|
||||
|
||||
row_2 = dict(rows[1]["data"])
|
||||
assert row_2.pop("timestamp") is not None
|
||||
assert row_2 == {
|
||||
"duration_nano": 1 * 1e9,
|
||||
"http_method": "PATCH",
|
||||
"name": "HTTP PATCH",
|
||||
"response_status_code": "404",
|
||||
"service.name": "http-service",
|
||||
"span_id": http_service_patch_span_id,
|
||||
"trace_id": http_service_trace_id,
|
||||
}
|
||||
|
||||
row_3 = dict(rows[2]["data"])
|
||||
assert row_3.pop("timestamp") is not None
|
||||
assert row_3 == {
|
||||
"duration_nano": 0.5 * 1e9,
|
||||
"http_method": "",
|
||||
"name": "SELECT",
|
||||
"response_status_code": "",
|
||||
"service.name": "http-service",
|
||||
"span_id": http_service_db_span_id,
|
||||
"trace_id": http_service_trace_id,
|
||||
}
|
||||
|
||||
row_1 = dict(rows[3]["data"])
|
||||
assert row_1.pop("timestamp") is not None
|
||||
assert row_1 == {
|
||||
"duration_nano": 3 * 1e9,
|
||||
"http_method": "POST",
|
||||
"name": "POST /integration",
|
||||
"response_status_code": "200",
|
||||
"service.name": "http-service",
|
||||
"span_id": http_service_span_id,
|
||||
"trace_id": http_service_trace_id,
|
||||
}
|
||||
|
||||
# Query root spans for the last 5 minutes and check if the spans are returned in the correct order
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
"end": int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"limit": 10,
|
||||
"offset": 0,
|
||||
"filter": {"expression": "isRoot = 'true'"},
|
||||
"order": [
|
||||
{"key": {"name": "timestamp"}, "direction": "desc"},
|
||||
],
|
||||
"selectFields": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
rows = results[0]["rows"]
|
||||
assert len(rows) == 2
|
||||
|
||||
assert rows[0]["data"]["service.name"] == "topic-service"
|
||||
assert rows[1]["data"]["service.name"] == "http-service"
|
||||
|
||||
# Query values of http.request.method attribute from the autocomplete API
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v3/autocomplete/attribute_values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"aggregateOperator": "noop",
|
||||
"dataSource": "traces",
|
||||
"aggregateAttribute": "",
|
||||
"attributeKey": "http.request.method",
|
||||
"searchText": "",
|
||||
"filterAttributeKeyDataType": "string",
|
||||
"tagType": "tag",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["stringAttributeValues"]
|
||||
assert len(values) == 2
|
||||
|
||||
assert set(values) == set(["POST", "PATCH"])
|
||||
|
||||
# Query values of http.request.method attribute from the fields API
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "traces",
|
||||
"name": "http.request.method",
|
||||
"searchText": "",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert len(values) == 2
|
||||
|
||||
assert set(values) == set(["POST", "PATCH"])
|
||||
|
||||
# Query keys from the fields API with context specified in the key
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "traces",
|
||||
"searchText": "resource.servic",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert "service.name" in keys
|
||||
assert any(k["fieldContext"] == "resource" for k in keys["service.name"])
|
||||
|
||||
# Query values of service.name resource attribute using context-prefixed key
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
params={
|
||||
"signal": "traces",
|
||||
"name": "resource.service.name",
|
||||
"searchText": "",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
values = response.json()["data"]["values"]["stringValues"]
|
||||
assert set(values) == set(["topic-service", "http-service"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload,status_code,results",
|
||||
[
|
||||
# Case 1: order by timestamp; empty selectFields returns the full
|
||||
# response shape (all intrinsic + calculated columns plus the merged
|
||||
# `attributes` and `resource` maps). x[3] (topic-service) is latest.
|
||||
pytest.param(
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
|
||||
"limit": 1,
|
||||
},
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
lambda x: [
|
||||
{
|
||||
**x[3].attribute_string,
|
||||
**x[3].attributes_number,
|
||||
**x[3].attributes_bool,
|
||||
}, # attributes
|
||||
x[3].db_name,
|
||||
x[3].db_operation,
|
||||
int(x[3].duration_nano),
|
||||
x[3].events,
|
||||
x[3].external_http_method,
|
||||
x[3].external_http_url,
|
||||
int(x[3].flags),
|
||||
x[3].has_error,
|
||||
x[3].http_host,
|
||||
x[3].http_method,
|
||||
x[3].http_url,
|
||||
x[3].is_remote,
|
||||
int(x[3].kind),
|
||||
x[3].kind_string,
|
||||
x[3].links,
|
||||
x[3].name,
|
||||
x[3].parent_span_id,
|
||||
x[3].resources_string,
|
||||
x[3].response_status_code,
|
||||
x[3].span_id,
|
||||
int(x[3].status_code),
|
||||
x[3].status_code_string,
|
||||
x[3].status_message,
|
||||
format_timestamp(x[3].timestamp),
|
||||
x[3].trace_id,
|
||||
x[3].trace_state,
|
||||
], # type: Callable[[List[Traces]], List[Any]]
|
||||
),
|
||||
# Case 2: order by attribute.timestamp. The key resolves to the
|
||||
# intrinsic span.timestamp column, so the latest span (x[3]) is
|
||||
# returned with the same full response shape as Case 1.
|
||||
pytest.param(
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"order": [{"key": {"name": "attribute.timestamp"}, "direction": "desc"}],
|
||||
"limit": 1,
|
||||
},
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
lambda x: [
|
||||
{
|
||||
**x[3].attribute_string,
|
||||
**x[3].attributes_number,
|
||||
**x[3].attributes_bool,
|
||||
}, # attributes
|
||||
x[3].db_name,
|
||||
x[3].db_operation,
|
||||
int(x[3].duration_nano),
|
||||
x[3].events,
|
||||
x[3].external_http_method,
|
||||
x[3].external_http_url,
|
||||
int(x[3].flags),
|
||||
x[3].has_error,
|
||||
x[3].http_host,
|
||||
x[3].http_method,
|
||||
x[3].http_url,
|
||||
x[3].is_remote,
|
||||
int(x[3].kind),
|
||||
x[3].kind_string,
|
||||
x[3].links,
|
||||
x[3].name,
|
||||
x[3].parent_span_id,
|
||||
x[3].resources_string,
|
||||
x[3].response_status_code,
|
||||
x[3].span_id,
|
||||
int(x[3].status_code),
|
||||
x[3].status_code_string,
|
||||
x[3].status_message,
|
||||
format_timestamp(x[3].timestamp),
|
||||
x[3].trace_id,
|
||||
x[3].trace_state,
|
||||
], # type: Callable[[List[Traces]], List[Any]]
|
||||
),
|
||||
# Case 3: select timestamp with empty order by
|
||||
pytest.param(
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"selectFields": [{"name": "timestamp"}],
|
||||
"limit": 1,
|
||||
},
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
lambda x: [
|
||||
x[2].span_id,
|
||||
format_timestamp(x[2].timestamp),
|
||||
x[2].trace_id,
|
||||
], # type: Callable[[List[Traces]], List[Any]]
|
||||
),
|
||||
# Case 4: select attribute.timestamp with empty order by
|
||||
# This returns the one span which has attribute.timestamp
|
||||
pytest.param(
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"filter": {"expression": "attribute.timestamp exists"},
|
||||
"disabled": False,
|
||||
"selectFields": [{"name": "attribute.timestamp"}],
|
||||
"limit": 1,
|
||||
},
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
lambda x: [
|
||||
x[0].span_id,
|
||||
format_timestamp(x[0].timestamp),
|
||||
x[0].trace_id,
|
||||
], # type: Callable[[List[Traces]], List[Any]]
|
||||
),
|
||||
# Case 5: select timestamp with timestamp order by
|
||||
pytest.param(
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"selectFields": [{"name": "timestamp"}],
|
||||
"limit": 1,
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "asc"}],
|
||||
},
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
lambda x: [
|
||||
x[0].span_id,
|
||||
format_timestamp(x[0].timestamp),
|
||||
x[0].trace_id,
|
||||
], # type: Callable[[List[Traces]], List[Any]]
|
||||
),
|
||||
# Case 6: select duration_nano with duration order by
|
||||
pytest.param(
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"selectFields": [{"name": "duration_nano"}],
|
||||
"limit": 1,
|
||||
"order": [{"key": {"name": "duration_nano"}, "direction": "desc"}],
|
||||
},
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
lambda x: [
|
||||
x[1].duration_nano,
|
||||
x[1].span_id,
|
||||
format_timestamp(x[1].timestamp),
|
||||
x[1].trace_id,
|
||||
], # type: Callable[[List[Traces]], List[Any]]
|
||||
),
|
||||
# Case 7: select attribute.duration_nano with attribute.duration_nano order by
|
||||
pytest.param(
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"selectFields": [{"name": "attribute.duration_nano"}],
|
||||
"filter": {"expression": "attribute.duration_nano exists"},
|
||||
"limit": 1,
|
||||
"order": [
|
||||
{
|
||||
"key": {"name": "attribute.duration_nano"},
|
||||
"direction": "desc",
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
lambda x: [
|
||||
"corrupt_data",
|
||||
x[3].span_id,
|
||||
format_timestamp(x[3].timestamp),
|
||||
x[3].trace_id,
|
||||
], # type: Callable[[List[Traces]], List[Any]]
|
||||
),
|
||||
# Case 8: select attribute.duration_nano with duration order by
|
||||
pytest.param(
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"selectFields": [{"name": "attribute.duration_nano"}],
|
||||
"limit": 1,
|
||||
"order": [{"key": {"name": "duration_nano"}, "direction": "desc"}],
|
||||
},
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
lambda x: [
|
||||
x[1].duration_nano,
|
||||
x[1].span_id,
|
||||
format_timestamp(x[1].timestamp),
|
||||
x[1].trace_id,
|
||||
], # type: Callable[[List[Traces]], List[Any]]
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_traces_list_with_corrupt_data(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
payload: dict[str, Any],
|
||||
status_code: HTTPStatus,
|
||||
results: Callable[[list[Traces]], list[Any]],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 4 traces with corrupt attributes.
|
||||
Tests:
|
||||
"""
|
||||
|
||||
traces = generate_traces_with_corrupt_metadata()
|
||||
insert_traces(traces)
|
||||
# 4 Traces with corrupt metadata inserted
|
||||
# traces[i] occured before traces[j] where i < j
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[payload],
|
||||
)
|
||||
|
||||
assert response.status_code == status_code
|
||||
|
||||
if response.status_code == HTTPStatus.OK:
|
||||
if not results(traces):
|
||||
# No results expected
|
||||
assert response.json()["data"]["data"]["results"][0]["rows"] is None
|
||||
else:
|
||||
data = response.json()["data"]["data"]["results"][0]["rows"][0]["data"]
|
||||
# Cannot compare values as they are randomly generated
|
||||
for key, value in zip(list(data.keys()), results(traces)):
|
||||
assert data[key] == value
|
||||
|
||||
|
||||
def _verify_events_links_full(rows: list[dict], traces: list[Traces]) -> None:
|
||||
"""Empty-selectFields case: events/links arrive parsed into structured objects.
|
||||
Every row's events/links should match the fixture's stored parsed shape
|
||||
(the fixture's `.events`/`.links` mirror the API response shape directly).
|
||||
"""
|
||||
for row, trace in zip(rows, traces, strict=True):
|
||||
assert row["data"]["events"] == trace.events
|
||||
assert row["data"]["links"] == trace.links
|
||||
# Jaeger-era `refType` is dropped at the consume layer.
|
||||
for link in row["data"]["links"]:
|
||||
assert "refType" not in link
|
||||
|
||||
|
||||
def _verify_events_links_skip(rows: list[dict], traces: list[Traces]) -> None:
|
||||
"""Projected-selectFields case: nothing to verify beyond the key set."""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"select_fields,status_code,expected_keys,verify_values",
|
||||
[
|
||||
pytest.param(
|
||||
[],
|
||||
HTTPStatus.OK,
|
||||
ALL_SELECT_FIELDS,
|
||||
_verify_events_links_full,
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{"name": "service.name"},
|
||||
],
|
||||
HTTPStatus.OK,
|
||||
["timestamp", "trace_id", "span_id", "service.name"],
|
||||
_verify_events_links_skip,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_traces_list_with_select_fields(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
select_fields: list[dict],
|
||||
status_code: HTTPStatus,
|
||||
expected_keys: list[str],
|
||||
verify_values: Callable[[list[dict], list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert a root span with no events/links and a child span carrying two
|
||||
events and one user-supplied link.
|
||||
|
||||
Tests:
|
||||
1. Empty select fields should return all the fields, and the `events` /
|
||||
`links` columns should arrive parsed into structured objects (events
|
||||
carry `attributes`, links carry only `traceId`/`spanId` — refType is
|
||||
dropped at the consume layer).
|
||||
2. Non-empty select field should return the select field along with
|
||||
timestamp, trace_id and span_id.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
parent_trace_id = TraceIdGenerator.trace_id()
|
||||
parent_span_id = TraceIdGenerator.span_id()
|
||||
child_span_id = TraceIdGenerator.span_id()
|
||||
linked_trace_id = TraceIdGenerator.trace_id()
|
||||
linked_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
event_one = TracesEvent(
|
||||
name="request_received",
|
||||
timestamp=now - timedelta(seconds=3, microseconds=500_000),
|
||||
attribute_map={"http.method": "GET", "http.route": "/api/chat"},
|
||||
)
|
||||
event_two = TracesEvent(
|
||||
name="cache_lookup",
|
||||
timestamp=now - timedelta(seconds=3, microseconds=400_000),
|
||||
attribute_map={"cache.hit": "true", "cache.key": "user:123:prompt"},
|
||||
)
|
||||
user_link = TracesLink(
|
||||
trace_id=linked_trace_id,
|
||||
span_id=linked_span_id,
|
||||
ref_type=TracesRefType.REF_TYPE_FOLLOWS_FROM,
|
||||
)
|
||||
|
||||
traces = [
|
||||
# Root span: no events, no links. Verifies the empty-case parsed shape.
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
duration=timedelta(seconds=3),
|
||||
trace_id=parent_trace_id,
|
||||
span_id=parent_span_id,
|
||||
parent_span_id="",
|
||||
name="root span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
resources={"service.name": "events-links-service"},
|
||||
attributes={"http.request.method": "GET"},
|
||||
),
|
||||
# Child span: two events + one user-supplied link. The fixture
|
||||
# auto-inserts a CHILD_OF link for the parent, so the parsed response
|
||||
# contains two links total — the auto-inserted one first.
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=parent_trace_id,
|
||||
span_id=child_span_id,
|
||||
parent_span_id=parent_span_id,
|
||||
name="child span",
|
||||
kind=TracesKind.SPAN_KIND_INTERNAL,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
resources={"service.name": "events-links-service"},
|
||||
attributes={"http.request.method": "GET"},
|
||||
events=[event_one, event_two],
|
||||
links=[user_link],
|
||||
),
|
||||
]
|
||||
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
payload = {
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"filter": {"expression": "resource.service.name = 'events-links-service'"},
|
||||
"selectFields": select_fields,
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "asc"}],
|
||||
"limit": 10,
|
||||
},
|
||||
}
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[payload],
|
||||
)
|
||||
assert response.status_code == status_code
|
||||
|
||||
if response.status_code != HTTPStatus.OK:
|
||||
return
|
||||
|
||||
rows = response.json()["data"]["data"]["results"][0]["rows"]
|
||||
assert len(rows) == 2
|
||||
for row in rows:
|
||||
assert set(row["data"].keys()) == set(expected_keys)
|
||||
|
||||
verify_values(rows, traces)
|
||||
408
tests/integration/tests/queriertraces/02_aggregation.py
Normal file
408
tests/integration/tests/queriertraces/02_aggregation.py
Normal file
@@ -0,0 +1,408 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import (
|
||||
make_query_request,
|
||||
)
|
||||
from fixtures.traces import (
|
||||
TraceIdGenerator,
|
||||
Traces,
|
||||
TracesKind,
|
||||
TracesStatusCode,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"order_by,aggregation_alias,expected_status",
|
||||
[
|
||||
# Case 1a: count by count()
|
||||
pytest.param({"name": "count()"}, "count_", HTTPStatus.OK),
|
||||
# Case 1b: count by count() with alias span.count_
|
||||
pytest.param({"name": "count()"}, "span.count_", HTTPStatus.OK),
|
||||
# Case 2a: count by count() with context specified in the key
|
||||
pytest.param({"name": "count()", "fieldContext": "span"}, "count_", HTTPStatus.OK),
|
||||
# Case 2b: count by count() with context specified in the key with alias span.count_
|
||||
pytest.param({"name": "count()", "fieldContext": "span"}, "span.count_", HTTPStatus.OK),
|
||||
# Case 3a: count by span.count() and context specified in the key [BAD REQUEST]
|
||||
pytest.param(
|
||||
{"name": "span.count()", "fieldContext": "span"},
|
||||
"count_",
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
),
|
||||
# Case 3b: count by span.count() and context specified in the key with alias span.count_ [BAD REQUEST]
|
||||
pytest.param(
|
||||
{"name": "span.count()", "fieldContext": "span"},
|
||||
"span.count_",
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
),
|
||||
# Case 4a: count by span.count() and context specified in the key
|
||||
pytest.param({"name": "span.count()", "fieldContext": ""}, "count_", HTTPStatus.OK),
|
||||
# Case 4b: count by span.count() and context specified in the key with alias span.count_
|
||||
pytest.param({"name": "span.count()", "fieldContext": ""}, "span.count_", HTTPStatus.OK),
|
||||
# Case 5a: count by count_
|
||||
pytest.param({"name": "count_"}, "count_", HTTPStatus.OK),
|
||||
# Case 5b: count by count_ with alias span.count_
|
||||
pytest.param({"name": "count_"}, "count_", HTTPStatus.OK),
|
||||
# Case 6a: count by span.count_
|
||||
pytest.param({"name": "span.count_"}, "count_", HTTPStatus.OK),
|
||||
# Case 6b: count by span.count_ with alias span.count_
|
||||
pytest.param({"name": "span.count_"}, "span.count_", HTTPStatus.OK),
|
||||
# Case 7a: count by span.count_ and context specified in the key [BAD REQUEST]
|
||||
pytest.param(
|
||||
{"name": "span.count_", "fieldContext": "span"},
|
||||
"count_",
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
),
|
||||
# Case 7b: count by span.count_ and context specified in the key with alias span.count_
|
||||
pytest.param(
|
||||
{"name": "span.count_", "fieldContext": "span"},
|
||||
"span.count_",
|
||||
HTTPStatus.OK,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_traces_aggergate_order_by_count(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
order_by: dict[str, str],
|
||||
aggregation_alias: str,
|
||||
expected_status: HTTPStatus,
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 4 traces with different attributes.
|
||||
http-service: POST /integration -> SELECT, HTTP PATCH
|
||||
topic-service: topic publish
|
||||
|
||||
Tests:
|
||||
1. Query traces count for spans grouped by service.name and host.name
|
||||
"""
|
||||
http_service_trace_id = TraceIdGenerator.trace_id()
|
||||
http_service_span_id = TraceIdGenerator.span_id()
|
||||
http_service_db_span_id = TraceIdGenerator.span_id()
|
||||
http_service_patch_span_id = TraceIdGenerator.span_id()
|
||||
topic_service_trace_id = TraceIdGenerator.trace_id()
|
||||
topic_service_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
duration=timedelta(seconds=3),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_span_id,
|
||||
parent_span_id="",
|
||||
name="POST /integration",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"net.transport": "IP.TCP",
|
||||
"http.scheme": "http",
|
||||
"http.user_agent": "Integration Test",
|
||||
"http.request.method": "POST",
|
||||
"http.response.status_code": "200",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3.5),
|
||||
duration=timedelta(seconds=0.5),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_db_span_id,
|
||||
parent_span_id=http_service_span_id,
|
||||
name="SELECT",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"db.name": "integration",
|
||||
"db.operation": "SELECT",
|
||||
"db.statement": "SELECT * FROM integration",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_patch_span_id,
|
||||
parent_span_id=http_service_span_id,
|
||||
name="HTTP PATCH",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"http.request.method": "PATCH",
|
||||
"http.status_code": "404",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
duration=timedelta(seconds=4),
|
||||
trace_id=topic_service_trace_id,
|
||||
span_id=topic_service_span_id,
|
||||
parent_span_id="",
|
||||
name="topic publish",
|
||||
kind=TracesKind.SPAN_KIND_PRODUCER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "topic-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-001",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "001",
|
||||
},
|
||||
attributes={
|
||||
"message.type": "SENT",
|
||||
"messaging.operation": "publish",
|
||||
"messaging.message.id": "001",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
query = {
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"order": [{"key": {"name": "count()"}, "direction": "desc"}],
|
||||
"aggregations": [{"expression": "count()", "alias": "count_"}],
|
||||
},
|
||||
}
|
||||
|
||||
# Query traces count for spans
|
||||
|
||||
query["spec"]["order"][0]["key"] = order_by
|
||||
query["spec"]["aggregations"][0]["alias"] = aggregation_alias
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
request_type="time_series",
|
||||
queries=[query],
|
||||
)
|
||||
|
||||
assert response.status_code == expected_status
|
||||
if expected_status != HTTPStatus.OK:
|
||||
return
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 1
|
||||
assert series[0]["values"][0]["value"] == 4
|
||||
|
||||
|
||||
def test_traces_aggregate_with_mixed_field_selectors(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 4 traces with different attributes.
|
||||
http-service: POST /integration -> SELECT, HTTP PATCH
|
||||
topic-service: topic publish
|
||||
|
||||
Tests:
|
||||
1. Query traces count for spans grouped by service.name
|
||||
"""
|
||||
http_service_trace_id = TraceIdGenerator.trace_id()
|
||||
http_service_span_id = TraceIdGenerator.span_id()
|
||||
http_service_db_span_id = TraceIdGenerator.span_id()
|
||||
http_service_patch_span_id = TraceIdGenerator.span_id()
|
||||
topic_service_trace_id = TraceIdGenerator.trace_id()
|
||||
topic_service_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
duration=timedelta(seconds=3),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_span_id,
|
||||
parent_span_id="",
|
||||
name="POST /integration",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"net.transport": "IP.TCP",
|
||||
"http.scheme": "http",
|
||||
"http.user_agent": "Integration Test",
|
||||
"http.request.method": "POST",
|
||||
"http.response.status_code": "200",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3.5),
|
||||
duration=timedelta(seconds=0.5),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_db_span_id,
|
||||
parent_span_id=http_service_span_id,
|
||||
name="SELECT",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"db.name": "integration",
|
||||
"db.operation": "SELECT",
|
||||
"db.statement": "SELECT * FROM integration",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_patch_span_id,
|
||||
parent_span_id=http_service_span_id,
|
||||
name="HTTP PATCH",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "000",
|
||||
},
|
||||
attributes={
|
||||
"http.request.method": "PATCH",
|
||||
"http.status_code": "404",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
duration=timedelta(seconds=4),
|
||||
trace_id=topic_service_trace_id,
|
||||
span_id=topic_service_span_id,
|
||||
parent_span_id="",
|
||||
name="topic publish",
|
||||
kind=TracesKind.SPAN_KIND_PRODUCER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "topic-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-001",
|
||||
"cloud.provider": "integration",
|
||||
"cloud.account.id": "001",
|
||||
},
|
||||
attributes={
|
||||
"message.type": "SENT",
|
||||
"messaging.operation": "publish",
|
||||
"messaging.message.id": "001",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
query = {
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldContext": "resource",
|
||||
"fieldDataType": "string",
|
||||
}
|
||||
],
|
||||
"aggregations": [
|
||||
{"expression": "p99(duration_nano)", "alias": "p99"},
|
||||
{"expression": "avg(duration_nano)", "alias": "avgDuration"},
|
||||
{"expression": "count()", "alias": "numCalls"},
|
||||
{"expression": "countIf(status_code = 2)", "alias": "numErrors"},
|
||||
{
|
||||
"expression": "countIf(response_status_code >= 400 AND response_status_code < 500)",
|
||||
"alias": "num4XX",
|
||||
},
|
||||
],
|
||||
"order": [{"key": {"name": "count()"}, "direction": "desc"}],
|
||||
},
|
||||
}
|
||||
|
||||
# Query traces count for spans
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
|
||||
request_type="time_series",
|
||||
queries=[query],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
aggregations = results[0]["aggregations"]
|
||||
|
||||
assert aggregations[0]["series"][0]["values"][0]["value"] >= 2.5 * 1e9 # p99 for http-service
|
||||
935
tests/integration/tests/queriertraces/03_fill.py
Normal file
935
tests/integration/tests/queriertraces/03_fill.py
Normal file
@@ -0,0 +1,935 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import (
|
||||
assert_minutely_bucket_values,
|
||||
find_named_result,
|
||||
index_series_by_label,
|
||||
)
|
||||
from fixtures.traces import (
|
||||
TraceIdGenerator,
|
||||
Traces,
|
||||
TracesKind,
|
||||
TracesStatusCode,
|
||||
)
|
||||
|
||||
|
||||
def test_traces_fill_gaps(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillGaps for traces without groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
|
||||
traces: list[Traces] = [
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=trace_id,
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="test-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "test-service"},
|
||||
attributes={"http.method": "GET"},
|
||||
),
|
||||
]
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": True},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) >= 1
|
||||
|
||||
values = series[0]["values"]
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
assert_minutely_bucket_values(
|
||||
values,
|
||||
now,
|
||||
expected_by_ts={ts_min_3: 1},
|
||||
context="traces/fillGaps",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_fill_gaps_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillGaps for traces with groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
traces: list[Traces] = [
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="span-a",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "service-a"},
|
||||
attributes={"http.method": "GET"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="span-b",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "service-b"},
|
||||
attributes={"http.method": "POST"},
|
||||
),
|
||||
]
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": True},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2, "Expected 2 series for 2 service groups"
|
||||
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
series_by_service = index_series_by_label(series, "service.name")
|
||||
assert set(series_by_service.keys()) == {"service-a", "service-b"}
|
||||
|
||||
expectations: dict[str, dict[int, float]] = {
|
||||
"service-a": {ts_min_3: 1},
|
||||
"service-b": {ts_min_2: 1},
|
||||
}
|
||||
|
||||
for service_name, s in series_by_service.items():
|
||||
assert_minutely_bucket_values(
|
||||
s["values"],
|
||||
now,
|
||||
expected_by_ts=expectations[service_name],
|
||||
context=f"traces/fillGaps/{service_name}",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_fill_gaps_formula(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillGaps for traces with formula.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
traces: list[Traces] = [
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="test-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "test"},
|
||||
attributes={"http.method": "GET"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="another-test-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "another-test"},
|
||||
attributes={"http.method": "POST"},
|
||||
),
|
||||
]
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"filter": {"expression": "service.name = 'test'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "B",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"filter": {"expression": "service.name = 'another-test'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_formula",
|
||||
"spec": {
|
||||
"name": "F1",
|
||||
"expression": "A + B",
|
||||
"disabled": False,
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": True},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
f1 = find_named_result(results, "F1")
|
||||
assert f1 is not None, "Expected formula result named F1"
|
||||
|
||||
aggregations = f1.get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) >= 1
|
||||
|
||||
assert_minutely_bucket_values(
|
||||
series[0]["values"],
|
||||
now,
|
||||
expected_by_ts={ts_min_3: 1, ts_min_2: 1},
|
||||
context="traces/fillGaps/F1",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_fill_gaps_formula_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillGaps for traces with formula and groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
traces: list[Traces] = [
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="span-group1",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "group1"},
|
||||
attributes={"http.method": "GET"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="span-group2",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "group2"},
|
||||
attributes={"http.method": "POST"},
|
||||
),
|
||||
]
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "B",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_formula",
|
||||
"spec": {
|
||||
"name": "F1",
|
||||
"expression": "A + B",
|
||||
"disabled": False,
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": True},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
f1 = find_named_result(results, "F1")
|
||||
assert f1 is not None, "Expected formula result named F1"
|
||||
|
||||
aggregations = f1.get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2
|
||||
|
||||
series_by_service = index_series_by_label(series, "service.name")
|
||||
assert set(series_by_service.keys()) == {"group1", "group2"}
|
||||
|
||||
expectations: dict[str, dict[int, float]] = {
|
||||
"group1": {ts_min_3: 2},
|
||||
"group2": {ts_min_2: 2},
|
||||
}
|
||||
|
||||
for service_name, s in series_by_service.items():
|
||||
assert_minutely_bucket_values(
|
||||
s["values"],
|
||||
now,
|
||||
expected_by_ts=expectations[service_name],
|
||||
context=f"traces/fillGaps/F1/{service_name}",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_fill_zero(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillZero function for traces without groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
traces: list[Traces] = [
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="test-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "test"},
|
||||
attributes={"http.method": "GET"},
|
||||
),
|
||||
]
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
"functions": [{"name": "fillZero"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0].get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) >= 1
|
||||
values = series[0]["values"]
|
||||
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
assert_minutely_bucket_values(
|
||||
values,
|
||||
now,
|
||||
expected_by_ts={ts_min_3: 1},
|
||||
context="traces/fillZero",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_fill_zero_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillZero function for traces with groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
traces: list[Traces] = [
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="span-a",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "service-a"},
|
||||
attributes={"http.method": "GET"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="span-b",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "service-b"},
|
||||
attributes={"http.method": "POST"},
|
||||
),
|
||||
]
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
"functions": [{"name": "fillZero"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2, "Expected 2 series for 2 service groups"
|
||||
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
series_by_service = index_series_by_label(series, "service.name")
|
||||
assert set(series_by_service.keys()) == {"service-a", "service-b"}
|
||||
|
||||
expectations: dict[str, dict[int, float]] = {
|
||||
"service-a": {ts_min_3: 1},
|
||||
"service-b": {ts_min_2: 1},
|
||||
}
|
||||
|
||||
for service_name, s in series_by_service.items():
|
||||
assert_minutely_bucket_values(
|
||||
s["values"],
|
||||
now,
|
||||
expected_by_ts=expectations[service_name],
|
||||
context=f"traces/fillZero/{service_name}",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_fill_zero_formula(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillZero function for traces with formula.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
traces: list[Traces] = [
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="test-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "test"},
|
||||
attributes={"http.method": "GET"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="another-test-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "another-test"},
|
||||
attributes={"http.method": "POST"},
|
||||
),
|
||||
]
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"filter": {"expression": "service.name = 'test'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "B",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"filter": {"expression": "service.name = 'another-test'"},
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_formula",
|
||||
"spec": {
|
||||
"name": "F1",
|
||||
"expression": "A + B",
|
||||
"disabled": False,
|
||||
"functions": [{"name": "fillZero"}],
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
f1 = find_named_result(results, "F1")
|
||||
assert f1 is not None, "Expected formula result named F1"
|
||||
aggregations = f1.get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) >= 1
|
||||
|
||||
assert_minutely_bucket_values(
|
||||
series[0]["values"],
|
||||
now,
|
||||
expected_by_ts={ts_min_3: 1, ts_min_2: 1},
|
||||
context="traces/fillZero/F1",
|
||||
)
|
||||
|
||||
|
||||
def test_traces_fill_zero_formula_with_group_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Test fillZero function for traces with formula and groupBy.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
traces: list[Traces] = [
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=3),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="span-group1",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "group1"},
|
||||
attributes={"http.method": "GET"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(minutes=2),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name="span-group2",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={"service.name": "group2"},
|
||||
attributes={"http.method": "POST"},
|
||||
),
|
||||
]
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v5/query_range"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"schemaVersion": "v1",
|
||||
"start": start_ms,
|
||||
"end": end_ms,
|
||||
"requestType": "time_series",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "B",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": True,
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_formula",
|
||||
"spec": {
|
||||
"name": "F1",
|
||||
"expression": "A + B",
|
||||
"disabled": False,
|
||||
"functions": [{"name": "fillZero"}],
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
"formatOptions": {"formatTableResultForUI": False, "fillGaps": False},
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000)
|
||||
|
||||
f1 = find_named_result(results, "F1")
|
||||
assert f1 is not None, "Expected formula result named F1"
|
||||
aggregations = f1.get("aggregations") or []
|
||||
assert len(aggregations) == 1
|
||||
series = aggregations[0]["series"]
|
||||
assert len(series) == 2
|
||||
|
||||
series_by_service = index_series_by_label(series, "service.name")
|
||||
assert set(series_by_service.keys()) == {"group1", "group2"}
|
||||
|
||||
expectations: dict[str, dict[int, float]] = {
|
||||
"group1": {ts_min_3: 2},
|
||||
"group2": {ts_min_2: 2},
|
||||
}
|
||||
|
||||
for service_name, s in series_by_service.items():
|
||||
assert_minutely_bucket_values(
|
||||
s["values"],
|
||||
now,
|
||||
expected_by_ts=expectations[service_name],
|
||||
context=f"traces/fillZero/F1/{service_name}",
|
||||
)
|
||||
267
tests/integration/tests/queriertraces/04_trace_id_filter.py
Normal file
267
tests/integration/tests/queriertraces/04_trace_id_filter.py
Normal file
@@ -0,0 +1,267 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import (
|
||||
make_query_request,
|
||||
)
|
||||
from fixtures.traces import (
|
||||
TraceIdGenerator,
|
||||
Traces,
|
||||
TracesKind,
|
||||
TracesStatusCode,
|
||||
)
|
||||
|
||||
|
||||
def test_traces_list_filter_by_trace_id(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Tests that filtering by trace_id:
|
||||
1. Returns the matching span (narrow window, single bucket).
|
||||
2. Does not return duplicate spans when the query window spans multiple
|
||||
exponential buckets (>1 h)
|
||||
3. Returns no results when the query window does not contain the trace.
|
||||
"""
|
||||
target_trace_id = TraceIdGenerator.trace_id()
|
||||
other_trace_id = TraceIdGenerator.trace_id()
|
||||
span_id_root = TraceIdGenerator.span_id()
|
||||
other_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
common_resources = {
|
||||
"deployment.environment": "production",
|
||||
"service.name": "trace-filter-service",
|
||||
"cloud.provider": "integration",
|
||||
}
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
duration=timedelta(seconds=5),
|
||||
trace_id=target_trace_id,
|
||||
span_id=span_id_root,
|
||||
parent_span_id="",
|
||||
name="root-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources=common_resources,
|
||||
attributes={"http.request.method": "GET"},
|
||||
),
|
||||
# span from a different trace — must not appear in results
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=other_trace_id,
|
||||
span_id=other_span_id,
|
||||
parent_span_id="",
|
||||
name="other-root-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
trace_filter = f"trace_id = '{target_trace_id}'"
|
||||
|
||||
def _query(start_ms: int, end_ms: int) -> tuple[list, list[str]]:
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=start_ms,
|
||||
end_ms=end_ms,
|
||||
request_type="raw",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"disabled": False,
|
||||
"limit": 100,
|
||||
"offset": 0,
|
||||
"filter": {"expression": trace_filter},
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
|
||||
"selectFields": [
|
||||
{
|
||||
"name": "name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "span",
|
||||
"signal": "traces",
|
||||
}
|
||||
],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
rows = response.json()["data"]["data"]["results"][0]["rows"] or []
|
||||
warning = (response.json().get("data") or {}).get("warning") or {}
|
||||
messages = [w.get("message", "") for w in (warning.get("warnings") or [])]
|
||||
return rows, messages
|
||||
|
||||
outside_range_msg = "lies outside the selected time range"
|
||||
|
||||
now_ms = int(now.timestamp() * 1000)
|
||||
|
||||
# --- Test 1: narrow window (single bucket, <1 h) ---
|
||||
narrow_start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
narrow_rows, narrow_warnings = _query(narrow_start_ms, now_ms)
|
||||
|
||||
assert len(narrow_rows) == 1, f"Expected 1 span for trace_id filter (narrow window), got {len(narrow_rows)}"
|
||||
assert narrow_rows[0]["data"]["span_id"] == span_id_root
|
||||
assert narrow_rows[0]["data"]["trace_id"] == target_trace_id
|
||||
assert not any(outside_range_msg in m for m in narrow_warnings), f"Did not expect outside-range warning, got {narrow_warnings}"
|
||||
|
||||
# --- Test 2: wide window (>1 h, triggers multiple exponential buckets) ---
|
||||
# should just return 1 span, not duplicate
|
||||
wide_start_ms = int((now - timedelta(hours=12)).timestamp() * 1000)
|
||||
wide_rows, wide_warnings = _query(wide_start_ms, now_ms)
|
||||
|
||||
assert len(wide_rows) == 1, f"Expected 1 span for trace_id filter (wide window, multi-bucket), got {len(wide_rows)} — possible duplicate-span regression"
|
||||
assert wide_rows[0]["data"]["span_id"] == span_id_root
|
||||
assert wide_rows[0]["data"]["trace_id"] == target_trace_id
|
||||
assert not any(outside_range_msg in m for m in wide_warnings), f"Did not expect outside-range warning, got {wide_warnings}"
|
||||
|
||||
# --- Test 3: window that does not contain the trace returns no results + warning ---
|
||||
past_start_ms = int((now - timedelta(hours=6)).timestamp() * 1000)
|
||||
past_end_ms = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
past_rows, past_warnings = _query(past_start_ms, past_end_ms)
|
||||
|
||||
assert len(past_rows) == 0, f"Expected 0 spans for trace_id filter outside time window, got {len(past_rows)}"
|
||||
assert any(outside_range_msg in m for m in past_warnings), f"Expected outside-range warning, got warnings={past_warnings}"
|
||||
|
||||
|
||||
def test_traces_aggregation_filter_by_trace_id(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Tests that the trace_id time-range optimization also applies to
|
||||
non-window-list (time_series / aggregation) traces queries:
|
||||
1. Wide query window containing the trace returns the correct count.
|
||||
2. Query window outside the trace's time range short-circuits to empty.
|
||||
3. Filter referencing a trace_id with no row in trace_summary
|
||||
short-circuits to empty (trace_summary is authoritative for traces).
|
||||
"""
|
||||
target_trace_id = TraceIdGenerator.trace_id()
|
||||
target_root_span_id = TraceIdGenerator.span_id()
|
||||
target_child_span_id = TraceIdGenerator.span_id()
|
||||
missing_trace_id = TraceIdGenerator.trace_id()
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
common_resources = {
|
||||
"deployment.environment": "production",
|
||||
"service.name": "traces-agg-filter-service",
|
||||
"cloud.provider": "integration",
|
||||
}
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
duration=timedelta(seconds=5),
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_root_span_id,
|
||||
parent_span_id="",
|
||||
name="root-span",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources=common_resources,
|
||||
attributes={"http.request.method": "GET"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=9),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=target_trace_id,
|
||||
span_id=target_child_span_id,
|
||||
parent_span_id=target_root_span_id,
|
||||
name="child-span",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources=common_resources,
|
||||
attributes={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
def _count(start_ms: int, end_ms: int, trace_id: str) -> tuple[float, list[str]]:
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=start_ms,
|
||||
end_ms=end_ms,
|
||||
request_type="time_series",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"filter": {"expression": f"trace_id = '{trace_id}'"},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
warning = (response.json().get("data") or {}).get("warning") or {}
|
||||
messages = [w.get("message", "") for w in (warning.get("warnings") or [])]
|
||||
aggregations = results[0].get("aggregations") or []
|
||||
if not aggregations:
|
||||
return 0, messages
|
||||
series = aggregations[0].get("series") or []
|
||||
if not series:
|
||||
return 0, messages
|
||||
return sum(v["value"] for v in series[0]["values"]), messages
|
||||
|
||||
outside_range_msg = "lies outside the selected time range"
|
||||
|
||||
now_ms = int(now.timestamp() * 1000)
|
||||
|
||||
# --- Test 1: wide window (>1 h) containing the trace returns both spans ---
|
||||
wide_start_ms = int((now - timedelta(hours=12)).timestamp() * 1000)
|
||||
wide_count, wide_warnings = _count(wide_start_ms, now_ms, target_trace_id)
|
||||
assert wide_count == 2, f"Expected count=2 for trace_id aggregation (wide window), got {wide_count}"
|
||||
assert not any(outside_range_msg in m for m in wide_warnings), f"Did not expect outside-range warning, got {wide_warnings}"
|
||||
|
||||
# --- Test 2: window outside the trace short-circuits to empty + warning ---
|
||||
past_start_ms = int((now - timedelta(hours=6)).timestamp() * 1000)
|
||||
past_end_ms = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
past_count, past_warnings = _count(past_start_ms, past_end_ms, target_trace_id)
|
||||
assert past_count == 0, f"Expected count=0 for trace_id aggregation outside time window, got {past_count}"
|
||||
assert any(outside_range_msg in m for m in past_warnings), f"Expected outside-range warning, got warnings={past_warnings}"
|
||||
|
||||
# --- Test 3: trace_id with no entry in trace_summary short-circuits (no warning) ---
|
||||
missing_start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
|
||||
missing_count, missing_warnings = _count(missing_start_ms, now_ms, missing_trace_id)
|
||||
assert missing_count == 0, f"Expected count=0 for trace_id absent from trace_summary, got {missing_count}"
|
||||
assert not any(outside_range_msg in m for m in missing_warnings), f"Did not expect outside-range warning for missing trace_id, got {missing_warnings}"
|
||||
53
tests/integration/tests/queriertraces/07_filter_errors.py
Normal file
53
tests/integration/tests/queriertraces/07_filter_errors.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import make_query_request
|
||||
|
||||
# Traces filter validation: a truly-unknown key is rejected, and the body-JSON
|
||||
# functions has()/hasToken() are not valid on the traces signal. All are pre-query
|
||||
# validation errors (400) independent of ingested data.
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression",
|
||||
[
|
||||
# unknown key. NOTE: current HEAD rejects (400); the parked
|
||||
# convert-not-found-to-warning change will make this a 200 + warning.
|
||||
pytest.param('totally.unknown.key = "x"', id="key_not_found"),
|
||||
# has()/hasToken() support only the logs body JSON field.
|
||||
pytest.param('has(service.name, "x")', id="has_on_traces"),
|
||||
pytest.param('hasToken(name, "x")', id="hastoken_on_traces"),
|
||||
],
|
||||
)
|
||||
def test_traces_filter_validation_errors(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
expression: str,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="scalar",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"filter": {"expression": expression},
|
||||
"aggregations": [{"expression": "count()"}],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
@@ -0,0 +1,98 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import index_series_by_label, make_query_request
|
||||
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
|
||||
|
||||
# min()/max() aggregation expressions over a span field. The traces aggregation
|
||||
# suite (02_aggregation.py) covers count/countIf/avg/p99 but not min/max.
|
||||
|
||||
|
||||
def test_traces_aggregate_min_max(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""max(duration_nano)/min(duration_nano) grouped by service.name return the extremes."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
duration=timedelta(seconds=3), # 3e9
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
name="POST /x",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
resources={"service.name": "http-service"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
duration=timedelta(milliseconds=500), # 0.5e9 (min)
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
name="SELECT",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
resources={"service.name": "http-service"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
duration=timedelta(seconds=1), # 1e9
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
name="PATCH",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
resources={"service.name": "http-service"},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
duration=timedelta(seconds=4), # 4e9
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
name="topic publish",
|
||||
kind=TracesKind.SPAN_KIND_PRODUCER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
resources={"service.name": "topic-service"},
|
||||
),
|
||||
]
|
||||
)
|
||||
token = get_token(email=USER_ADMIN_EMAIL, password=USER_ADMIN_PASSWORD)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int((now + timedelta(seconds=5)).timestamp() * 1000),
|
||||
request_type="time_series",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "traces",
|
||||
"groupBy": [{"name": "service.name", "fieldContext": "resource", "fieldDataType": "string"}],
|
||||
"aggregations": [
|
||||
{"expression": "max(duration_nano)", "alias": "maxDuration"},
|
||||
{"expression": "min(duration_nano)", "alias": "minDuration"},
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
aggregations = response.json()["data"]["data"]["results"][0]["aggregations"]
|
||||
|
||||
max_by_svc = index_series_by_label(aggregations[0]["series"], "service.name")
|
||||
min_by_svc = index_series_by_label(aggregations[1]["series"], "service.name")
|
||||
assert max_by_svc["http-service"]["values"][0]["value"] == 3_000_000_000.0
|
||||
assert min_by_svc["http-service"]["values"][0]["value"] == 500_000_000.0
|
||||
assert max_by_svc["topic-service"]["values"][0]["value"] == 4_000_000_000.0
|
||||
assert min_by_svc["topic-service"]["values"][0]["value"] == 4_000_000_000.0
|
||||
Reference in New Issue
Block a user