Compare commits

...

2 Commits

Author SHA1 Message Date
srikanthccv
f8cf2c5f79 chore(tests): keep only the non-obvious constraint in integration test docstrings
The Setup and Tests paragraphs restated the test body. A docstring now
stays only where it says something the name and the code do not.

Assisted-by: Claude Fable 5.1
2026-11-10 18:40:37 +05:30
Tushar Vats
f02b5521c6 feat(querier): heatmaps for the logs and traces signals (#13318)
Some checks failed
build-staging / prepare (push) Has been cancelled
build-staging / js-build (push) Has been cancelled
build-staging / go-build (push) Has been cancelled
build-staging / staging (push) Has been cancelled
cacheci / tests (push) Has been cancelled
Release Drafter / update_release_draft (push) Has been cancelled
#### Description

Extends the metrics heatmap to logs, traces and audit logs, so a heatmap
can be built ad hoc from raw events with any filter instead of from a
pre-generated metric.

- The aggregation is the bare field to bucket, e.g. `duration_nano`.
Each cell counts events. Any other aggregation, such as `count()`, is
rejected, and a query takes one aggregation.
- Rows without the field are excluded by default: `<field> EXISTS` is
ANDed onto the filter, so it also applies to the resource filter and the
group limit. A present but non-numeric value counts as 0.
- `bucketOptions` is optional and defaults to the log axis, as for
metrics. Distribution requests work the same way.
- The heatmap is the time series query with the value bucket joining the
group keys, so a row is one cell and the response shape is unchanged.
Results are cached, and a cached axis is trimmed to the requested
window.
- The bucket expressions move to `pkg/querybuilder/heatmap.go`, shared
with metrics. The value is read once through a `WITH` alias instead of
being repeated in every bound. The metrics SQL is unchanged.
- A disabled query in a heatmap request feeds a formula, so it builds as
a time series, as metrics already did.
- A field nothing carries draws an empty chart with the key-not-found
warning.

#### Issues closed by this PR

Closes https://github.com/SigNoz/pulse-pod/issues/338

#### Additional Information

- No API schema change.
- Integration tests (`querierlogs/17_heatmap.py`,
`queriertraces/16_heatmap.py`, `auditquerier/02_heatmap.py`) cover
linear, log and default axes, group by, rows without the field, and the
rejections.
- Backend only. Dashboards first, explorers later.
2026-11-10 09:05:04 +00:00
45 changed files with 1268 additions and 1078 deletions

View File

@@ -915,13 +915,15 @@ func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, q qbtypes
// as empty rows that the same request never has when it bypasses the cache.
//
// PromQL and ClickHouse queries name their own buckets, and a histogram's are
// its `le` labels, so only a gauge, sum or summary axis is cut.
// its `le` labels, so only a gauge, sum or summary axis and an event axis are cut.
func (bc *bucketCache) trimHeatmapAxisToTheWindow(q qbtypes.Query, tsData *qbtypes.TimeSeriesData) {
metricQuery, isMetricQuery := q.(*builderQuery[qbtypes.MetricAggregation])
if !isMetricQuery {
return
}
if metricQuery.spec.Aggregations[0].Type == metrictypes.HistogramType {
switch query := q.(type) {
case *builderQuery[qbtypes.MetricAggregation]:
if query.spec.Aggregations[0].Type == metrictypes.HistogramType {
return
}
case *builderQuery[qbtypes.LogAggregation], *builderQuery[qbtypes.TraceAggregation]:
default:
return
}

View File

@@ -77,7 +77,7 @@ func newBuilderQuery[T any](
func (q *builderQuery[T]) Fingerprint() string {
if (q.spec.Signal == telemetrytypes.SignalTraces ||
q.spec.Signal == telemetrytypes.SignalLogs) && q.kind != qbtypes.RequestTypeTimeSeries {
q.spec.Signal == telemetrytypes.SignalLogs) && !q.kind.IsBuiltFromTimeSeries() {
// No caching for non-timeseries queries
return ""
}
@@ -150,6 +150,12 @@ func (q *builderQuery[T]) Fingerprint() string {
parts = append(parts, fmt.Sprintf("aggs=[%s]", strings.Join(aggParts, ",")))
}
// A metrics heatmap carries its axis on the aggregation, which the loop above
// already fingerprinted. Logs and traces carry it on the spec.
if q.kind.CountsIntoBuckets() && q.spec.Signal != telemetrytypes.SignalMetrics {
parts = append(parts, "buckets="+fingerprintHeatmapBucketing(q.spec.BucketOptions.ToHeatmapBucketing()))
}
// Add filter if present
if q.spec.Filter != nil && q.spec.Filter.Expression != "" {
parts = append(parts, fmt.Sprintf("filter=%s", q.spec.Filter.Expression))

View File

@@ -191,6 +191,14 @@ func postProcessBuilderQuery[T any](
req *qbtypes.QueryRangeRequest,
) *qbtypes.Result {
// A disabled query feeds a formula as a time series, so only an enabled one
// came back bucketed.
if req.RequestType.CountsIntoBuckets() && !query.Disabled {
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
resolveHeatmapBucketAxis(tsData, query.BucketOptions.ToHeatmapBucketing())
}
}
// the statement's top-N already skipped offset groups by their whole-window value; only reorder here
result = q.applySeriesLimit(result, query.Limit, 0, query.Order)

View File

@@ -257,27 +257,30 @@ func (q *querier) buildQueries(
if !ok {
return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid AI builder query spec %T", query.Spec)
}
requestType := requestTypeForQuery(req.RequestType, spec.Disabled)
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, requestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, query.Type, spec, timeRange, requestType, tmplVars, builderConfig{})
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryTypeBuilder:
switch spec := query.Spec.(type) {
case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
requestType := requestTypeForQuery(req.RequestType, spec.Disabled)
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, requestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, query.Type, spec, timeRange, requestType, tmplVars, builderConfig{})
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
requestType := requestTypeForQuery(req.RequestType, spec.Disabled)
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, requestType)
stmtBuilder := q.logStmtBuilder
if spec.Source == telemetrytypes.SourceAudit {
stmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, q.builderConfig)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, query.Type, spec, timeRange, requestType, tmplVars, q.builderConfig)
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
@@ -288,13 +291,7 @@ func (q *querier) buildQueries(
if missingMetricQuerySet[spec.Name] {
continue
}
requestType := req.RequestType
if requestType.CountsIntoBuckets() && spec.Disabled {
// A disabled query in a heatmap request feeds a formula, and the
// formula converts time series into heatmap data, so its inputs
// run as time series queries.
requestType = qbtypes.RequestTypeTimeSeries
}
requestType := requestTypeForQuery(req.RequestType, spec.Disabled)
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, requestType)
var bq *builderQuery[qbtypes.MetricAggregation]
@@ -1196,6 +1193,17 @@ func clampStep(qe *qbtypes.QueryEnvelope, recommended, min uint64, warnings *[]s
}
}
// requestTypeForQuery downgrades a disabled query in a heatmap request to a
// time series one. A disabled query feeds a formula rather than the chart, and
// the formula is what turns time series into cells, so its inputs state no
// bucket axis of their own.
func requestTypeForQuery(requestType qbtypes.RequestType, disabled bool) qbtypes.RequestType {
if requestType.CountsIntoBuckets() && disabled {
return qbtypes.RequestTypeTimeSeries
}
return requestType
}
// extractShiftFromBuilderQuery extracts the shift value from timeShift function if present.
func extractShiftFromBuilderQuery[T any](spec qbtypes.QueryBuilderQuery[T]) int64 {
for _, fn := range spec.Functions {

136
pkg/querybuilder/heatmap.go Normal file
View File

@@ -0,0 +1,136 @@
package querybuilder
import (
"context"
"fmt"
"math"
"strconv"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/huandu/go-sqlbuilder"
)
// heatmapBucketValueAlias names the value a heatmap's bounds are cut from. The
// bounds read it several times, so it is declared once as a WITH expression
// alias rather than repeated.
const heatmapBucketValueAlias = "__bucket_value"
// RenderHeatmapBucketExprs renders the bucket (min, max] that valueExpr falls
// in. Only the bucket under everything the axis covers is open below, and only
// the one over it is open above.
//
// valueExpr is repeated several times in each expression, so it has to be free
// of placeholders — HeatmapBucketColumns passes an alias for that reason.
func RenderHeatmapBucketExprs(bucketing qbtypes.HeatmapBucketing, valueExpr string) (minExpr, maxExpr string) {
if bucketing.Kind == qbtypes.BucketsKindLinear {
return renderLinearBucketExprs(bucketing, valueExpr)
}
return renderLogBucketExprs(valueExpr)
}
func renderLinearBucketExprs(bucketing qbtypes.HeatmapBucketing, valueExpr string) (string, string) {
maxValue := FormatHeatmapFloat(bucketing.MaxValue)
numBuckets := strconv.Itoa(bucketing.NumBuckets)
index := fmt.Sprintf("least(greatest(ceil(%s * %s / %s), 1), %s)", valueExpr, numBuckets, maxValue, numBuckets)
minExpr := fmt.Sprintf(
"multiIf(%s <= 0, toFloat64('-Inf'), %s > %s, toFloat64(%s), (%s - 1) * %s / %s)",
valueExpr, valueExpr, maxValue, maxValue, index, maxValue, numBuckets,
)
maxExpr := fmt.Sprintf(
"multiIf(%s <= 0, toFloat64(0), %s > %s, toFloat64('+Inf'), %s * %s / %s)",
valueExpr, valueExpr, maxValue, index, maxValue, numBuckets,
)
return minExpr, maxExpr
}
// ClickHouse buckets at MaxLogScale whatever HeatmapBucketing.LogScale asks for;
// postprocessing folds the axis down afterwards.
func renderLogBucketExprs(valueExpr string) (string, string) {
bucketsPerDoubling := FormatHeatmapFloat(math.Exp2(qbtypes.MaxLogScale))
lowest := FormatHeatmapFloat(qbtypes.MinLogUpperBound)
highest := FormatHeatmapFloat(qbtypes.MaxLogUpperBound)
minExpr := fmt.Sprintf(
"multiIf(%s <= 0, toFloat64('-Inf'), %s <= %s, toFloat64(0), %s > %s, toFloat64(%s), pow(2, (ceil(log2(%s) * %s) - 1) / %s))",
valueExpr, valueExpr, lowest, valueExpr, highest, highest, valueExpr, bucketsPerDoubling, bucketsPerDoubling,
)
maxExpr := fmt.Sprintf(
"multiIf(%s <= 0, toFloat64(0), %s <= %s, %s, %s > %s, toFloat64('+Inf'), pow(2, ceil(log2(%s) * %s) / %s))",
valueExpr, valueExpr, lowest, lowest, valueExpr, highest, valueExpr, bucketsPerDoubling, bucketsPerDoubling,
)
return minExpr, maxExpr
}
// FormatHeatmapFloat renders a float64 as the shortest literal that reads back
// as the same value, so an upper bound computed from it is identical on every
// row.
func FormatHeatmapFloat(v float64) string {
return strconv.FormatFloat(v, 'g', -1, 64)
}
// HeatmapFieldMayBeAbsent reports whether some row can lack key. A key that
// resolves to a table column every row carries, such as a span's duration,
// cannot, as the column is read first, and an EXISTS on it would drop its
// zeros instead.
func HeatmapFieldMayBeAbsent(
ctx context.Context,
info qbtypes.QueryInfo,
storage qbtypes.Storage,
key *telemetrytypes.TelemetryFieldKey,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
) (bool, error) {
resolved, err := Resolve(ctx, info, storage, key, qbtypes.FilterOperatorUnknown, nil, keys)
if err != nil {
return false, err
}
for _, field := range resolved.Fields {
read, err := LogicalRead(ctx, info, storage, field)
if err != nil {
return false, err
}
if read.WhenAbsent == qbtypes.AlwaysPresent {
return false, nil
}
}
return true, nil
}
// HeatmapBucketColumns adds the bucket a row's value falls in to sb's select
// list, as the (min, max] pair a heatmap statement reports. key is the field
// the axis is cut from, already adjusted for context and data type.
//
// The bounds read the value through an expression alias rather than repeating
// it, so the returned fragment has to be declared alongside the caller's CTEs.
// It carries no arguments of its own.
//
// A row without a numeric value counts as 0.
func HeatmapBucketColumns(
ctx context.Context,
info qbtypes.QueryInfo,
storage qbtypes.Storage,
sb *sqlbuilder.SelectBuilder,
key *telemetrytypes.TelemetryFieldKey,
bucketOptions *qbtypes.BucketOptions,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
) (valueFragment string, groupByAliases []string, warnings []string, err error) {
// Resolved directly rather than through ResolveColumn: a key nothing carries
// puts every row at 0, and the warning is the only thing that says why.
resolved, err := Resolve(ctx, info, storage, key, qbtypes.FilterOperatorUnknown, nil, keys)
if err != nil {
return "", nil, nil, err
}
valueExpr, err := Column(ctx, info, storage, resolved, telemetrytypes.FieldDataTypeFloat64)
if err != nil {
return "", nil, nil, err
}
bucketMin, bucketMax := RenderHeatmapBucketExprs(bucketOptions.ToHeatmapBucketing(), heatmapBucketValueAlias)
sb.SelectMore(sqlbuilder.Escape(fmt.Sprintf("%s AS %s", bucketMin, qbtypes.HeatmapBucketMinColumn)))
sb.SelectMore(sqlbuilder.Escape(fmt.Sprintf("%s AS %s", bucketMax, qbtypes.HeatmapBucketMaxColumn)))
valueFragment = fmt.Sprintf("ifNull(%s, 0) AS %s", valueExpr, heatmapBucketValueAlias)
return valueFragment, []string{qbtypes.HeatmapBucketMinColumn, qbtypes.HeatmapBucketMaxColumn}, resolved.Warnings, nil
}

View File

@@ -110,6 +110,8 @@ func (b *auditQueryStatementBuilder) Build(
stmt, err = b.buildListQuery(ctx, orgID, q, query, start, end, keys, variables)
case qbtypes.RequestTypeTimeSeries:
stmt, err = b.buildTimeSeriesQuery(ctx, orgID, q, query, start, end, keys, variables)
case qbtypes.RequestTypeHeatmap, qbtypes.RequestTypeDistribution:
stmt, err = b.buildHeatmapTimeSeriesQuery(ctx, orgID, q, query, start, end, keys, variables)
case qbtypes.RequestTypeScalar:
stmt, err = b.buildScalarQuery(ctx, orgID, q, query, start, end, keys, false, variables)
default:
@@ -322,6 +324,48 @@ func (b *auditQueryStatementBuilder) buildTimeSeriesQuery(
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
return b.buildTimeSeriesOrHeatmapQuery(ctx, orgID, sb, query, start, end, keys, variables, nil)
}
// buildHeatmapTimeSeriesQuery counts rows per (time bucket, group, value
// bucket) of the field its aggregation names.
func (b *auditQueryStatementBuilder) buildHeatmapTimeSeriesQuery(
ctx context.Context,
orgID valuer.UUID,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
heatmapField, countQuery, err := qbtypes.HeatmapCountQuery(query)
if err != nil {
return nil, err
}
info := querybuilder.NewQueryInfo(ctx, orgID, b.fl, telemetrytypes.SignalLogs, nil, start, end)
mayBeAbsent, err := querybuilder.HeatmapFieldMayBeAbsent(ctx, info, b.storage, heatmapField, keys)
if err != nil {
return nil, err
}
if mayBeAbsent {
countQuery = qbtypes.RequireHeatmapField(countQuery, *heatmapField)
}
return b.buildTimeSeriesOrHeatmapQuery(ctx, orgID, sb, countQuery, start, end, keys, variables, heatmapField)
}
// buildTimeSeriesOrHeatmapQuery builds one row per (time bucket, group). A
// non-nil heatmapField makes it a heatmap instead: the bucket a row's
// heatmapField falls in joins the group keys, so a row is one cell.
func (b *auditQueryStatementBuilder) buildTimeSeriesOrHeatmapQuery(
ctx context.Context,
orgID valuer.UUID,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
heatmapField *telemetrytypes.TelemetryFieldKey,
) (*qbtypes.Statement, error) {
var (
cteFragments []string
@@ -357,6 +401,17 @@ func (b *auditQueryStatementBuilder) buildTimeSeriesQuery(
fieldNames = append(fieldNames, sqlbuilder.Escape(clickhousesql.Identifier(gb.Name)))
}
var bucketAliases []string
if heatmapField != nil {
valueFragment, aliases, warnings, err := querybuilder.HeatmapBucketColumns(ctx, info, b.storage, sb, heatmapField, query.BucketOptions, keys)
if err != nil {
return nil, err
}
cteFragments = append(cteFragments, valueFragment)
bucketAliases = aliases
columnWarnings = append(columnWarnings, warnings...)
}
allAggChArgs := make([]any, 0)
for i, agg := range query.Aggregations {
rewritten, chArgs, aggWarnings, err := b.aggExprRewriter.Rewrite(ctx, orgID, start, end, agg.Expression, uint64(query.StepInterval.Seconds()), keys)
@@ -393,6 +448,7 @@ func (b *auditQueryStatementBuilder) buildTimeSeriesQuery(
sb.GroupBy("ts")
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
sb.GroupBy(bucketAliases...)
if query.Having != nil && query.Having.Expression != "" {
rewriter := querybuilder.NewHavingExpressionRewriter()
rewrittenExpr, err := rewriter.RewriteForLogs(query.Having.Expression, query.Aggregations)
@@ -420,6 +476,7 @@ func (b *auditQueryStatementBuilder) buildTimeSeriesQuery(
} else {
sb.GroupBy("ts")
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
sb.GroupBy(bucketAliases...)
if query.Having != nil && query.Having.Expression != "" {
rewriter := querybuilder.NewHavingExpressionRewriter()
rewrittenExpr, err := rewriter.RewriteForLogs(query.Having.Expression, query.Aggregations)

View File

@@ -13,6 +13,7 @@ import (
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -224,3 +225,59 @@ func TestStatementBuilder(t *testing.T) {
})
}
}
func TestStatementBuilderHeatmap(t *testing.T) {
statementBuilder := newTestAuditStatementBuilder(t)
testCases := []struct {
name string
requestType qbtypes.RequestType
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
expected qbtypes.Statement
}{
{
name: "AttributeField_GroupByMaterialized_LinearAxis_RequiresField",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.LogAggregation{{Expression: "http.response.status_code"}},
Filter: &qbtypes.Filter{Expression: "signoz.audit.resource.kind = 'dashboard'"},
GroupBy: []qbtypes.GroupByKey{{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "signoz.audit.action"}}},
BucketOptions: &qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLinear, Spec: qbtypes.LinearBucketsSpec{MaxValue: 500, NumBuckets: 5}},
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE ((simpleJSONExtractString(labels, 'signoz.audit.resource.kind') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), ifNull(multiIf(mapContains(attributes_number, 'http.response.status_code'), toFloat64(attributes_number['http.response.status_code']), NULL), 0) AS __bucket_value SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toString(multiIf(`attribute_string_signoz$$audit$$action_exists` = true, `attribute_string_signoz$$audit$$action`, NULL)) AS `signoz.audit.action`, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value > 500, toFloat64(500), (least(greatest(ceil(__bucket_value * 5 / 500), 1), 5) - 1) * 500 / 5) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value > 500, toFloat64('+Inf'), least(greatest(ceil(__bucket_value * 5 / 500), 1), 5) * 500 / 5) AS __bucket_max, count() AS __result_0 FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND mapContains(attributes_number, 'http.response.status_code') AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, `signoz.audit.action`, __bucket_min, __bucket_max",
Args: []any{"dashboard", "%signoz.audit.resource.kind%", "%signoz.audit.resource.kind\":\"dashboard%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
},
},
{
name: "Distribution_AttributeField_SameStatementAsHeatmap",
requestType: qbtypes.RequestTypeDistribution,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.LogAggregation{{Expression: "http.response.status_code"}},
Filter: &qbtypes.Filter{Expression: "signoz.audit.resource.kind = 'dashboard'"},
GroupBy: []qbtypes.GroupByKey{{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "signoz.audit.action"}}},
BucketOptions: &qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLinear, Spec: qbtypes.LinearBucketsSpec{MaxValue: 500, NumBuckets: 5}},
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE ((simpleJSONExtractString(labels, 'signoz.audit.resource.kind') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), ifNull(multiIf(mapContains(attributes_number, 'http.response.status_code'), toFloat64(attributes_number['http.response.status_code']), NULL), 0) AS __bucket_value SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toString(multiIf(`attribute_string_signoz$$audit$$action_exists` = true, `attribute_string_signoz$$audit$$action`, NULL)) AS `signoz.audit.action`, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value > 500, toFloat64(500), (least(greatest(ceil(__bucket_value * 5 / 500), 1), 5) - 1) * 500 / 5) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value > 500, toFloat64('+Inf'), least(greatest(ceil(__bucket_value * 5 / 500), 1), 5) * 500 / 5) AS __bucket_max, count() AS __result_0 FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND mapContains(attributes_number, 'http.response.status_code') AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, `signoz.audit.action`, __bucket_min, __bucket_max",
Args: []any{"dashboard", "%signoz.audit.resource.kind%", "%signoz.audit.resource.kind\":\"dashboard%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
statement, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, testCase.requestType, testCase.query, nil)
require.NoError(t, err)
assert.Equal(t, testCase.expected.Query, statement.Query)
assert.Equal(t, testCase.expected.Args, statement.Args)
assert.Equal(t, testCase.expected.Warnings, statement.Warnings)
})
}
}

View File

@@ -142,6 +142,8 @@ func (b *logQueryStatementBuilder) Build(
stmt, err = b.buildListQuery(ctx, orgID, q, query, start, end, keys, variables)
case qbtypes.RequestTypeTimeSeries:
stmt, err = b.buildTimeSeriesQuery(ctx, orgID, q, query, start, end, keys, variables)
case qbtypes.RequestTypeHeatmap, qbtypes.RequestTypeDistribution:
stmt, err = b.buildHeatmapTimeSeriesQuery(ctx, orgID, q, query, start, end, keys, variables)
case qbtypes.RequestTypeScalar:
stmt, err = b.buildScalarQuery(ctx, orgID, q, query, start, end, keys, false, variables)
default:
@@ -442,6 +444,48 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
return b.buildTimeSeriesOrHeatmapQuery(ctx, orgID, sb, query, start, end, keys, variables, nil)
}
// buildHeatmapTimeSeriesQuery counts rows per (time bucket, group, value
// bucket) of the field its aggregation names.
func (b *logQueryStatementBuilder) buildHeatmapTimeSeriesQuery(
ctx context.Context,
orgID valuer.UUID,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
heatmapField, countQuery, err := qbtypes.HeatmapCountQuery(query)
if err != nil {
return nil, err
}
info := querybuilder.NewQueryInfo(ctx, orgID, b.fl, telemetrytypes.SignalLogs, nil, start, end)
mayBeAbsent, err := querybuilder.HeatmapFieldMayBeAbsent(ctx, info, b.storage, columnKey(heatmapField), keys)
if err != nil {
return nil, err
}
if mayBeAbsent {
countQuery = qbtypes.RequireHeatmapField(countQuery, *heatmapField)
}
return b.buildTimeSeriesOrHeatmapQuery(ctx, orgID, sb, countQuery, start, end, keys, variables, heatmapField)
}
// buildTimeSeriesOrHeatmapQuery builds one row per (time bucket, group). A
// non-nil heatmapField makes it a heatmap instead: the bucket a row's
// heatmapField falls in joins the group keys, so a row is one cell.
func (b *logQueryStatementBuilder) buildTimeSeriesOrHeatmapQuery(
ctx context.Context,
orgID valuer.UUID,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
heatmapField *telemetrytypes.TelemetryFieldKey,
) (*qbtypes.Statement, error) {
var (
cteFragments []string
@@ -481,6 +525,19 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
fieldNames = append(fieldNames, sqlbuilder.Escape(fieldAlias))
}
var bucketAliases []string
if heatmapField != nil {
valueFragment, aliases, warnings, err := querybuilder.HeatmapBucketColumns(
ctx, info, b.storage, sb, columnKey(heatmapField), query.BucketOptions, keys,
)
if err != nil {
return nil, err
}
cteFragments = append(cteFragments, valueFragment)
bucketAliases = aliases
columnWarnings = append(columnWarnings, warnings...)
}
// Aggregations
allAggChArgs := make([]any, 0)
for i, agg := range query.Aggregations {
@@ -527,6 +584,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
// Group by all dimensions
sb.GroupBy("ts")
sb.GroupBy(fieldNames...)
sb.GroupBy(bucketAliases...)
if query.Having != nil && query.Having.Expression != "" {
// Rewrite having expression to use SQL column names
rewriter := querybuilder.NewHavingExpressionRewriter()
@@ -561,6 +619,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
} else {
sb.GroupBy("ts")
sb.GroupBy(fieldNames...)
sb.GroupBy(bucketAliases...)
if query.Having != nil && query.Having.Expression != "" {
rewriter := querybuilder.NewHavingExpressionRewriter()
rewrittenExpr, err := rewriter.RewriteForLogs(query.Having.Expression, query.Aggregations)

View File

@@ -19,6 +19,7 @@ import (
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -1271,3 +1272,119 @@ func newSkipResourceFingerprintLogsBuilder(
statementbuilder.Config{SkipResourceFingerprint: statementbuilder.SkipResourceFingerprint{Enabled: skipEnable, Threshold: threshold}},
)
}
func TestStatementBuilderHeatmap(t *testing.T) {
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
startTs := uint64(releaseTime.UnixNano()) + uint64(24*time.Hour.Nanoseconds())
endTs := uint64(releaseTime.UnixNano()) + uint64(48*time.Hour.Nanoseconds())
testCases := []struct {
name string
requestType qbtypes.RequestType
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
expected qbtypes.Statement
}{
{
name: "AttributeField_LogAxis_RequiresField",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.LogAggregation{{Expression: "duration"}},
Filter: &qbtypes.Filter{Expression: "service.name = 'cartservice'"},
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), ifNull(multiIf(mapContains(attributes_number, 'duration'), toFloat64(attributes_number['duration']), NULL), 0) AS __bucket_value SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value <= 2.3283064365386963e-10, toFloat64(0), __bucket_value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(__bucket_value) * 16) - 1) / 16)) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value <= 2.3283064365386963e-10, 2.3283064365386963e-10, __bucket_value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(__bucket_value) * 16) / 16)) AS __bucket_max, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND mapContains(attributes_number, 'duration') AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, __bucket_min, __bucket_max",
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
},
},
{
name: "AttributeField_GroupByWithLimit_LinearAxis_RequiresFieldInLimit",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.LogAggregation{{Expression: "http.status_code"}},
GroupBy: []qbtypes.GroupByKey{{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}},
Limit: 5,
BucketOptions: &qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLinear, Spec: qbtypes.LinearBucketsSpec{MaxValue: 600, NumBuckets: 6}},
},
expected: qbtypes.Statement{
Query: "WITH ifNull(multiIf(mapContains(attributes_number, 'http.status_code'), toFloat64(attributes_number['http.status_code']), NULL), 0) AS __bucket_value, __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE mapContains(attributes_number, 'http.status_code') AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value > 600, toFloat64(600), (least(greatest(ceil(__bucket_value * 6 / 600), 1), 6) - 1) * 600 / 6) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value > 600, toFloat64('+Inf'), least(greatest(ceil(__bucket_value * 6 / 600), 1), 6) * 600 / 6) AS __bucket_max, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE mapContains(attributes_number, 'http.status_code') AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, __bucket_min, __bucket_max",
Args: []any{"1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 5, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
},
},
{
name: "QualifiedField_OrderedByAggregation_OrdersByCount",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.LogAggregation{{Expression: "attribute.amount"}},
GroupBy: []qbtypes.GroupByKey{{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}},
Limit: 3,
Order: []qbtypes.OrderBy{{
Key: qbtypes.OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "amount", FieldContext: telemetrytypes.FieldContextAttribute}},
Direction: qbtypes.OrderDirectionAsc,
}},
},
expected: qbtypes.Statement{
Query: "WITH ifNull(multiIf(mapContains(attributes_number, 'amount'), toFloat64(attributes_number['amount']), NULL), 0) AS __bucket_value, __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE mapContains(attributes_number, 'amount') AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 asc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value <= 2.3283064365386963e-10, toFloat64(0), __bucket_value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(__bucket_value) * 16) - 1) / 16)) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value <= 2.3283064365386963e-10, 2.3283064365386963e-10, __bucket_value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(__bucket_value) * 16) / 16)) AS __bucket_max, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE mapContains(attributes_number, 'amount') AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, __bucket_min, __bucket_max ORDER BY ts desc",
Args: []any{"1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 3, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
},
},
{
name: "IntrinsicColumn_NoExists",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.LogAggregation{{Expression: "log.severity_number"}},
},
expected: qbtypes.Statement{
Query: "WITH ifNull(toFloat64(severity_number), 0) AS __bucket_value SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value <= 2.3283064365386963e-10, toFloat64(0), __bucket_value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(__bucket_value) * 16) - 1) / 16)) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value <= 2.3283064365386963e-10, 2.3283064365386963e-10, __bucket_value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(__bucket_value) * 16) / 16)) AS __bucket_max, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, __bucket_min, __bucket_max",
Args: []any{"1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
},
},
{
name: "Distribution_AttributeField_SameStatementAsHeatmap",
requestType: qbtypes.RequestTypeDistribution,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.LogAggregation{{Expression: "duration"}},
Filter: &qbtypes.Filter{Expression: "service.name = 'cartservice'"},
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), ifNull(multiIf(mapContains(attributes_number, 'duration'), toFloat64(attributes_number['duration']), NULL), 0) AS __bucket_value SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value <= 2.3283064365386963e-10, toFloat64(0), __bucket_value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(__bucket_value) * 16) - 1) / 16)) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value <= 2.3283064365386963e-10, 2.3283064365386963e-10, __bucket_value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(__bucket_value) * 16) / 16)) AS __bucket_max, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND mapContains(attributes_number, 'duration') AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, __bucket_min, __bucket_max",
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
},
},
}
fl := flaggertest.New(t)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = logstelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
storage := logstelemetryschema.NewStorage()
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, storage, fl, telemetrytypes.SignalLogs)
statementBuilder := NewLogQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
storage,
aggExprRewriter,
logstelemetryschema.DefaultFullTextColumn,
fl,
nil,
statementbuilder.Config{SkipResourceFingerprint: statementbuilder.SkipResourceFingerprint{Enabled: false, Threshold: 100000}},
)
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
statement, err := statementBuilder.Build(context.Background(), valuer.UUID{}, startTs, endTs, testCase.requestType, testCase.query, nil)
require.NoError(t, err)
assert.Equal(t, testCase.expected.Query, statement.Query)
assert.Equal(t, testCase.expected.Args, statement.Args)
assert.Equal(t, testCase.expected.Warnings, statement.Warnings)
})
}
}

View File

@@ -4,14 +4,11 @@ import (
"context"
"fmt"
"log/slog"
"math"
"slices"
"strconv"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/clickhousesql"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -917,6 +914,10 @@ const (
heatmapValueAlias = "__result_0"
heatmapWindow = "__heatmap_window"
// The column __spatial_aggregation_cte carries the sample in, which is what
// the bucket axis is cut from.
heatmapValueColumn = "value"
)
func isHistogramBucket(k qbtypes.GroupByKey) bool { return k.Name == histogramBucketKey }
@@ -982,10 +983,7 @@ func buildValueHeatmapFinalSelect(
args []any,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
) (*qbtypes.Statement, error) {
bucketMin, bucketMax, err := renderHeatmapBucketExprs(*query.Aggregations[0].HeatmapBucketing)
if err != nil {
return nil, err
}
bucketMin, bucketMax := querybuilder.RenderHeatmapBucketExprs(*query.Aggregations[0].HeatmapBucketing, heatmapValueColumn)
groupAliases := GroupByAliases(query.GroupBy)
@@ -1003,61 +1001,6 @@ func buildValueHeatmapFinalSelect(
return &qbtypes.Statement{Query: combined + q, Args: append(args, a...)}, nil
}
// renderHeatmapBucketExprs renders the bucket (min, max] that `value` falls in.
// Only the bucket under everything the axis covers is open below, and only the
// one over it is open above.
func renderHeatmapBucketExprs(bucketing qbtypes.HeatmapBucketing) (minExpr, maxExpr string, err error) {
switch bucketing.Kind {
case qbtypes.BucketsKindLinear:
return renderLinearBucketExprs(bucketing)
case qbtypes.BucketsKindLog:
return renderLogBucketExprs()
default:
return "", "", errors.NewInvalidInputf(errors.CodeInvalidInput,
"unsupported bucketsScaling %q for heatmap requests", bucketing.Kind.StringValue())
}
}
func renderLinearBucketExprs(bucketing qbtypes.HeatmapBucketing) (string, string, error) {
maxValue := formatFloat(bucketing.MaxValue)
numBuckets := strconv.Itoa(bucketing.NumBuckets)
index := fmt.Sprintf("least(greatest(ceil(value * %s / %s), 1), %s)", numBuckets, maxValue, numBuckets)
minExpr := fmt.Sprintf(
"multiIf(value <= 0, toFloat64('-Inf'), value > %s, toFloat64(%s), (%s - 1) * %s / %s)",
maxValue, maxValue, index, maxValue, numBuckets,
)
maxExpr := fmt.Sprintf(
"multiIf(value <= 0, toFloat64(0), value > %s, toFloat64('+Inf'), %s * %s / %s)",
maxValue, index, maxValue, numBuckets,
)
return minExpr, maxExpr, nil
}
// ClickHouse buckets at MaxLogScale whatever HeatmapBucketing.LogScale asks for;
// postprocessing folds the axis down afterwards.
func renderLogBucketExprs() (string, string, error) {
bucketsPerDoubling := formatFloat(math.Exp2(qbtypes.MaxLogScale))
lowest := formatFloat(qbtypes.MinLogUpperBound)
highest := formatFloat(qbtypes.MaxLogUpperBound)
minExpr := fmt.Sprintf(
"multiIf(value <= 0, toFloat64('-Inf'), value <= %s, toFloat64(0), value > %s, toFloat64(%s), pow(2, (ceil(log2(value) * %s) - 1) / %s))",
lowest, highest, highest, bucketsPerDoubling, bucketsPerDoubling,
)
maxExpr := fmt.Sprintf(
"multiIf(value <= 0, toFloat64(0), value <= %s, %s, value > %s, toFloat64('+Inf'), pow(2, ceil(log2(value) * %s) / %s))",
lowest, lowest, highest, bucketsPerDoubling, bucketsPerDoubling,
)
return minExpr, maxExpr, nil
}
// formatFloat renders a float64 as the shortest literal that reads back as the
// same value, so an upper bound computed from it is identical on every row.
func formatFloat(v float64) string {
return strconv.FormatFloat(v, 'g', -1, 64)
}
func GroupByColumnAlias(i int, name string) string {
if name == histogramBucketKey {
return clickhousesql.Identifier(histogramBucketKey)

View File

@@ -178,6 +178,8 @@ func (b *traceQueryStatementBuilder) Build(
return b.buildListQuery(ctx, orgID, q, query, start, end, keys, variables, isSelectFieldsEmpty)
case qbtypes.RequestTypeTimeSeries:
return b.buildTimeSeriesQuery(ctx, orgID, q, query, start, end, keys, variables)
case qbtypes.RequestTypeHeatmap, qbtypes.RequestTypeDistribution:
return b.buildHeatmapTimeSeriesQuery(ctx, orgID, q, query, start, end, keys, variables)
case qbtypes.RequestTypeScalar:
return b.buildScalarQuery(ctx, orgID, q, query, start, end, keys, variables, false, false)
case qbtypes.RequestTypeTrace:
@@ -570,6 +572,48 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
return b.buildTimeSeriesOrHeatmapQuery(ctx, orgID, sb, query, start, end, keys, variables, nil)
}
// buildHeatmapTimeSeriesQuery counts rows per (time bucket, group, value
// bucket) of the field its aggregation names.
func (b *traceQueryStatementBuilder) buildHeatmapTimeSeriesQuery(
ctx context.Context,
orgID valuer.UUID,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
heatmapField, countQuery, err := qbtypes.HeatmapCountQuery(query)
if err != nil {
return nil, err
}
info := querybuilder.NewQueryInfo(ctx, orgID, b.fl, telemetrytypes.SignalTraces, nil, start, end)
mayBeAbsent, err := querybuilder.HeatmapFieldMayBeAbsent(ctx, info, b.storage, heatmapField, keys)
if err != nil {
return nil, err
}
if mayBeAbsent {
countQuery = qbtypes.RequireHeatmapField(countQuery, *heatmapField)
}
return b.buildTimeSeriesOrHeatmapQuery(ctx, orgID, sb, countQuery, start, end, keys, variables, heatmapField)
}
// buildTimeSeriesOrHeatmapQuery builds one row per (time bucket, group). A
// non-nil heatmapField makes it a heatmap instead: the bucket a row's
// heatmapField falls in joins the group keys, so a row is one cell.
func (b *traceQueryStatementBuilder) buildTimeSeriesOrHeatmapQuery(
ctx context.Context,
orgID valuer.UUID,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
heatmapField *telemetrytypes.TelemetryFieldKey,
) (*qbtypes.Statement, error) {
var (
cteFragments []string
@@ -610,6 +654,19 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
fieldNames = append(fieldNames, sqlbuilder.Escape(fieldAlias))
}
var bucketAliases []string
if heatmapField != nil {
valueFragment, aliases, warnings, err := querybuilder.HeatmapBucketColumns(
ctx, info, b.storage, sb, heatmapField, query.BucketOptions, keys,
)
if err != nil {
return nil, err
}
cteFragments = append(cteFragments, valueFragment)
bucketAliases = aliases
columnWarnings = append(columnWarnings, warnings...)
}
// Aggregations
allAggChArgs := make([]any, 0)
for i, agg := range query.Aggregations {
@@ -653,6 +710,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
// Group by all dimensions
sb.GroupBy("ts")
sb.GroupBy(fieldNames...)
sb.GroupBy(bucketAliases...)
if query.Having != nil && query.Having.Expression != "" {
rewriter := querybuilder.NewHavingExpressionRewriter()
rewrittenExpr, err := rewriter.RewriteForTraces(query.Having.Expression, query.Aggregations)
@@ -686,6 +744,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
} else {
sb.GroupBy("ts")
sb.GroupBy(fieldNames...)
sb.GroupBy(bucketAliases...)
if query.Having != nil && query.Having.Expression != "" {
rewriter := querybuilder.NewHavingExpressionRewriter()
rewrittenExpr, err := rewriter.RewriteForTraces(query.Having.Expression, query.Aggregations)

View File

@@ -2113,3 +2113,103 @@ func TestStatementBuilderSemconvSingleSpelling(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'deployment.environment') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT toString(multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, mapContains(resources_string, 'deployment.environment'), resources_string['deployment.environment'], NULL)) AS `__GROUP_BY_KEY_0_deployment.environment.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_deployment.environment.name` ORDER BY __result_0 DESC", q.Query)
}
func TestStatementBuilderHeatmap(t *testing.T) {
testCases := []struct {
name string
requestType qbtypes.RequestType
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]
expected qbtypes.Statement
}{
{
name: "DurationNano_CollidingAttribute_NoExists",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{{Expression: "duration_nano"}},
Filter: &qbtypes.Filter{Expression: "service.name = 'redis-manual'"},
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), ifNull(toFloat64(duration_nano), 0) AS __bucket_value SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value <= 2.3283064365386963e-10, toFloat64(0), __bucket_value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(__bucket_value) * 16) - 1) / 16)) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value <= 2.3283064365386963e-10, 2.3283064365386963e-10, __bucket_value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(__bucket_value) * 16) / 16)) AS __bucket_max, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY ts, __bucket_min, __bucket_max",
Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
{
name: "AttributeField_GroupByWithLimit_LinearAxis_RequiresFieldInLimit",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.TraceAggregation{{Expression: "cart.items_count"}},
GroupBy: []qbtypes.GroupByKey{{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}},
Limit: 5,
BucketOptions: &qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLinear, Spec: qbtypes.LinearBucketsSpec{MaxValue: 50, NumBuckets: 5}},
},
expected: qbtypes.Statement{
Query: "WITH ifNull(multiIf(`attribute_number_cart$$items_count_exists` = true, toFloat64(`attribute_number_cart$$items_count`), NULL), 0) AS __bucket_value, __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE `attribute_number_cart$$items_count_exists` = true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value > 50, toFloat64(50), (least(greatest(ceil(__bucket_value * 5 / 50), 1), 5) - 1) * 50 / 5) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value > 50, toFloat64('+Inf'), least(greatest(ceil(__bucket_value * 5 / 50), 1), 5) * 50 / 5) AS __bucket_max, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE `attribute_number_cart$$items_count_exists` = true AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, __bucket_min, __bucket_max",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 5, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
{
name: "QualifiedField_OrderedByAggregation_OrdersByCount",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{{Expression: "attribute.metric.max_count"}},
GroupBy: []qbtypes.GroupByKey{{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}}},
Limit: 3,
Order: []qbtypes.OrderBy{{
Key: qbtypes.OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "metric.max_count", FieldContext: telemetrytypes.FieldContextAttribute}},
Direction: qbtypes.OrderDirectionDesc,
}},
},
expected: qbtypes.Statement{
Query: "WITH ifNull(multiIf(mapContains(attributes_number, 'metric.max_count'), toFloat64(attributes_number['metric.max_count']), NULL), 0) AS __bucket_value, __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE mapContains(attributes_number, 'metric.max_count') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 desc LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value <= 2.3283064365386963e-10, toFloat64(0), __bucket_value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(__bucket_value) * 16) - 1) / 16)) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value <= 2.3283064365386963e-10, 2.3283064365386963e-10, __bucket_value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(__bucket_value) * 16) / 16)) AS __bucket_max, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE mapContains(attributes_number, 'metric.max_count') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, __bucket_min, __bucket_max ORDER BY ts desc",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 3, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
{
name: "Distribution_DurationNano_SameStatementAsHeatmap",
requestType: qbtypes.RequestTypeDistribution,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{{Expression: "duration_nano"}},
Filter: &qbtypes.Filter{Expression: "service.name = 'redis-manual'"},
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), ifNull(toFloat64(duration_nano), 0) AS __bucket_value SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, multiIf(__bucket_value <= 0, toFloat64('-Inf'), __bucket_value <= 2.3283064365386963e-10, toFloat64(0), __bucket_value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(__bucket_value) * 16) - 1) / 16)) AS __bucket_min, multiIf(__bucket_value <= 0, toFloat64(0), __bucket_value <= 2.3283064365386963e-10, 2.3283064365386963e-10, __bucket_value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(__bucket_value) * 16) / 16)) AS __bucket_max, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY ts, __bucket_min, __bucket_max",
Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
}
releaseTime := time.Date(2025, 5, 22, 0, 0, 0, 0, time.UTC)
fl := flaggertest.New(t)
storage := tracestelemetryschema.NewStorage(nil)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, storage, fl, telemetrytypes.SignalTraces)
statementBuilder := NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
storage,
aggExprRewriter,
nil,
fl,
false,
100000,
)
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
statement, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, testCase.requestType, testCase.query, nil)
require.NoError(t, err)
assert.Equal(t, testCase.expected.Query, statement.Query)
assert.Equal(t, testCase.expected.Args, statement.Args)
assert.Equal(t, testCase.expected.Warnings, statement.Warnings)
})
}
}

View File

@@ -3,9 +3,11 @@ package querybuildertypesv5
import (
"math"
"slices"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
const (
@@ -16,6 +18,9 @@ const (
HeatmapBucketMinColumn = "__bucket_min"
HeatmapBucketMaxColumn = "__bucket_max"
// A logs or traces heatmap counts events, so its field renders as count().
heatmapCountExpression = "count()"
DefaultNumBuckets = 60
// MaxLogScale is the resolution ClickHouse buckets every log heatmap at:
@@ -81,6 +86,68 @@ func (b *BucketOptions) ToHeatmapBucketing() HeatmapBucketing {
return resolved
}
// HeatmapAggregationField returns the field a logs or traces heatmap buckets,
// written as the bare field. ok is false for any other aggregation.
func HeatmapAggregationField(expression string) (telemetrytypes.TelemetryFieldKey, bool) {
keyText := strings.TrimSpace(expression)
if keyText == "" || strings.ContainsAny(keyText, "() \t\n") {
return telemetrytypes.TelemetryFieldKey{}, false
}
return telemetrytypes.GetFieldKeyFromKeyText(keyText), true
}
// HeatmapCountQuery splits a logs or traces heatmap query into the field its
// aggregation buckets and a copy of query that counts events in its place,
// which is what every cell holds.
func HeatmapCountQuery[T LogAggregation | TraceAggregation](query QueryBuilderQuery[T]) (*telemetrytypes.TelemetryFieldKey, QueryBuilderQuery[T], error) {
if len(query.Aggregations) != 1 {
return nil, query, errors.NewInvalidInputf(errors.CodeInvalidInput,
"a heatmap renders one bucket axis, but the query has %d aggregations", len(query.Aggregations))
}
query.Aggregations = slices.Clone(query.Aggregations)
var expression *string
switch aggregation := any(&query.Aggregations[0]).(type) {
case *LogAggregation:
expression = &aggregation.Expression
case *TraceAggregation:
expression = &aggregation.Expression
}
field, ok := HeatmapAggregationField(*expression)
if !ok {
return nil, query, errors.NewInvalidInputf(errors.CodeInvalidInput,
"a logs or traces heatmap counts events by the field named as its aggregation, got %q", *expression)
}
// An order on the aggregation names it by its expression, which decoding
// splits into a context, a name and a data type, and which metadata may
// complete, as it does any field.
query.Order = slices.Clone(query.Order)
for i := range query.Order {
key := query.Order[i].Key.TelemetryFieldKey
sameField := key.Name == field.Name &&
(field.FieldContext == telemetrytypes.FieldContextUnspecified || key.FieldContext == field.FieldContext) &&
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || key.FieldDataType == field.FieldDataType)
if key.Name == *expression || sameField {
query.Order[i].Key = OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: heatmapCountExpression}}
}
}
*expression = heatmapCountExpression
return &field, query, nil
}
// RequireHeatmapField keeps only the rows of query that carry field.
func RequireHeatmapField[T LogAggregation | TraceAggregation](query QueryBuilderQuery[T], field telemetrytypes.TelemetryFieldKey) QueryBuilderQuery[T] {
exists := telemetrytypes.TelemetryFieldKeyToText(&field) + " EXISTS"
if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
exists = "(" + query.Filter.Expression + ") AND " + exists
}
query.Filter = &Filter{Expression: exists}
return query
}
// This cannot be called in validateBucketCounting cuz metric type is resolved in querier.go.
func (a *MetricAggregation) VerifyAndApplyBucketOptions(bucketOptions *BucketOptions) error {
switch a.Type {

View File

@@ -934,15 +934,26 @@ func (r *QueryRangeRequest) validateBucketCounting() error {
continue
}
enabled++
// An AI query decodes to the traces spec, so it lands here too. Admitting
// either signal means capping Aggregations at one: each carries its own
// Meta.Buckets, and a heatmap renders against a single bucket axis.
case QueryBuilderQuery[LogAggregation], QueryBuilderQuery[TraceAggregation]:
return errors.New(errors.TypeUnsupported, errors.CodeUnsupported,
"heatmaps are not supported for the logs and traces signals yet")
// An AI query decodes to the traces spec, so it lands here too.
case QueryBuilderQuery[LogAggregation]:
if err := validateEventHeatmapQuery(r.RequestType, spec.Functions, spec.Having, aggregationExpressions(spec.Aggregations), spec.Disabled); err != nil {
return err
}
if spec.Disabled {
continue
}
enabled++
case QueryBuilderQuery[TraceAggregation]:
if err := validateEventHeatmapQuery(r.RequestType, spec.Functions, spec.Having, aggregationExpressions(spec.Aggregations), spec.Disabled); err != nil {
return err
}
if spec.Disabled {
continue
}
enabled++
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"%s requests support one metrics builder query, one formula over them, one clickhouse query, or one promql query, got %q", r.RequestType.StringValue(), envelope.Type.StringValue())
"%s requests support one builder query, one formula over them, one clickhouse query, or one promql query, got %q", r.RequestType.StringValue(), envelope.Type.StringValue())
}
}
@@ -1046,6 +1057,48 @@ func validateBucketCountingQuery(requestType RequestType, functions []Function,
return nil
}
// validateEventHeatmapQuery holds a logs or traces query to what a per-event
// heatmap can draw. A disabled query feeds a formula rather than the chart, so
// it aggregates as a time series.
func validateEventHeatmapQuery(requestType RequestType, functions []Function, having *Having, expressions []string, disabled bool) error {
if err := validateBucketCountingQuery(requestType, functions, having); err != nil {
return err
}
if disabled {
return nil
}
// Every aggregation carries its own Meta.Buckets, and a heatmap renders
// against a single bucket axis.
if len(expressions) != 1 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"a heatmap renders one bucket axis, but the query has %d aggregations", len(expressions)).
WithAdditional("Keep the one aggregation whose distribution you want to plot")
}
if _, ok := HeatmapAggregationField(expressions[0]); !ok {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"a logs or traces heatmap counts events by the field named as its aggregation, got %q", expressions[0]).
WithAdditional("Use an aggregation such as duration_nano")
}
return nil
}
func aggregationExpressions[T LogAggregation | TraceAggregation](aggregations []T) []string {
expressions := make([]string, len(aggregations))
for i, aggregation := range aggregations {
switch a := any(aggregation).(type) {
case LogAggregation:
expressions[i] = a.Expression
case TraceAggregation:
expressions[i] = a.Expression
}
}
return expressions
}
func GetValidationOptions(requestType RequestType) []ValidationOption {
switch requestType {
case RequestTypeTimeSeries, RequestTypeHeatmap, RequestTypeDistribution:

View File

@@ -730,6 +730,44 @@ def build_scalar_query(
return {"type": "builder_query", "spec": spec}
def build_heatmap_query(
name: str,
signal: str,
expression: str,
*,
source: str | None = None,
group_by: list[dict] | None = None,
order: list[dict] | None = None,
filter_expression: str | None = None,
bucket_options: dict | None = None,
step_interval: int = DEFAULT_STEP_INTERVAL,
) -> dict:
"""Omitting bucket_options leaves the server's default log axis."""
spec: dict[str, Any] = {
"name": name,
"signal": signal,
"stepInterval": step_interval,
"aggregations": [{"expression": expression}],
}
if source:
spec["source"] = source
if group_by:
spec["groupBy"] = group_by
if order:
spec["order"] = order
if filter_expression:
spec["filter"] = {"expression": filter_expression}
if bucket_options is not None:
spec["bucketOptions"] = bucket_options
return {"type": "builder_query", "spec": spec}
def build_traces_scalar_query(
aggregations: list[dict],
*,

View File

@@ -0,0 +1,83 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.audit import AuditLog
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import (
RequestType,
assert_distribution_sums_heatmap,
build_group_by_field,
build_heatmap_query,
build_linear_bucket_options,
get_all_series,
get_buckets,
index_series_by_label,
make_query_request,
)
def test_audit_heatmap(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_audit_logs: Callable[[list[AuditLog]], None],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
changed_by_action = {"update": [3, 8], "delete": [25]}
insert_audit_logs(
[
AuditLog(
timestamp=now - timedelta(seconds=index + 1),
resources={"service.name": "signoz", "signoz.audit.resource.kind": "dashboard"},
attributes={"signoz.audit.action": action, "signoz.audit.fields_changed": fields_changed},
event_name=f"dashboard.{action}",
)
for action, values in changed_by_action.items()
for index, fields_changed in enumerate(values)
]
# an event without the field is not counted
+ [
AuditLog(
timestamp=now - timedelta(seconds=5),
resources={"service.name": "signoz", "signoz.audit.resource.kind": "dashboard"},
attributes={"signoz.audit.action": "update"},
event_name="dashboard.update",
)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int((now + timedelta(seconds=5)).timestamp() * 1000)
queries = [
build_heatmap_query(
"A",
"logs",
"signoz.audit.fields_changed",
source="audit",
group_by=[build_group_by_field("signoz.audit.action", "string", "attribute")],
filter_expression="signoz.audit.resource.kind = 'dashboard'",
bucket_options=build_linear_bucket_options(20, 4),
)
]
response = make_query_request(signoz, token, start_ms, end_ms, queries, request_type=RequestType.HEATMAP)
assert response.status_code == HTTPStatus.OK, response.text
assert_distribution_sums_heatmap(signoz, token, start_ms, end_ms, queries, response)
data = response.json()
# 20 split into 4 gives (0, 5], (5, 10], (10, 15], (15, 20]; the axis stops
# at the highest bucket reached, and 25 is past 20 so it counts in the
# trailing overflow
buckets = get_buckets(data, "A")
assert buckets == pytest.approx([0.0, 5.0, 10.0])
series_by_action = index_series_by_label(get_all_series(data, "A"), "signoz.audit.action")
totals_by_action = {action: [sum(column["values"][slot] for column in series["values"]) for slot in range(len(buckets) + 1)] for action, series in series_by_action.items()}
assert totals_by_action == {
"update": [0, 1, 1, 0],
"delete": [0, 0, 0, 1],
}

View File

@@ -12,16 +12,6 @@ def test_create_logs_pipeline_success(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
Create a new log parsing pipeline with a simple regex parser.
Tests:
1. Create a pipeline with valid configuration
2. Verify the pipeline is created successfully
3. Verify the response contains version information
4. Verify the pipeline is returned in the response
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
pipeline_payload = {
@@ -99,15 +89,6 @@ def test_list_logs_pipelines_success(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
Create a pipeline first, then list all pipelines.
Tests:
1. Create a pipeline
2. List all pipelines and verify the created pipeline is present
3. Verify the response structure
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a pipeline first
@@ -188,16 +169,6 @@ def test_list_logs_pipelines_by_version(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
Create multiple pipeline versions and list by specific version.
Tests:
1. Create initial pipeline (version 1)
2. Create updated pipeline (version 2)
3. List pipelines by version 1 and verify it returns the original
4. List pipelines by version 2 and verify it returns the updated version
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create version 1
@@ -350,14 +321,6 @@ def test_preview_logs_pipelines_success(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
Preview a json_parser pipeline with one JSON log and one plain-text log.
Tests:
1. JSON body gets parsed into attributes
2. Non-JSON body passes through unchanged instead of being dropped
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
empty_log_fields = {
@@ -454,15 +417,6 @@ def test_create_multiple_pipelines_success(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
Create multiple pipelines in a single request.
Tests:
1. Create multiple pipelines with different configurations
2. Verify all pipelines are created
3. Verify pipelines are ordered correctly
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
multi_pipeline_payload = {
@@ -568,16 +522,6 @@ def test_delete_all_pipelines_success(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
Create a pipeline first, then delete all pipelines by sending empty array.
Tests:
1. Create a pipeline
2. Delete all pipelines by sending empty pipelines array
3. Verify pipelines are deleted
4. Verify new version is created
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a pipeline first

View File

@@ -40,15 +40,8 @@ def test_json_body_unknown_key_collision_groupby(
insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None],
) -> None:
"""
Setup:
6 logs carrying `field` in different physical places: 2 as a string attribute
("sa"), 1 as a number attribute (10), 1 as a bool attribute (True), 2 in the body
JSON ("bs"). `field` is stripped from attribute metadata, so it is unknown
everywhere and takes the synthesize path.
Tests:
Grouping by the unknown `field` walks the whole multiIf (attribute maps then body)
and buckets every record by its value, no matter which place holds it.
`field` is stripped from the attribute metadata, so every place it lives is
reached through the synthesized multiIf, attribute maps then body.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs = [
@@ -102,14 +95,8 @@ def test_json_body_unknown_key_collision_filter(
expected_rids: set[str],
) -> None:
"""
Setup:
The attribute/body collision dataset, each record tagged with a unique `rid`
resource so the filter result is checked by exact record identity, not just count.
Tests:
Filtering the unknown `field` resolves attributes OR the body JSON path and selects
exactly the expected records: a typed operand hits its attribute map, a body value
is matched via the body branch, and `exists` finds all six records.
Each record carries a unique `rid` resource, so the result is checked by record
identity, not by count.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs = [
@@ -144,14 +131,8 @@ def test_json_body_unknown_key_attribute_parity(
insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None],
) -> None:
"""
Setup:
4 logs carrying the same value under known_key (in metadata) and unknown_key
(stripped), string / number / bool across records — no body values this time.
Tests:
Even with use_json_body on (the synthesized multiIf carries a trailing body
branch), grouping by the unknown attribute key still matches grouping by the known
one: the extra body branch does not perturb attribute-map resolution.
With use_json_body on, the synthesized multiIf carries a trailing body branch.
It must not change how the attribute maps resolve.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
specs = [("log-a", "alpha"), ("log-b", 42), ("log-c", 42), ("log-d", True)]
@@ -197,16 +178,6 @@ def test_json_body_unknown_key_aggregation_over_body(
get_token: Callable[[str, str], str],
insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None],
) -> None:
"""
Setup:
The same attribute/body collision dataset — `field` carried as a string / number /
bool attribute and in the body JSON, one place per record.
Tests:
Aggregating over the unknown `field` walks the whole multiIf including the body
branch: count() sees every record (all 6) and count_distinct() counts the distinct
stringified values across attribute maps AND the body ("sa", "10", "true", "bs").
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs = [
Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": SERVICE}, attributes={"field": "sa"}, body="attr-str-0"),
@@ -250,17 +221,8 @@ def test_json_body_unknown_key_body_path_parity(
prefix: str,
) -> None:
"""
Setup:
Logs whose body carries the same value under two paths — known_field and
unknown_field — but only known_field is exported to the body path metadata
(distributed_field_keys); unknown_field is left unregistered.
Tests:
Grouping, counting distinct values, and selecting by the registered body path
(known_field, metadata-driven) and by the unregistered one (unknown_field,
fallback body path) return identical results, for the bare key and the
body.-prefixed key — the body counterpart of the attribute parity in
13_unknown_key_collisions.py.
known_field is registered in the body path metadata and unknown_field is not,
so one resolves through metadata and the other through the fallback body path.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs = []

View File

@@ -35,13 +35,8 @@ def test_fields_keys_by_context(
absent: list[str],
) -> None:
"""
Setup:
Insert a log with a code.file attribute and a span with an http.method attribute and a host.name resource.
Tests:
1. Keys for a context list that context's intrinsic columns and the stored keys the context maps to,
each with its context; intrinsics of other contexts are not listed. The span context also keeps
listing attributes because `span.<attribute>` resolves attributes in queries.
The span context also lists attributes, because `span.<attribute>` resolves
attributes in queries.
"""
now = datetime.now(tz=UTC)
insert_logs([Logs(timestamp=now, attributes={"code.file": "/opt/integration.go"}, body="a log line")])
@@ -83,8 +78,7 @@ def test_fields_keys_by_data_type(
absent: list[str],
) -> None:
"""
Tests:
1. A data type filter keeps the intrinsic columns of that type; number, int64 and float64 are one family.
number, int64 and float64 are one family.
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
@@ -117,11 +111,6 @@ def test_fields_keys_search_matches_intrinsics_case_insensitively(
search_text: str,
present: list[str],
) -> None:
"""
Tests:
1. The search text matches intrinsic columns case-insensitively, as it does for stored keys,
with or without a context prefix.
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
@@ -158,14 +147,8 @@ def test_fields_values_bool_fields(
expected: list[bool],
) -> None:
"""
Setup:
Insert a log with a bool attribute.
Tests:
1. Values for a bool field are true and false (narrowed by the search text): for the calculated span
field, for a stored bool attribute whose tag rows carry no value, and for a key the caller
declares bool.
2. A span scope selector (isRoot) only takes true.
The tag rows of a bool attribute carry no value, so true and false come from
the type, not from the data.
"""
insert_logs([Logs(timestamp=datetime.now(tz=UTC), attributes={"retry": True}, body="retrying")])
@@ -189,16 +172,9 @@ def test_fields_values_start_excludes_span_values_not_seen_since_the_day(
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert a span three days old and a span now, with different service names.
Tests:
1. Values with startUnixMilli an hour ago contain only the service seen today: the start is
floored to the day, the tag table's deduplication unit.
2. Values without a start contain both services.
Logs are not covered: the logs collector stamps tag rows with the ingestion hour, not the
log's timestamp, and the fixture writes the log's timestamp.
The start is floored to the day, the tag table's deduplication unit.
Logs are not covered: the logs collector stamps tag rows with the ingestion
hour, and the fixture writes the log's timestamp.
"""
signal = "traces"
now = datetime.now(tz=UTC)

View File

@@ -20,19 +20,6 @@ def test_logs_list(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert 2 logs with different attributes
Tests:
1. Query logs for the last 10 seconds and check if the logs are returned in the correct order
2. Query values of severity_text attribute from the autocomplete API
3. Query values of severity_text attribute from the fields API
4. Query values of code.file attribute from the autocomplete API
5. Query values of code.file attribute from the fields API
6. Query values of code.line attribute from the autocomplete API
7. Query values of code.line attribute from the fields API
"""
insert_logs(
[
Logs(
@@ -410,10 +397,6 @@ def test_logs_list_with_order_by(
order_by_context: str,
expected_order: list[str],
) -> None:
"""
Setup:
Insert 3 logs with service.name in attributes and resources
"""
attribute_resource_pair = [
[{"id": "log-001", "service.name": "c"}, {}],

View File

@@ -19,17 +19,6 @@ def test_logs_time_series_count(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert 17 logs with service.name attribute set to "java" and severity_text attribute set to "DEBUG", 23 logs with service.name attribute set to "erlang" and severity_text attribute set to "ERROR", 29 logs with service.name attribute set to "go" and severity_text attribute set to "WARNING".
All logs have incrementing code.line attribute, modulo 2 for host.name and cloud.account.id.
Tests:
1. count() of all logs for the last 5 minutes
2. count() of all logs where code.line = 7 for last 5 minutes
3. count() of all logs where service.name = "erlang" OR cloud.account.id = "000" for last 5 minutes
4. count() of all logs grouped by host.name for the last 5 minutes
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs: list[Logs] = []
@@ -441,16 +430,6 @@ def test_datatype_collision(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert logs with data type collision scenarios to test DataTypeCollisionHandledFieldName function
Tests:
1. severity_number comparison with string value
2. http.status_code with mixed string/number values
3. response.time with string values in numeric field
4. Edge cases: empty strings
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs: list[Logs] = []

View File

@@ -18,17 +18,6 @@ def test_logs_json_body_simple_searches(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert logs with JSON bodies containing simple key-value pairs
Tests:
1. Search by body.message Contains "value"
2. Search by body.status = 200 (numeric)
3. Search by body.active = true (boolean)
4. Search by body.level = "error" with CONTAINS
5. Search by body.code > 100 (comparison)
"""
now = datetime.now(tz=UTC)
# Log with simple JSON body
@@ -305,16 +294,6 @@ def test_logs_json_body_nested_keys(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert logs with JSON bodies containing nested objects
Tests:
1. Search by body.user.name = "value"
2. Search by body.request.secure = true (boolean)
3. Search by body.response.latency = 123.45 (floating point)
4. Search by body.response.status.code = 200
"""
now = datetime.now(tz=UTC)
log1_body = json.dumps(
@@ -571,17 +550,6 @@ def test_logs_json_body_listing(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert multiple logs with JSON bodies
Tests:
1. List all logs with JSON bodies
2. List logs with pagination (limit and offset)
3. List logs ordered by timestamp
4. List logs with multiple filters combined (AND/OR)
5. Count logs matching JSON body filters
"""
now = datetime.now(tz=UTC)
logs_data = [
@@ -935,7 +903,9 @@ def test_logs_json_body_in_operator(
expression: str,
expected_services: set[str],
) -> None:
"""IN over a body JSON path fans out to one comparison per value."""
"""
IN over a body JSON path fans out to one comparison per value.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
specs = [("auth", 200, ["alpha", "beta"]), ("payment", 500, ["gamma"]), ("search", 404, ["beta", "alpha"])]
insert_logs(

View File

@@ -427,12 +427,6 @@ def test_logs_list_query_timestamp_expectations(
query: BuilderQuery,
result: Callable[[list[Logs]], list[Any]],
) -> None:
"""
Setup:
Insert logs with corrupt data
Tests:
"""
logs = generate_logs_with_corrupt_metadata()
insert_logs(logs)
@@ -723,15 +717,8 @@ def test_logs_list_query_trace_id_expectations(
results: Callable[[list[Logs]], list[Any]],
) -> None:
"""
Justification for expected rows and ordering:
Four logs differ by where trace_id is set: top-level (x[0]), attribute (x[1]),
resource (x[2]), or only in body text (x[3]). Each parametrized case documents
which column AdjustKeys/field mapping reads and why DESC ties break in storage order.
Setup:
Insert logs with corrupt trace_id
Tests:
The four logs differ by where trace_id is set: top level, attribute, resource,
or only in the body text. DESC ties break in storage order.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)

View File

@@ -66,15 +66,6 @@ def test_logs_filter_operators(
expression: str,
expected_bodies: set[str],
) -> None:
"""
Setup:
Five logs across services / environments / methods, each body naming itself,
with numeric `resp_code` / `latency_ms` attributes and an `error.type`
attribute present on one.
Tests:
Each production-shaped filter operator selects exactly the expected logs.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# (body, service, deployment.environment, http.request.method, resp_code, latency_ms, has_error_type)
specs = [

View File

@@ -37,17 +37,6 @@ def test_logs_aggregate_functions(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
svc-a: 3 logs (latency_ms 10/20/30, endpoints /x /x /y); svc-b: 1 log
(latency_ms 40, endpoint /z).
Tests:
A grouped scalar query computes count / sum / avg / min / max / p50 / p90 /
p95 / p99 over latency_ms, countIf over a numeric threshold, rate and
rate_sum over the query window, and count_distinct over a string attribute —
all matching values derived from the inserted logs, ordered by count() desc.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# A scalar rate divides by the whole query window, so both sides agree on it.
lookback_minutes = 5
@@ -134,16 +123,6 @@ def test_logs_aggregate_if_predicates_use_filter_syntax(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
svc-a: 3 logs, two with a k8s.pod.name and an x-request-id header, two
with "timeout" in the body; svc-b: 1 log without either, with "timeout".
Tests:
countIf predicates accept the filter-only operators EXISTS, NOT EXISTS,
CONTAINS and NOT CONTAINS, a combination of them, and a key with hyphens;
two predicates in one expression each keep their own values.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# (service, k8s.pod.name, x-request-id header, body)
specs = [
@@ -209,16 +188,6 @@ def test_logs_aggregate_arguments_are_field_names(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
svc-a: 3 logs, svc-b: 1 log, each with a body size under a key with a
hyphen and an app label under a key with slashes.
Tests:
Aggregation arguments are read as field names, so keys with hyphens,
slashes and a type suffix resolve, count(*) counts every log, and DISTINCT
keeps its meaning.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# (service, http.response.body-size, k8s.pod.labels.app.kubernetes.io/name)
specs = [
@@ -311,15 +280,6 @@ def test_logs_aggregate_having(
aggregations: list[str],
expected_services: set[str],
) -> None:
"""
Setup:
svc-a: 3 logs (latency_ms 10/20/30, sum 60); svc-b: 1 log (latency_ms 40).
Tests:
A grouped scalar query with `having` keeps only the qualifying groups, for a
count() predicate (both directions) and a sum() predicate — extending logs
HAVING coverage beyond the single count() case in querierscalar/04_having.py.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
specs = [("svc-a", 10), ("svc-a", 20), ("svc-a", 30), ("svc-b", 40)]
logs = [

View File

@@ -34,14 +34,6 @@ def test_logs_aggregate_min_max(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
svc-a: 3 logs (latency_ms 10/20/30); svc-b: 1 log (latency_ms 40).
Tests:
max(latency_ms) / min(latency_ms) as two aggregations in one time_series query
grouped by service.name return each service's latency extremes.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
specs = [("svc-a", 10), ("svc-a", 20), ("svc-a", 30), ("svc-b", 40)]
logs = [
@@ -104,15 +96,6 @@ def test_logs_aggregate_time_series_limit(
limit: int,
direction: str,
) -> None:
"""
Setup:
Four services with distinct log counts (a=5, b=3, c=7, d=1).
Tests:
A time_series group-by with a limit returns only the N series that the ordered
aggregation keeps — top-N for desc, bottom-N for asc — each series summing to
that service's log count.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
counts = {"svc-a": 5, "svc-b": 3, "svc-c": 7, "svc-d": 1}
logs = [
@@ -163,15 +146,6 @@ def test_logs_aggregate_multi_group_by(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Logs across (service.name, deployment.environment) pairs:
(svc-a, prod)=2, (svc-a, dev)=1, (svc-b, prod)=3.
Tests:
A scalar query grouped by two resource keys returns one row per present pair
(the cross product, not the full Cartesian) with the correct per-pair count.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
pair_counts = {("svc-a", "prod"): 2, ("svc-a", "dev"): 1, ("svc-b", "prod"): 3}
logs = [

View File

@@ -62,14 +62,8 @@ def test_logs_unknown_key_collision_groupby_parity(
insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None],
) -> None:
"""
Setup:
8 logs sharing a value under known_key (in metadata) and unknown_key (stripped
from metadata); the value is a string / number / bool depending on the record, so
the shared value spans attributes_string / attributes_number / attributes_bool.
Tests:
Grouping by unknown_key (synthesized multiIf) yields the same buckets and counts
as grouping by known_key (metadata-driven), across all three attribute maps.
The shared value is a string, a number or a bool depending on the record, so
it spans all three attribute maps.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs = [
@@ -130,16 +124,6 @@ def test_logs_unknown_key_collision_filter_parity(
predicate: str,
expected_bodies: set[str],
) -> None:
"""
Setup:
The same collision dataset as the group-by parity test.
Tests:
Filtering unknown_key (synthesized) selects the same records as filtering known_key
(metadata-driven) across operators: typed equality (the operand type picks the
matching attribute map), `exists` (fans out across all three maps), and a
mixed-type `IN` list (string + number operands union their maps).
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs = [
Logs(
@@ -180,15 +164,6 @@ def test_logs_unknown_key_collision_aggregation_parity(
get_token: Callable[[str, str], str],
insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None],
) -> None:
"""
Setup:
The same collision dataset as the group-by parity test.
Tests:
Aggregating over unknown_key (synthesized) matches aggregating over known_key
(metadata-driven): count() sees every record (all 8 carry the key across the three
attribute maps) and count_distinct() sees the 6 distinct stringified values.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs = [
Logs(

View File

@@ -38,14 +38,6 @@ def test_logs_list_select_field_types(
expected: Any,
expected_type: type,
) -> None:
"""
Setup:
One log carrying typed attributes (string / number / bool) and severity ERROR.
Tests:
Projecting a string / number / bool attribute or the severity_text intrinsic
returns the value with the right Python type under its short-name key.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
[
@@ -93,15 +85,6 @@ def test_logs_list_select_projects_columns(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
One log with a string attribute plus body and resources.
Tests:
Projecting a single field returns it as a top-level key and drops the default
row payload — the resources_string / attributes_* maps and body are no longer
present when a selectFields list is given.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
[
@@ -153,17 +136,9 @@ def test_logs_list_unknown_log_context_synthesizes(
surface: str,
) -> None:
"""
Setup:
One log.
Tests:
The intrinsic `log.` context is forgiving, like bare / `attribute.` /
`resource.` keys (and the traces `span.` context): an unknown key there
synthesizes against the log attribute maps (and the body path) rather than
erroring, because existence is a property of the data, not of metadata. A
filter reference matches nothing and surfaces a not-found warning (under the
stripped name); a select projects a null column; an order clause succeeds
and returns the log.
An unknown key under the `log.` context synthesizes against the attribute maps
and the body path instead of erroring: existence is a property of the data,
not of the metadata.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
service = "log-forgiving-ctx-svc"

View File

@@ -0,0 +1,207 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import (
RequestType,
assert_distribution_sums_heatmap,
build_aggregation,
build_group_by_field,
build_heatmap_query,
build_linear_bucket_options,
build_log_bucket_options,
build_order_by,
build_scalar_query,
get_all_series,
get_buckets,
get_error_message,
get_heatmap_columns,
index_series_by_label,
make_query_request,
)
def test_linear_heatmap_grouped(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
bytes_by_service = {"heatmap-svc-a": [50, 250], "heatmap-svc-b": [950, 5000]}
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=index + 1),
resources={"service.name": service},
attributes={"response_bytes": response_bytes},
body=f"{service} served {response_bytes} bytes",
)
for service, values in bytes_by_service.items()
for index, response_bytes in enumerate(values)
]
# a log without the field is not counted
+ [
Logs(
timestamp=now - timedelta(seconds=index + 1),
resources={"service.name": "heatmap-svc-a"},
body="no response size",
)
for index in range(2)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int((now + timedelta(seconds=5)).timestamp() * 1000)
queries = [
build_heatmap_query(
"A",
"logs",
"response_bytes",
group_by=[build_group_by_field("service.name", "string", "resource")],
filter_expression="service.name IN ('heatmap-svc-a', 'heatmap-svc-b')",
bucket_options=build_linear_bucket_options(1000, 10),
)
]
response = make_query_request(signoz, token, start_ms, end_ms, queries, request_type=RequestType.HEATMAP)
assert response.status_code == HTTPStatus.OK, response.text
assert_distribution_sums_heatmap(signoz, token, start_ms, end_ms, queries, response)
data = response.json()
# 1000 split into 10 gives (0, 100], (100, 200], ... (900, 1000], and 5000
# is past 1000 so it counts in the trailing overflow
buckets = get_buckets(data, "A")
assert buckets == pytest.approx([0.0, 100.0, 200.0, 300.0, 400.0, 500.0, 600.0, 700.0, 800.0, 900.0, 1000.0])
series_by_service = index_series_by_label(get_all_series(data, "A"), "service.name")
assert set(series_by_service) == set(bytes_by_service)
totals_by_service = {service: [sum(column["values"][slot] for column in series["values"]) for slot in range(len(buckets) + 1)] for service, series in series_by_service.items()}
assert totals_by_service == {
"heatmap-svc-a": [0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
"heatmap-svc-b": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],
}
def test_log_heatmap(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=index + 1),
resources={"service.name": "heatmap-log-axis"},
attributes={"queue_depth": queue_depth},
body=f"queue depth {queue_depth}",
)
for index, queue_depth in enumerate([100, 300, 600])
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int((now + timedelta(seconds=5)).timestamp() * 1000)
queries = [
build_heatmap_query(
"A",
"logs",
"attribute.queue_depth",
order=[build_order_by("attribute.queue_depth")],
filter_expression="service.name = 'heatmap-log-axis'",
bucket_options=build_log_bucket_options(0),
)
]
response = make_query_request(signoz, token, start_ms, end_ms, queries, request_type=RequestType.HEATMAP)
assert response.status_code == HTTPStatus.OK, response.text
assert_distribution_sums_heatmap(signoz, token, start_ms, end_ms, queries, response)
data = response.json()
# a doubling per bucket, with one rung under the lowest bucket reached so
# its lower bound is on the axis; (128, 256] holds nothing but stays
buckets = get_buckets(data, "A")
assert buckets == pytest.approx([64.0, 128.0, 256.0, 512.0, 1024.0])
assert [sum(column["values"][slot] for column in get_heatmap_columns(data, "A")) for slot in range(len(buckets) + 1)] == [0, 1, 0, 1, 1, 0]
def test_default_axis(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=index + 1),
resources={"service.name": "heatmap-default-axis"},
attributes={"batch_size": 1000},
body="batch of 1000",
)
for index in range(2)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int((now + timedelta(seconds=5)).timestamp() * 1000)
queries = [build_heatmap_query("A", "logs", "batch_size", filter_expression="service.name = 'heatmap-default-axis'")]
response = make_query_request(signoz, token, start_ms, end_ms, queries, request_type=RequestType.HEATMAP)
assert response.status_code == HTTPStatus.OK, response.text
assert_distribution_sums_heatmap(signoz, token, start_ms, end_ms, queries, response)
data = response.json()
# no bucketOptions is the finest log axis, 16 buckets per doubling: 1000
# sits in (2^(159/16), 2^10]
buckets = get_buckets(data, "A")
totals = [sum(column["values"][slot] for column in get_heatmap_columns(data, "A")) for slot in range(len(buckets) + 1)]
holding = [slot for slot, total in enumerate(totals) if total]
assert holding == [len(buckets) - 1]
assert totals[holding[0]] == 2
assert buckets[-2:] == pytest.approx([2 ** (159 / 16), 1024.0])
@pytest.mark.parametrize(
"aggregations, expected_message",
[
pytest.param(
[build_aggregation("count()")],
'a logs or traces heatmap counts events by the field named as its aggregation, got "count()"',
id="count_aggregation",
),
pytest.param(
[build_aggregation("response_bytes"), build_aggregation("queue_depth")],
"a heatmap renders one bucket axis, but the query has 2 aggregations",
id="two_aggregations",
),
],
)
def test_heatmap_rejections(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
aggregations: list[dict],
expected_message: str,
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_query_request(
signoz,
token,
int((now - timedelta(minutes=5)).timestamp() * 1000),
int(now.timestamp() * 1000),
[build_scalar_query("A", "logs", aggregations)],
request_type=RequestType.HEATMAP,
)
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
assert expected_message in get_error_message(response.json())

View File

@@ -40,14 +40,6 @@ def test_metrics_scalar_space_aggregations(
space_aggregation: str,
expected: float,
) -> None:
"""
Setup:
Four gauge series with latest values 50 / 30 / 70 / 10.
Tests:
An ungrouped scalar query combines the four series with the given space
aggregation: sum=160, avg=40, min=10, max=70, count=4.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_metrics([Metrics(metric_name="test.metric", labels={"service.name": service}, timestamp=now - timedelta(seconds=1), temporality="Unspecified", type_="Gauge", is_monotonic=False, value=value) for service, value in METRIC_VALUES.items()])
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
@@ -81,14 +73,6 @@ def test_metrics_scalar_having(
having_expression: str,
expected_services: set[str],
) -> None:
"""
Setup:
Four gauge series (per service.name) with latest values 50 / 30 / 70 / 10.
Tests:
A grouped metrics scalar query with `having` on the space-aggregated value keeps
only the qualifying services — HAVING was previously logs-only.
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_metrics([Metrics(metric_name="test.metric", labels={"service.name": service}, timestamp=now - timedelta(seconds=1), temporality="Unspecified", type_="Gauge", is_monotonic=False, value=value) for service, value in METRIC_VALUES.items()])
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)

View File

@@ -49,20 +49,6 @@ def test_traces_list(
insert_traces: Callable[[list[Traces]], None],
noise: str,
) -> None:
"""
Setup:
Insert 4 spans across two services.
http-service: POST /integration -> SELECT, HTTP PATCH
topic-service: topic publish
Tests:
1. A raw list ordered by timestamp desc returns the spans in order with the
projected intrinsic / calculated / resource columns (field context and
datatype specified inline in the key name).
2. isRoot = 'true' returns only the two root spans.
Under the corrupt variant the projected columns still resolve to the real
intrinsic/calculated/resource values (never the colliding attributes).
"""
extra_attrs, extra_resources = trace_noise(noise)
http_service_trace_id = TraceIdGenerator.trace_id()
http_service_span_id = TraceIdGenerator.span_id()
@@ -227,17 +213,8 @@ def test_traces_list_select_fields(
events_links: str,
) -> None:
"""
Setup:
Insert a root span with no events/links and a child span carrying two
events and one user-supplied link.
Tests:
1. Empty selectFields returns all fields, and the `events` / `links` columns
arrive parsed into structured objects (events carry `attributes`, links
carry only `traceId`/`spanId` — refType is dropped at the consume layer).
2. A non-empty selectField returns that field along with timestamp, trace_id
and span_id.
The returned key set is invariant to the corrupt-variant collisions.
Events arrive parsed, and links carry only traceId and spanId: refType is
dropped at the consume layer.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
@@ -358,17 +335,6 @@ def test_traces_list_select_field_types(
expected: Callable[[Traces], Any],
expected_type: type,
) -> None:
"""
Setup:
Insert one span carrying typed attributes and db attributes.
Tests:
Projecting a string / number / bool span attribute, a calculated column, an
intrinsic column and its deprecated alias returns the value (derived from
the inserted span) with the right Python type; a bare non-existent attribute
projects as null. Under the corrupt variant the same-named colliding
attributes must not shadow the intrinsic/calculated columns.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
span = Traces(
@@ -417,13 +383,6 @@ def test_traces_list_select_field_dedup(
insert_traces: Callable[[list[Traces]], None],
noise: str,
) -> None:
"""
Setup:
Insert one span.
Tests:
Selecting the same field twice collapses to a single key in the response.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
span = Traces(
@@ -478,16 +437,8 @@ def test_traces_list_select_calculated_field_collision(
expected: Callable[[Traces], Any],
) -> None:
"""
Setup:
Insert one span whose derived calculated columns (db_name from db.name,
http_method from http.request.method) collide with literal attributes of the
same name.
Tests:
Selecting a calculated field by its bare name or `span.` context returns the
derived column value (the same-named attribute is shadowed); the explicit
`attribute.` prefix reaches the colliding attribute value instead. Every form
projects under the calculated field's name.
A bare name or the `span.` context returns the derived column and shadows the
same-named attribute; the `attribute.` prefix reaches the attribute.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
span = Traces(
@@ -528,19 +479,6 @@ def test_traces_list_response_shape_values(
insert_traces: Callable[[list[Traces]], None],
noise: str,
) -> None:
"""
Setup:
Insert a curated set of spans exercising every derived/intrinsic response
field: an error span, a client span with HTTP url/method, a db span, a
host span, and remote/kind variants.
Tests:
With empty selectFields every intrinsic/calculated column (has_error,
is_remote, external_http_url, db_name, http_host, status_code_string,
kind_string, ...) round-trips with its populated value, and the merged
`attributes` map preserves string / number / bool types. Under the corrupt
variant these columns keep winning over the same-named colliding attributes.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
@@ -662,17 +600,6 @@ def test_traces_list_ordering(
sort_value: Callable[[dict[str, Any]], Any],
direction: str,
) -> None:
"""
Setup:
Insert 4 spans whose fields are permuted so that each order key produces a
distinct ordering.
Tests:
Ordering by an intrinsic (timestamp, duration_nano, name, kind), a
calculated column (response_status_code), a resource attribute and a number
span attribute all sort correctly in both directions — including when
same-named corrupt attributes collide with those columns.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
@@ -733,14 +660,6 @@ def test_traces_list_order_multi_key_tiebreak(
insert_traces: Callable[[list[Traces]], None],
noise: str,
) -> None:
"""
Setup:
Insert 3 spans sharing the same timestamp with distinct durations.
Tests:
A secondary order key (duration_nano) breaks the timestamp tie
deterministically, and flipping its direction flips the tie-break.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
@@ -795,12 +714,7 @@ def test_traces_list_no_default_order(
noise: str,
) -> None:
"""
Setup:
Insert 4 spans.
Tests:
A raw list query with no order clause succeeds and returns every matching
span (the builder adds no default ORDER BY, so only membership is pinned).
The builder adds no default ORDER BY, so only membership is pinned.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
@@ -862,14 +776,6 @@ def test_traces_list_pagination(
limit: int | None,
offset: int,
) -> None:
"""
Setup:
Insert 5 spans with strictly increasing timestamps.
Tests:
Ordered by timestamp desc, limit caps the row count and offset pages
through the result; an offset past the end returns no rows.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
@@ -931,13 +837,8 @@ def test_traces_list_time_boundaries(
noise: str,
) -> None:
"""
Setup:
Insert spans exactly at the query start, in the middle, and exactly at the
query end (all second-aligned so the ms->ns conversion lands exactly).
Tests:
The query window is [start, end): the span at start is included and the
span at end is excluded.
The spans are second-aligned, so the ms to ns conversion lands exactly on the
window edges.
"""
extra_attrs, extra_resources = trace_noise(noise)
start_dt = datetime.now(tz=UTC).replace(microsecond=0) - timedelta(seconds=30)
@@ -978,14 +879,6 @@ def test_traces_list_empty_result(
insert_traces: Callable[[list[Traces]], None],
noise: str,
) -> None:
"""
Setup:
Insert one span.
Tests:
A filter matching no span, and a time window that excludes the span, both
return zero rows (rows is null, surfaced as an empty list).
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_traces(
@@ -1045,14 +938,7 @@ def test_traces_list_span_scope(
noise: str,
) -> None:
"""
Setup:
Insert a root span, a non-root entry-point span (its (name, service) is
seeded into top_level_operations) and a non-root internal span.
Tests:
1. isRoot = 'true' returns only the root span.
2. isEntryPoint = 'true' returns only the non-root span whose operation is
registered in top_level_operations.
isEntryPoint reads top_level_operations, so the entry-point span is seeded there.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(microsecond=0)
@@ -1262,17 +1148,6 @@ def test_traces_list_with_corrupt_data(
status_code: HTTPStatus,
expected: Callable[[list[Traces]], dict[str, Any]],
) -> None:
"""
Setup:
Insert 4 spans whose metadata carries intrinsic/calculated field names as
attributes (e.g. attribute.timestamp, attribute.duration_nano).
Tests:
Ordering / selecting a name that exists both as an intrinsic column and as
an attribute resolves to the intrinsic column, while the explicit
`attribute.` prefix reaches the attribute value. Each expected row is derived
from the inserted span objects.
"""
traces = generate_traces_with_corrupt_metadata()
insert_traces(traces)
# traces[i] occurred before traces[j] where i < j.
@@ -1346,24 +1221,10 @@ def test_traces_list_with_scope_filter(
expected_indices: list[int],
) -> None:
"""
Setup three spans with different scope key resolution:
- x[0]: scope.name/version 'io.signoz.checkout'/'2.3.1'; span attribute
env.tier='gold'.
- x[1]: scope.name/version 'io.signoz.payment'/'4.5.6'; scope attributes
telemetry.sdk.language='python', env.tier='gold', and a `name` scope
attribute colliding with x[0]'s scope.name value.
- x[2]: span name 'io.signoz.checkout' (colliding with x[0]'s scope.name
value) and a span attribute literally named `scope.name`.
Tests:
- Filtering on scope.name / scope.version / a scope attribute.
- An unprefixed key is resolved across contexts (scope checked alongside
attribute / intrinsic), while a `scope.`-prefixed key is scope-only.
- `scope.name`/`scope.version` name every home they resolve to: the declared JSON
sub-column and a same-named `name`/`version` scope attribute. The explicit
`scope.attribute.` prefix addresses the attribute alone.
- a bare `name` reaches the span `name` column and a `name` scope attribute, but
never the declared scope.name field.
`scope.name` and `scope.version` resolve to the declared JSON sub-column and to
a same-named scope attribute; `scope.attribute.` addresses the attribute alone.
A bare `name` reaches the span name column and a `name` scope attribute, never
the declared scope.name.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
@@ -1465,18 +1326,8 @@ def test_traces_list_unknown_span_context_synthesizes(
surface: str,
) -> None:
"""
Setup:
Insert one span.
Tests:
The intrinsic `span.` context is forgiving, like bare / `attribute.` /
`resource.` keys (and the logs `log.` context): an unknown key there
synthesizes against the span attribute maps rather than erroring, because
existence is a property of the data, not of metadata. A filter reference
matches nothing and surfaces a not-found warning (under the stripped name);
a select projects a null column; an order clause succeeds and returns the
span. (See test_traces_list_unknown_other_context_synthesizes for the
`attribute.`/`resource.` contexts.)
An unknown key under the `span.` context synthesizes against the attribute maps
instead of erroring: existence is a property of the data, not of the metadata.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
span = Traces(
@@ -1535,17 +1386,6 @@ def test_traces_list_unknown_other_context_synthesizes(
context: str,
surface: str,
) -> None:
"""
Setup:
Insert 3 spans.
Tests:
Like the intrinsic `span.` context (see
test_traces_list_unknown_span_context_synthesizes), the `attribute.`
and `resource.` contexts synthesize an unknown key (existence is a property
of the data, not metadata): a filter reference matches nothing, a select
projects a null column, and an order clause succeeds and returns every span.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
spans = [
@@ -1601,15 +1441,6 @@ def test_traces_list_order_unknown_key_synthesizes(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert 3 spans.
Tests:
Ordering by a bare (context-less) unknown key synthesizes a null column and
succeeds, returning every matching span — the ordering counterpart of the
bare-select-null case and the unknown-filter synthesis in 09_unknown_keys.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
spans = [

View File

@@ -77,14 +77,7 @@ def test_traces_aggregate_order_by_count(
expected_status: HTTPStatus,
) -> None:
"""
Setup:
Insert 4 spans of a single service.
Tests:
An aggregation can be referenced in `order by` by its expression (count()),
its `span.`-prefixed expression, or its alias — with or without an explicit
fieldContext. Only the combinations that double-specify the context are a bad
request; the rest resolve and return the span count.
Only the forms that specify the context twice are a bad request.
"""
use_attribute_backend(attribute_backend)
@@ -149,20 +142,6 @@ def test_traces_aggregate_functions(
attribute_backend: str,
noise: str,
) -> None:
"""
Setup:
svc-a: 3 spans (durations 1s/2s/3s; one ERROR; one 4XX via
http.response.status_code); svc-b: 1 span (duration 4s). A number attribute
`latency_ms` rides along on every span.
Tests:
A grouped scalar query computes count / sum / avg / min / max / p50 / p90 over
duration_nano, countIf over the intrinsic status_code and the calculated
response_status_code, rate and rate_sum over the query window, and avg over a
numeric attribute — all matching values derived from the inserted spans. Under
the corrupt variant the same-named colliding attributes must not change any of
these.
"""
use_attribute_backend(attribute_backend)
extra_attrs, extra_resources = trace_noise(noise)
@@ -256,14 +235,6 @@ def test_traces_aggregate_calculated_range_countif(
use_attribute_backend: Callable[[str], None],
attribute_backend: str,
) -> None:
"""
Setup:
3 spans whose derived response_status_code is 200 / 503 / 404.
Tests:
countIf(response_status_code >= 400 AND response_status_code < 500) counts the
single 4XX span (404).
"""
use_attribute_backend(attribute_backend)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
@@ -311,14 +282,6 @@ def test_traces_aggregate_having(
attribute_backend: str,
noise: str,
) -> None:
"""
Setup:
svc-a: 3 spans, svc-b: 1 span.
Tests:
A grouped scalar query with `having count() > 2` returns only the groups
whose count qualifies (svc-a), derived from the inserted spans.
"""
use_attribute_backend(attribute_backend)
extra_attrs, extra_resources = trace_noise(noise)
@@ -385,16 +348,6 @@ def test_traces_aggregate_time_series_limit(
limit: int,
direction: str,
) -> None:
"""
Setup:
Four services with distinct span counts (a=5, b=3, c=7, d=1).
Tests:
A time_series group-by with a limit returns only the N series that the
ordered aggregation keeps — top-N for desc, bottom-N for asc — each series
summing to that service's span count. Under the corrupt variant the grouping
(resource.service.name) and count are unaffected by the colliding attributes.
"""
use_attribute_backend(attribute_backend)
extra_attrs, extra_resources = trace_noise(noise)
@@ -468,14 +421,6 @@ def test_traces_aggregate_unknown_key(
expression: str,
expected_value: Any,
) -> None:
"""
Setup:
Insert 3 spans.
Tests:
Aggregating a bare unknown key synthesizes a null column: count() of it is 0
(no non-null values) and sum()/avg() are null — the query never errors.
"""
use_attribute_backend(attribute_backend)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)

View File

@@ -39,14 +39,6 @@ def test_traces_fill_plain(
noise: str,
fill_mode: str,
) -> None:
"""
Setup:
One span 3 minutes ago.
Tests:
fillGaps / fillZero over count() with no group by leave the single populated
bucket at the span count and every other bucket in the window at 0.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
span = Traces(
@@ -93,14 +85,6 @@ def test_traces_fill_group_by(
noise: str,
fill_mode: str,
) -> None:
"""
Setup:
service-a span 3 minutes ago, service-b span 2 minutes ago.
Tests:
fillGaps / fillZero over count() grouped by service.name yield one series per
service, each with its single populated bucket and zeros elsewhere.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
span_a = Traces(
@@ -167,14 +151,6 @@ def test_traces_fill_formula(
noise: str,
fill_mode: str,
) -> None:
"""
Setup:
'test' service span 3 minutes ago, 'another-test' service span 2 minutes ago.
Tests:
A formula F1 = A + B over two disabled per-service count() queries fills empty
buckets to 0, so F1 carries a 1 in each service's populated bucket.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
span_test = Traces(
@@ -237,15 +213,6 @@ def test_traces_fill_formula_group_by(
noise: str,
fill_mode: str,
) -> None:
"""
Setup:
group1 span 3 minutes ago, group2 span 2 minutes ago.
Tests:
A formula F1 = A + B over two disabled group-by count() queries yields one
series per service, each carrying A+B (=2) in its populated bucket and zeros
elsewhere.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
span_g1 = Traces(

View File

@@ -34,15 +34,8 @@ def test_traces_list_filter_by_trace_id(
noise: str,
) -> None:
"""
Setup:
A target trace (one root span) and an unrelated trace.
Tests filtering a raw list by trace_id:
1. A narrow window (single bucket) returns just the target span.
2. A wide window (>1 h, multiple exponential buckets) returns it exactly once
(no duplicate-span regression).
3. A window that does not contain the trace returns nothing and warns that it
lies outside the selected time range.
A window wider than an hour spans several exponential buckets, so the span must
come back exactly once.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
@@ -130,14 +123,8 @@ def test_traces_aggregation_filter_by_trace_id(
noise: str,
) -> None:
"""
Setup:
A target trace with two spans (root + child).
Tests the trace_id time-range optimization for aggregation (time_series):
1. A wide window containing the trace returns the correct count (both spans).
2. A window outside the trace short-circuits to empty and warns.
3. A trace_id with no row in trace_summary short-circuits to empty with no
outside-range warning (trace_summary is authoritative).
trace_summary is authoritative: a trace_id with no row there short-circuits to
empty without the outside-range warning.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)

View File

@@ -587,10 +587,9 @@ def test_trace_operator_with_adjusted_keys(
assert_result: Callable[[requests.Response, list[Traces]], None],
) -> None:
"""
Trace operators build a CTE per referenced builder query and an outer
query on top. Both layers need the same key adjustment as regular trace
queries, otherwise deprecated keys and context-prefixed aliases don't
resolve.
A trace operator builds a CTE per referenced query and an outer query on top.
Both layers need the same key adjustment as a regular trace query, or deprecated
keys and context-prefixed aliases do not resolve.
"""
use_attribute_backend(attribute_backend)
@@ -648,24 +647,8 @@ def test_trace_operator_select_fields(
expansion: str,
) -> None:
"""
Setup:
Insert a parent (operation.type = 'parent') with one event and one
user-supplied link, plus a child span (operation.type = 'child').
Tests:
1. With selectFields=[], the `A => B` trace_operator returns every column
in ALL_SELECT_FIELDS, mirroring the builder_query path. Events arrive
as {name, timeUnixNano, attributes} and links as {traceId, spanId}
with refType dropped at the consume layer.
2. With an explicit selectFields=[{"name": "service.name"}], only the 6
hardcoded core columns plus service.name come back — no auto-expansion
to the full set.
See:
- pkg/telemetrytraces/trace_operator_cte_builder.go::buildFinalQuery for
the expansion gate.
- pkg/telemetrytraces/trace_operator_cte_builder.go::buildListQuery for
the per-row SELECT.
An explicit selectFields returns only the core columns plus the named fields,
with no expansion to the full set.
"""
use_attribute_backend(attribute_backend)
@@ -795,14 +778,6 @@ def test_trace_operator_scalar_limit(
use_attribute_backend: Callable[[str], None],
attribute_backend: str,
) -> None:
"""
Setup:
Insert 4, 3, 2 and 1 spans for services limit-a..limit-d.
Tests:
1. A scalar trace operator grouped by service.name with limit=2 returns
only the top two groups by count.
"""
use_attribute_backend(attribute_backend)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)

View File

@@ -30,14 +30,6 @@ def test_traces_aggregate_min_max(
insert_traces: Callable[[list[Traces]], None],
noise: str,
) -> None:
"""
Setup:
http-service: 3 spans (3s / 0.5s / 1s); topic-service: 1 span (4s).
Tests:
max(duration_nano) / min(duration_nano) grouped by service.name return each
service's duration extremes, derived from the inserted spans.
"""
extra_attrs, extra_resources = trace_noise(noise)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
specs = [

View File

@@ -75,14 +75,6 @@ def test_traces_filter_operators(
expression: str,
expected_names: set[str],
) -> None:
"""
Setup:
Five spans across services / environments / methods / statuses, with a
numeric `latency_ms` attribute and an `error.type` attribute present on one.
Tests:
Each production-shaped filter operator selects exactly the expected spans.
"""
use_attribute_backend(attribute_backend)
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)

View File

@@ -38,16 +38,10 @@ def test_traces_aggregate_percentiles(
noise: str,
) -> None:
"""
Setup:
5 spans with durations 1s..5s.
Tests:
p25 / p50 / p90 / p95 / p99 over duration_nano match numpy's linear-interpolated
percentiles (ClickHouse quantile() uses the same interpolation for small inputs).
Under the "collision" variant a numeric span attribute named duration_nano unions
with the intrinsic column into a multiIf; the aggregation must still resolve to the
intrinsic column rather than error with ClickHouse NO_COMMON_TYPE (386) — the
regression behind the span_percentile 500.
The percentiles match numpy's linear interpolation, which ClickHouse quantile()
uses for small inputs. Under the collision variant a numeric attribute named
duration_nano unions with the intrinsic column into a multiIf; the aggregation
must resolve to the column, not fail with NO_COMMON_TYPE.
"""
use_attribute_backend(attribute_backend)
@@ -107,15 +101,6 @@ def test_traces_aggregate_group_by_non_resource(
group_key: str,
expected: dict[str, int],
) -> None:
"""
Setup:
3 spans: (name op-a, endpoint /a) x2 and (name op-b, endpoint /b) x1.
Tests:
Grouping by an intrinsic column (name) or a string span attribute (endpoint) —
not just a resource key — buckets by the real value; under the corrupt variant a
same-named colliding attribute must not divert the grouping.
"""
use_attribute_backend(attribute_backend)
extra_attrs, extra_resources = trace_noise(noise)
@@ -161,15 +146,6 @@ def test_traces_aggregate_multi_group_by(
attribute_backend: str,
noise: str,
) -> None:
"""
Setup:
Spans across (service.name, name) pairs: (svc-1, op-a) x2, (svc-1, op-b) x1,
(svc-2, op-a) x1.
Tests:
Grouping by two keys (a resource key and an intrinsic column) returns one row per
present pair with the correct per-pair count.
"""
use_attribute_backend(attribute_backend)
extra_attrs, extra_resources = trace_noise(noise)
@@ -230,15 +206,6 @@ def test_traces_aggregate_having_breadth(
having_expression: str,
expected_services: set[str],
) -> None:
"""
Setup:
svc-a: 3 spans of 1s (sum 3s, count 3); svc-b: 1 span of 5s (sum 5s, count 1).
Tests:
HAVING on a sum() aggregation, on count(), and a compound count()+sum() predicate
each keep only the qualifying groups — extending the count()-only HAVING in
02_aggregation.py.
"""
use_attribute_backend(attribute_backend)
extra_attrs, extra_resources = trace_noise(noise)
@@ -293,15 +260,8 @@ def test_traces_duration_nano_qol_filter_with_collision(
duration_filter: str,
) -> None:
"""
Setup:
5 spans (durations 1s..5s) that also carry a same-named `duration_nano` span
attribute (numeric on some, string on others).
Tests the duration_nano QoL filter — as a duration string ('2s') and as raw
nanoseconds — resolves to the intrinsic column despite the collision: it parses the
duration, filters on the real durations (3 spans exceed 2s), and doesn't error with
ClickHouse NO_COMMON_TYPE (386). The string attribute is cast; the intrinsic column
stays a bare comparison.
The duration string is parsed against the intrinsic column; the colliding
string attribute is cast, so there is no NO_COMMON_TYPE.
"""
use_attribute_backend(attribute_backend)

View File

@@ -68,12 +68,8 @@ def test_timestamp_order_aggregations_return_the_instant(
bound: str,
) -> None:
"""
Setup:
Two spans for one service, 3 and 1 minutes ago.
Tests:
max/min answer with the extreme span's instant. Coerced to Float64 the value would be
~1.7e9 seconds, which reads as January 1970 when a consumer treats it as milliseconds.
Coerced to Float64 the value would be about 1.7e9 seconds, which reads as
January 1970 when a consumer takes it as milliseconds.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
oldest, newest = now - timedelta(minutes=3), now - timedelta(minutes=1)
@@ -98,9 +94,7 @@ def test_timestamp_quantile_returns_an_instant_within_range(
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Tests:
ClickHouse computes quantiles over DateTime64 natively, so p99 lands inside the
seeded range rather than erroring or collapsing to epoch.
ClickHouse computes quantiles over DateTime64 natively.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
oldest, newest = now - timedelta(minutes=3), now - timedelta(minutes=1)
@@ -125,11 +119,6 @@ def test_timestamp_counting_aggregations_are_numeric(
expression: str,
expected: int | None,
) -> None:
"""
Tests:
count and rate count rows rather than reading the column's value, so they stay
numeric for a time column.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
seed_two_spans(insert_traces, now - timedelta(minutes=3), now - timedelta(minutes=1))
@@ -165,11 +154,6 @@ def test_timestamp_arithmetic_aggregations_are_unsupported(
insert_traces: Callable[[list[Traces]], None],
expression: str,
) -> None:
"""
Tests:
Aggregations that cannot produce a time from a time column fail rather than
answering with a rescaled number.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
seed_two_spans(insert_traces, now - timedelta(minutes=3), now - timedelta(minutes=1))

View File

@@ -0,0 +1,113 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import (
RequestType,
assert_distribution_sums_heatmap,
build_heatmap_query,
build_linear_bucket_options,
build_log_bucket_options,
get_buckets,
get_heatmap_columns,
make_query_request,
)
from fixtures.traces import TraceIdGenerator, Traces, TracesKind
def test_duration_heatmap(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
durations = [timedelta(0)] + [timedelta(milliseconds=5)] * 6 + [timedelta(seconds=1)] * 3 + [timedelta(seconds=30)]
insert_traces(
[
Traces(
timestamp=now - timedelta(minutes=2, seconds=index),
duration=duration,
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
name="GET /cart",
kind=TracesKind.SPAN_KIND_SERVER,
resources={"service.name": "heatmap-cart"},
)
for index, duration in enumerate(durations)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(now.timestamp() * 1000)
queries = [build_heatmap_query("A", "traces", "duration_nano", filter_expression="service.name = 'heatmap-cart'", bucket_options=build_log_bucket_options(0))]
response = make_query_request(signoz, token, start_ms, end_ms, queries, request_type=RequestType.HEATMAP)
assert response.status_code == HTTPStatus.OK, response.text
assert_distribution_sums_heatmap(signoz, token, start_ms, end_ms, queries, response)
data = response.json()
# a doubling per bucket in nanoseconds: 5ms is in (2^22, 2^23], 1s in
# (2^29, 2^30] and 30s in (2^34, 2^35], and 2^22 is the rung under 5ms; a
# 0ns span is counted at 0
buckets = get_buckets(data, "A")
assert buckets == pytest.approx([0.0] + [float(2**power) for power in range(22, 36)])
totals = [sum(column["values"][slot] for column in get_heatmap_columns(data, "A")) for slot in range(len(buckets) + 1)]
assert {buckets[slot]: total for slot, total in enumerate(totals) if total} == {0.0: 1, 2.0**23: 6, 2.0**30: 3, 2.0**35: 1}
assert sum(totals) == len(durations)
def test_span_attribute_heatmap(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
rows_returned = [5, 5, 5, 45, 45, 250]
insert_traces(
[
Traces(
timestamp=now - timedelta(minutes=2, seconds=index),
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
name="SELECT orders",
kind=TracesKind.SPAN_KIND_CLIENT,
resources={"service.name": "heatmap-orders-db"},
attributes={"db.rows_returned": rows},
)
for index, rows in enumerate(rows_returned)
]
# a span without the attribute is not counted
+ [
Traces(
timestamp=now - timedelta(minutes=2, seconds=index),
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
name="BEGIN",
kind=TracesKind.SPAN_KIND_CLIENT,
resources={"service.name": "heatmap-orders-db"},
)
for index in range(2)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(now.timestamp() * 1000)
queries = [build_heatmap_query("A", "traces", "db.rows_returned", filter_expression="service.name = 'heatmap-orders-db'", bucket_options=build_linear_bucket_options(100, 10))]
response = make_query_request(signoz, token, start_ms, end_ms, queries, request_type=RequestType.HEATMAP)
assert response.status_code == HTTPStatus.OK, response.text
assert_distribution_sums_heatmap(signoz, token, start_ms, end_ms, queries, response)
data = response.json()
# 100 split into 10 gives (0, 10], (10, 20], ... (90, 100]: 5 is in the
# first, 45 in (40, 50], and 250 is past 100 so it counts in the overflow
buckets = get_buckets(data, "A")
assert buckets == pytest.approx([0.0, 10.0, 20.0, 30.0, 40.0, 50.0])
assert [sum(column["values"][slot] for column in get_heatmap_columns(data, "A")) for slot in range(len(buckets) + 1)] == [0, 3, 0, 0, 0, 2, 1]

View File

@@ -18,10 +18,6 @@ def test_export_raw_data_get_not_allowed(
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
1. GET request to export_raw_data is rejected with 405 Method Not Allowed
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
@@ -41,16 +37,6 @@ def test_export_logs_csv(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert 3 logs with different severity levels and attributes.
Tests:
1. Export logs as CSV format
2. Verify CSV structure and content
3. Validate headers are present
4. Check log data is correctly formatted
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
@@ -154,16 +140,6 @@ def test_export_logs_jsonl(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert 2 logs with different attributes.
Tests:
1. Export logs as JSONL format
2. Verify JSONL structure and content
3. Check each line is valid JSON
4. Validate log data is correctly formatted
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
@@ -250,14 +226,6 @@ def test_export_logs_with_filter(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert logs with different severity levels.
Tests:
1. Export logs with filter applied
2. Verify only filtered logs are returned
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
@@ -335,14 +303,6 @@ def test_export_logs_with_limit(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert 5 logs.
Tests:
1. Export logs with limit applied
2. Verify only limited number of logs are returned
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
logs = []
@@ -403,14 +363,6 @@ def test_export_logs_with_columns(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert logs with various attributes.
Tests:
1. Export logs with specific columns
2. Verify only specified columns are returned
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
@@ -489,14 +441,6 @@ def test_export_logs_with_order_by(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert logs at different timestamps.
Tests:
1. Export logs with ascending timestamp order
2. Verify logs are returned in correct order
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(
@@ -580,14 +524,6 @@ def test_export_logs_with_complex_filter(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""
Setup:
Insert logs with various service names and severity levels.
Tests:
1. Export logs with complex filter (multiple conditions)
2. Verify only logs matching all conditions are returned
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_logs(

View File

@@ -24,10 +24,6 @@ def test_export_raw_data_get_not_allowed(
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
1. GET request to export_raw_data is rejected with 405 Method Not Allowed
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
@@ -47,16 +43,6 @@ def test_export_traces_csv(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert 3 traces with different attributes.
Tests:
1. Export traces as CSV format
2. Verify CSV structure and content
3. Validate headers are present
4. Check trace data is correctly formatted
"""
http_service_trace_id = TraceIdGenerator.trace_id()
http_service_span_id = TraceIdGenerator.span_id()
http_service_db_span_id = TraceIdGenerator.span_id()
@@ -190,16 +176,6 @@ def test_export_traces_jsonl(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert 2 traces with different attributes.
Tests:
1. Export traces as JSONL format
2. Verify JSONL structure and content
3. Check each line is valid JSON
4. Validate trace data is correctly formatted
"""
http_service_trace_id = TraceIdGenerator.trace_id()
http_service_span_id = TraceIdGenerator.span_id()
topic_service_trace_id = TraceIdGenerator.trace_id()
@@ -307,14 +283,6 @@ def test_export_traces_with_filter(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert traces with different service names.
Tests:
1. Export traces with filter applied
2. Verify only filtered traces are returned
"""
service_a_trace_id = TraceIdGenerator.trace_id()
service_a_span_id = TraceIdGenerator.span_id()
service_b_trace_id = TraceIdGenerator.trace_id()
@@ -406,14 +374,6 @@ def test_export_traces_with_limit(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert 5 traces.
Tests:
1. Export traces with limit applied
2. Verify only limited number of traces are returned
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
traces = []
@@ -477,11 +437,6 @@ def test_export_traces_multiple_queries_rejected(
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
1. POST with multiple builder queries but no trace operator is rejected
2. Verify 400 error is returned
"""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
@@ -528,14 +483,6 @@ def test_export_traces_with_composite_query_trace_operator(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert multiple traces with parent-child relationships.
Tests:
1. Export traces using trace operator in composite query (POST)
2. Verify trace operator query works correctly
"""
parent_trace_id = TraceIdGenerator.trace_id()
parent_span_id = TraceIdGenerator.span_id()
child_span_id_1 = TraceIdGenerator.span_id()
@@ -667,14 +614,6 @@ def test_export_traces_with_select_fields(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Insert traces with various attributes.
Tests:
1. Export traces with specific select fields via POST
2. Verify only specified fields are returned in the output
"""
trace_id = TraceIdGenerator.trace_id()
span_id = TraceIdGenerator.span_id()

View File

@@ -14,22 +14,6 @@ def test_create_groups_and_simulate_with_backfill(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
Create a mapping group and a mapper through the CRUD API, then hit the
simulate endpoint with two groups: the saved one referenced by name only
(server backfills its mappers from the store) and a second, unsaved group
sent with inline mappers.
Tests:
1. Create a group (POST /span_mapper_groups) and read back its id
2. Create a mapper in that group (POST .../{groupId}/span_mappers)
3. List groups and mappers and verify what we created is persisted
4. Simulate spans against both groups: a matching span gets the saved
group's mappers (backfilled, copy) and the inline group's mappers
(move) applied, while a span matching no condition passes through
5. Delete the group and verify it (and its mappers) are gone
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
create_group = requests.post(

View File

@@ -15,19 +15,8 @@ def test_default_groups_are_seeded_and_shipped_items_are_toggle_only(
get_token: Callable[[str, str], str],
) -> None:
"""
Setup:
A fresh org. The reconciler seeds the shipped mapping groups at startup
and on org creation, so nothing has to be created here.
Tests:
1. The list contains llm, agent and tool as system groups with shipped
substrings
2. Shipped mappers and their sources are system-owned and enabled
3. A shipped name cannot be taken by a user group, and system groups and
mappers cannot be deleted
4. A shipped source can be switched off and a user override added; both
round-trip through PATCH and the simulator honours them
5. A shipped substring can be switched off and a user one added
The reconciler seeds the shipped groups at startup and on org creation, so
nothing is created here.
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
headers = {"authorization": f"Bearer {token}", "content-type": "application/json"}

View File

@@ -30,13 +30,8 @@ def test_domain_list_last_seen_is_the_span_instant(
noise: str,
) -> None:
"""
Setup:
Two client spans to api.stripe.com, 3 and 1 minutes ago.
Tests:
Last Seen resolves to the newer span's instant (#5824). A numeric coercion of the
DateTime64 column would yield seconds since epoch (~1.7e9), which lands in
January 1970 once read as milliseconds.
A numeric coercion of the DateTime64 column would give seconds since epoch,
which reads as January 1970 once taken as milliseconds.
"""
extra_attrs = {"timestamp": 1.0} if noise == "timestamp_attr" else {}
now = datetime.now(tz=UTC).replace(microsecond=0)
@@ -92,12 +87,7 @@ def test_domain_list_groups_by_domain_and_counts_endpoints(
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
api.github.com with 2 distinct URLs (one called twice); api.openai.com with 1.
Tests:
One row per domain, endpoint counts are distinct-URL counts, and the columns
used only to feed the error-rate formula are stripped from the response.
The columns that only feed the error-rate formula are stripped from the response.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
specs = [
@@ -162,14 +152,6 @@ def test_domain_list_excludes_non_client_and_urlless_spans(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
A client span with an http.url, a server span with one, and a client span without.
Tests:
The base filter (`http_url EXISTS AND kind_string = 'Client'`) keeps only the
first domain; the other two never reach the response.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
specs = [
("included.example.com", TracesKind.SPAN_KIND_CLIENT, "https://included.example.com/a"),
@@ -221,11 +203,7 @@ def test_domain_list_show_ip_masks_ip_domains(
show_ip: bool,
) -> None:
"""
Setup:
One client span each to a hostname, a bare IPv4 and a bare IPv6 address.
Tests:
show_ip gates both IP forms (net.ParseIP accepts either); the hostname always shows.
net.ParseIP accepts both IP forms, so show_ip gates IPv4 and IPv6 alike.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
specs = [
@@ -273,12 +251,7 @@ def test_domain_list_latency_and_rate_values(
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
3 client spans of 250ms on one domain, 1 on another.
Tests:
p99 is duration_nano in nanoseconds, and rps is rate() over the scalar window
(count / (end - start) seconds), so the busier domain's rate is 3x the other's.
rps is rate() over the scalar window, so the busier domain's rate is 3x the other's.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
busy, quiet = "busy.example.com", "quiet.example.com"
@@ -334,14 +307,6 @@ def test_domain_list_error_rate_is_a_percentage_of_spans(
total: int,
expected: float,
) -> None:
"""
Setup:
`total` client spans on one domain, `errors` of them failed.
Tests:
error_rate is (error/total_span)*100 across the range, including the all-failed and
none-failed edges.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
domain = f"errors-{label}.example.com"
statuses = [TracesStatusCode.STATUS_CODE_ERROR] * errors + [TracesStatusCode.STATUS_CODE_OK] * (total - errors)
@@ -384,14 +349,6 @@ def test_domain_list_user_filter_narrows_the_base_filter(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Two client spans to the same domain from different services.
Tests:
A caller filter is ANDed onto the base filter, so only the named service's span
contributes.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
domain = "filtered.example.com"
insert_traces(
@@ -434,14 +391,6 @@ def test_domain_list_caller_group_by_adds_a_dimension(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
Setup:
Two client spans to one domain, from two services.
Tests:
A caller groupBy is merged after the domain key, so the domain splits into one row
per group value and the extra key gets its own column.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
domain = "grouped.example.com"
insert_traces(
@@ -491,10 +440,6 @@ def test_domain_list_is_empty_outside_the_seeded_window(
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
A window with no matching spans answers 200 with no rows, not an error.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
long_ago = now - timedelta(days=400)
response = make_third_party_apis_request(
@@ -509,10 +454,6 @@ def test_domain_list_is_empty_outside_the_seeded_window(
def test_domain_list_requires_authentication(signoz: types.SigNoz) -> None:
"""
Tests:
The route sits behind ViewAccess, so an unauthenticated call is rejected.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
response = make_third_party_apis_request(
signoz,
@@ -529,11 +470,6 @@ def test_domain_list_rejects_inverted_time_range(
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
The request is validated before any query is built, so start >= end is a client
error rather than an empty result.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
response = make_third_party_apis_request(
signoz,