Compare commits

...

2 Commits

Author SHA1 Message Date
Naman Verma
00efffc127 fix: make ResolveHeatmapBucketing a method on MetricAggregation 2026-09-03 13:50:09 +05:30
Naman Verma
dec922a83f feat: add heatmap support in query and dashboards 2026-09-03 12:07:27 +05:30
28 changed files with 3953 additions and 123 deletions

View File

@@ -3045,6 +3045,58 @@ components:
- tags
- spec
type: object
DashboardtypesHeatmapColorMode:
enum:
- scheme
- opacity
type: string
DashboardtypesHeatmapColorScale:
enum:
- log
- sqrt
- linear
type: string
DashboardtypesHeatmapColors:
properties:
fill:
type: string
max:
nullable: true
type: number
min:
nullable: true
type: number
mode:
$ref: '#/components/schemas/DashboardtypesHeatmapColorMode'
reverse:
type: boolean
scale:
$ref: '#/components/schemas/DashboardtypesHeatmapColorScale'
scheme:
type: string
steps:
type: integer
type: object
DashboardtypesHeatmapPanelSpec:
properties:
colors:
$ref: '#/components/schemas/DashboardtypesHeatmapColors'
formatting:
$ref: '#/components/schemas/DashboardtypesPanelFormatting'
legend:
$ref: '#/components/schemas/DashboardtypesLegend'
showOverflow:
type: boolean
visualization:
$ref: '#/components/schemas/DashboardtypesHeatmapVisualization'
type: object
DashboardtypesHeatmapVisualization:
properties:
showVisualMap:
type: boolean
timePreference:
$ref: '#/components/schemas/DashboardtypesTimePreference'
type: object
DashboardtypesHistogramBuckets:
properties:
bucketCount:
@@ -3397,6 +3449,7 @@ components:
discriminator:
mapping:
signoz/BarChartPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesBarChartPanelSpec'
signoz/HeatmapPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec'
signoz/HistogramPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec'
signoz/ListPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec'
signoz/NumberPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesNumberPanelSpec'
@@ -3412,6 +3465,7 @@ components:
- $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTablePanelSpec'
- $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec'
- $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec'
- $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec'
type: object
DashboardtypesPanelPluginKind:
enum:
@@ -3422,6 +3476,7 @@ components:
- signoz/TablePanel
- signoz/HistogramPanel
- signoz/ListPanel
- signoz/HeatmapPanel
type: string
DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesBarChartPanelSpec:
properties:
@@ -3435,6 +3490,18 @@ components:
- kind
- spec
type: object
DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec:
properties:
kind:
enum:
- signoz/HeatmapPanel
type: string
spec:
$ref: '#/components/schemas/DashboardtypesHeatmapPanelSpec'
required:
- kind
- spec
type: object
DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec:
properties:
kind:
@@ -6950,10 +7017,7 @@ components:
$ref: '#/components/schemas/Querybuildertypesv5TimeSeries'
type: array
meta:
properties:
unit:
type: string
type: object
$ref: '#/components/schemas/Querybuildertypesv5AggregationMeta'
predictedSeries:
items:
$ref: '#/components/schemas/Querybuildertypesv5TimeSeries'
@@ -6968,12 +7032,51 @@ components:
$ref: '#/components/schemas/Querybuildertypesv5TimeSeries'
type: array
type: object
Querybuildertypesv5Bucket:
Querybuildertypesv5AggregationMeta:
properties:
step:
format: double
type: number
buckets:
items:
format: double
type: number
type: array
unit:
type: string
type: object
Querybuildertypesv5BucketOptions:
discriminator:
mapping:
linear: '#/components/schemas/Querybuildertypesv5BucketOptionsLinear'
log: '#/components/schemas/Querybuildertypesv5BucketOptionsLog'
propertyName: kind
oneOf:
- $ref: '#/components/schemas/Querybuildertypesv5BucketOptionsLinear'
- $ref: '#/components/schemas/Querybuildertypesv5BucketOptionsLog'
type: object
Querybuildertypesv5BucketOptionsLinear:
properties:
kind:
$ref: '#/components/schemas/Querybuildertypesv5BucketsKind'
spec:
$ref: '#/components/schemas/Querybuildertypesv5LinearBucketsSpec'
required:
- kind
- spec
type: object
Querybuildertypesv5BucketOptionsLog:
properties:
kind:
$ref: '#/components/schemas/Querybuildertypesv5BucketsKind'
spec:
$ref: '#/components/schemas/Querybuildertypesv5LogBucketsSpec'
required:
- kind
- spec
type: object
Querybuildertypesv5BucketsKind:
enum:
- linear
- log
type: string
Querybuildertypesv5BuilderQuerySpec:
discriminator:
mapping:
@@ -7154,6 +7257,16 @@ components:
value:
type: string
type: object
Querybuildertypesv5LinearBucketsSpec:
properties:
maxValue:
format: double
type: number
numBuckets:
type: integer
required:
- maxValue
type: object
Querybuildertypesv5LogAggregation:
properties:
alias:
@@ -7161,6 +7274,12 @@ components:
expression:
type: string
type: object
Querybuildertypesv5LogBucketsSpec:
properties:
scale:
nullable: true
type: integer
type: object
Querybuildertypesv5MetricAggregation:
properties:
comparisonSpaceAggregationParam:
@@ -7619,6 +7738,8 @@ components:
queries (traces, logs, metrics), formulas, joins, trace operators, PromQL,
and ClickHouse SQL queries.
properties:
bucketOptions:
$ref: '#/components/schemas/Querybuildertypesv5BucketOptions'
compositeQuery:
$ref: '#/components/schemas/Querybuildertypesv5CompositeQuery'
end:
@@ -7718,6 +7839,7 @@ components:
- raw
- raw_stream
- trace
- heatmap
type: string
Querybuildertypesv5ScalarData:
properties:
@@ -7792,8 +7914,6 @@ components:
type: object
Querybuildertypesv5TimeSeriesValue:
properties:
bucket:
$ref: '#/components/schemas/Querybuildertypesv5Bucket'
partial:
type: boolean
timestamp:

View File

@@ -451,7 +451,7 @@ func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.Cach
// Merge values based on type
var mergedValue any
switch resultType {
case qbtypes.RequestTypeTimeSeries:
case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
mergedValue = bc.mergeTimeSeriesValues(ctx, buckets)
// Raw and Scalar types are not cached, so no merge needed
}
@@ -476,14 +476,34 @@ func (bc *bucketCache) mergeTimeSeriesValues(ctx context.Context, buckets []*qbt
}
seriesMap := make(map[seriesKey]*qbtypes.TimeSeries, estimatedSeries)
decoded := make([]*qbtypes.TimeSeriesData, 0, len(buckets))
newestOf := map[int]*qbtypes.AggregationBucket{}
newestStartOf := map[int]uint64{}
for _, bucket := range buckets {
var tsData *qbtypes.TimeSeriesData
if err := json.Unmarshal(bucket.Value, &tsData); err != nil {
bc.logger.ErrorContext(ctx, "failed to unmarshal time series data", errors.Attr(err))
continue
}
decoded = append(decoded, tsData)
// The buckets are not guaranteed to arrive in order here, and Alias and
// Unit are taken from the most recent one, so track that explicitly
// rather than relying on iteration order.
for _, aggBucket := range tsData.Aggregations {
if _, seen := newestOf[aggBucket.Index]; !seen || bucket.StartMs >= newestStartOf[aggBucket.Index] {
newestOf[aggBucket.Index] = aggBucket
newestStartOf[aggBucket.Index] = bucket.StartMs
}
}
}
mergedBoundaries := qbtypes.MergeHeatmapAxes(decoded...)
for _, tsData := range decoded {
for _, aggBucket := range tsData.Aggregations {
qbtypes.RealignHeatmapValues(aggBucket.Series, aggBucket.Meta.Buckets, mergedBoundaries[aggBucket.Index])
for _, series := range aggBucket.Series {
// Create series key from labels
key := seriesKey{
@@ -556,10 +576,18 @@ func (bc *bucketCache) mergeTimeSeriesValues(ctx context.Context, buckets []*qbt
}
}
result.Aggregations = append(result.Aggregations, &qbtypes.AggregationBucket{
aggBucket := &qbtypes.AggregationBucket{
Index: index,
Series: seriesList,
})
}
if newest, ok := newestOf[index]; ok {
aggBucket.Alias = newest.Alias
aggBucket.Meta = newest.Meta
}
if boundaries, ok := mergedBoundaries[index]; ok {
aggBucket.Meta.Buckets = boundaries
}
result.Aggregations = append(result.Aggregations, aggBucket)
}
return result
@@ -572,7 +600,7 @@ func (bc *bucketCache) isEmptyResult(result *qbtypes.Result) (isEmpty bool, isFi
}
switch result.Type {
case qbtypes.RequestTypeTimeSeries:
case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
// No aggregations at all means truly empty
if len(tsData.Aggregations) == 0 {
@@ -699,14 +727,19 @@ func (bc *bucketCache) trimResultToFluxBoundary(result *qbtypes.Result, fluxBoun
}
switch result.Type {
case qbtypes.RequestTypeTimeSeries:
case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
// Trim time series data
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok && tsData != nil {
trimmedData := &qbtypes.TimeSeriesData{}
for _, aggBucket := range tsData.Aggregations {
// Meta has to survive the trim: a heatmap's counts are
// positional against Meta.Buckets, so a cached bucket that
// lost its axis cannot be read back against anything.
trimmedBucket := &qbtypes.AggregationBucket{
Index: aggBucket.Index,
Alias: aggBucket.Alias,
Meta: aggBucket.Meta,
}
for _, series := range aggBucket.Series {
@@ -766,7 +799,7 @@ func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, startMs,
}
switch result.Type {
case qbtypes.RequestTypeTimeSeries:
case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
filteredData := &qbtypes.TimeSeriesData{
Aggregations: make([]*qbtypes.AggregationBucket, 0, len(tsData.Aggregations)),

View File

@@ -92,6 +92,10 @@ func (q *builderQuery[T]) Fingerprint() string {
// This needs to include all fields that affect the query results
parts := []string{q.queryType.StringValue()}
// A heatmap and a time series query can share every spec field and still
// return different rows, so the request type has to separate their entries
parts = append(parts, fmt.Sprintf("requestType=%s", q.kind.StringValue()))
// Add signal type
parts = append(parts, fmt.Sprintf("signal=%s", q.spec.Signal.StringValue()))
@@ -130,6 +134,9 @@ func (q *builderQuery[T]) Fingerprint() string {
}
part += ":" + route
}
if a.HeatmapBucketing != nil {
part += ":" + fingerprintHeatmapBucketing(*a.HeatmapBucketing)
}
aggParts = append(aggParts, part)
}
}
@@ -185,6 +192,16 @@ func (q *builderQuery[T]) Fingerprint() string {
return strings.Join(parts, "&")
}
// fingerprintHeatmapBucketing captures only what changes the rows ClickHouse
// returns, which is why LogBucketsSpec.Scale is absent: coarsening it happens in
// postprocessing, so every scale reads one cache entry.
func fingerprintHeatmapBucketing(b qbtypes.HeatmapBucketing) string {
if b.Kind == qbtypes.BucketsKindLinear {
return fmt.Sprintf("%s:%v:%d", b.Kind.StringValue(), b.MaxValue, b.NumBuckets)
}
return b.Kind.StringValue()
}
func fingerprintGroupByKey(gb qbtypes.GroupByKey) string {
return fingerprintFieldKey(gb.TelemetryFieldKey)
}
@@ -412,7 +429,7 @@ func (q *builderQuery[T]) narrowWindowByTraceID(ctx context.Context, fromMS, toM
func emptyResultFor(kind qbtypes.RequestType, queryName string) *qbtypes.Result {
var value any
switch kind {
case qbtypes.RequestTypeTimeSeries:
case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
value = &qbtypes.TimeSeriesData{QueryName: queryName}
case qbtypes.RequestTypeScalar:
value = &qbtypes.ScalarData{QueryName: queryName}
@@ -465,8 +482,9 @@ func (q *builderQuery[T]) executeWithContext(ctx context.Context, query string,
queryWindow := &qbtypes.TimeRange{From: q.fromMS, To: q.toMS}
kind := q.kind
// all metric queries are time series then reduced if required
if q.spec.Signal == telemetrytypes.SignalMetrics {
// all metric queries are time series then reduced if required, except
// heatmaps, whose statement returns a row per bucket rather than per point
if q.spec.Signal == telemetrytypes.SignalMetrics && kind != qbtypes.RequestTypeHeatmap {
kind = qbtypes.RequestTypeTimeSeries
}

View File

@@ -6,6 +6,7 @@ import (
"time"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
@@ -120,6 +121,169 @@ func TestBuilderQueryFingerprintQueryType(t *testing.T) {
assert.Empty(t, ai.Fingerprint())
}
func TestBuilderQueryFingerprintHeatmapBucketing(t *testing.T) {
coarseLogScale := 1
testCases := []struct {
description string
left *builderQuery[qbtypes.MetricAggregation]
right *builderQuery[qbtypes.MetricAggregation]
expectedEqual bool
}{
{
// ResolveBucketOptions pins LogScale to MaxLogScale whatever the
// caller asked for, so the two are indistinguishable here by design
description: "a coarser logScale reads the same cache entry",
left: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLog, LogScale: qbtypes.MaxLogScale, NumBuckets: qbtypes.DefaultNumBuckets},
}},
},
},
right: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLog, LogScale: qbtypes.MaxLogScale, NumBuckets: qbtypes.DefaultNumBuckets},
}},
},
},
expectedEqual: true,
},
{
description: "linear separates on maxValue",
left: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 500, NumBuckets: 25},
}},
},
},
right: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 800, NumBuckets: 25},
}},
},
},
expectedEqual: false,
},
{
description: "linear separates on numBuckets",
left: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 500, NumBuckets: 25},
}},
},
},
right: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 500, NumBuckets: 40},
}},
},
},
expectedEqual: false,
},
{
description: "linear and log are separate entries",
left: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 500, NumBuckets: 25},
}},
},
},
right: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLog, LogScale: qbtypes.MaxLogScale, NumBuckets: qbtypes.DefaultNumBuckets},
}},
},
},
expectedEqual: false,
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
if testCase.expectedEqual {
assert.Equal(t, testCase.left.Fingerprint(), testCase.right.Fingerprint())
return
}
assert.NotEqual(t, testCase.left.Fingerprint(), testCase.right.Fingerprint())
})
}
t.Run("a coarser scale never reaches the axis clickhouse builds", func(t *testing.T) {
finest := (&qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLog, Spec: qbtypes.LogBucketsSpec{}}).ResolveBucketOptions()
coarse := (&qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLog, Spec: qbtypes.LogBucketsSpec{Scale: &coarseLogScale}}).ResolveBucketOptions()
assert.Equal(t, finest, coarse)
})
t.Run("a histogram folds in no bucket options at all", func(t *testing.T) {
// resolveHeatmapBucketing leaves histograms nil, so bucketOptions sent
// alongside one must not fragment its cache
histogram := &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeHeatmap,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{{
MetricName: "signoz_latency",
Type: metrictypes.HistogramType,
}},
},
}
fingerprint := histogram.Fingerprint()
assert.NotContains(t, fingerprint, qbtypes.BucketsKindLog.StringValue())
assert.NotContains(t, fingerprint, qbtypes.BucketsKindLinear.StringValue())
})
}
func TestMakeBucketsOrder(t *testing.T) {
// Test that makeBuckets returns buckets in reverse chronological order by default
// Using milliseconds as input - need > 1 hour range to get multiple buckets

View File

@@ -31,6 +31,11 @@ var (
// written clickhouse query. The column alias indcate which value is
// to be considered as final result (or target).
legacyReservedColumnTargetAliases = []string{"__result", "__value", "result", "res", "value"}
// legacyHeatmapBucketColumn is the alias a user written clickhouse query can
// give its bucket boundary column, alongside the HeatmapBucketColumn the
// statement builder emits.
legacyHeatmapBucketColumn = "bucket"
)
// stripKeyAlias removes the __SELECT_KEY_<n>_ / __GROUP_BY_KEY_<n>_ prefix from a result
@@ -83,6 +88,8 @@ func consume(rows driver.Rows, kind qbtypes.RequestType, queryWindow *qbtypes.Ti
payload, err = readAsTimeSeries(rows, queryWindow, step, queryName)
case qbtypes.RequestTypeScalar:
payload, err = readAsScalar(rows, queryName)
case qbtypes.RequestTypeHeatmap:
payload, err = readAsHeatmap(rows, queryWindow, step, queryName)
case qbtypes.RequestTypeRaw, qbtypes.RequestTypeTrace, qbtypes.RequestTypeRawStream:
payload, err = readAsRaw(rows, queryName)
// TODO: add support for other request types
@@ -112,35 +119,6 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt
stepMs := uint64(step.Milliseconds())
// Helper function to check if a timestamp represents a partial value
isPartialValue := func(timestamp int64) bool {
if stepMs == 0 || queryWindow == nil {
return false
}
timestampMs := uint64(timestamp)
// For the first interval, check if query start is misaligned
// The first complete interval starts at the first timestamp >= queryWindow.From that is aligned to step
firstCompleteInterval := queryWindow.From
if queryWindow.From%stepMs != 0 {
// Round up to next step boundary
firstCompleteInterval = ((queryWindow.From / stepMs) + 1) * stepMs
}
// If timestamp is before the first complete interval, it's partial
if timestampMs < firstCompleteInterval {
return true
}
// For the last interval, check if it would extend beyond query end
if timestampMs+stepMs > queryWindow.To {
return queryWindow.To%stepMs != 0
}
return false
}
// Pre-allocate for labels based on column count
lblValsCapacity := len(colNames) - 1 // -1 for timestamp
if lblValsCapacity < 0 {
@@ -271,7 +249,7 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt
series.Values = append(series.Values, &qbtypes.TimeSeriesValue{
Timestamp: ts,
Value: val,
Partial: isPartialValue(ts),
Partial: isPartialValue(ts, queryWindow, stepMs),
})
}
}
@@ -315,6 +293,223 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt
}, nil
}
// heatmapSeries accumulates one group's cells while the rows are read. Counts
// are held against their boundary rather than a slice because the axis is only
// known once every row has been seen.
type heatmapSeries struct {
labels []*qbtypes.Label
counts map[int64]map[float64]float64
}
// heatmapAccumulator is shared by the readers of the two things a heatmap can
// come back as: ClickHouse rows, and a PromQL matrix.
type heatmapAccumulator struct {
seriesByKey map[string]*heatmapSeries
seriesOrder []string
boundaries map[float64]struct{}
}
func newHeatmapAccumulator() *heatmapAccumulator {
return &heatmapAccumulator{
seriesByKey: map[string]*heatmapSeries{},
boundaries: map[float64]struct{}{},
}
}
// addCell files one cell under the group labelsKey identifies, keeping the
// labels from the first cell seen for it.
func (a *heatmapAccumulator) addCell(labelsKey string, lbls []*qbtypes.Label, ts int64, boundary, count float64) {
series, ok := a.seriesByKey[labelsKey]
if !ok {
series = &heatmapSeries{labels: lbls, counts: map[int64]map[float64]float64{}}
a.seriesByKey[labelsKey] = series
a.seriesOrder = append(a.seriesOrder, labelsKey)
}
if series.counts[ts] == nil {
series.counts[ts] = map[float64]float64{}
}
series.counts[ts][boundary] += count
if !math.IsInf(boundary, 1) {
a.boundaries[boundary] = struct{}{}
}
}
// foldSeries turns the collected cells into one series per group, in the order
// the groups first appeared.
func (a *heatmapAccumulator) foldSeries(queryWindow *qbtypes.TimeRange, stepMs uint64, queryName string) *qbtypes.TimeSeriesData {
if len(a.seriesOrder) == 0 {
return &qbtypes.TimeSeriesData{QueryName: queryName}
}
boundaries := make([]float64, 0, len(a.boundaries))
for boundary := range a.boundaries {
boundaries = append(boundaries, boundary)
}
slices.Sort(boundaries)
// the band past the last boundary is where the +Inf overflow lands
bandIndexByBoundary := make(map[float64]int, len(boundaries)+1)
for band, boundary := range boundaries {
bandIndexByBoundary[boundary] = band
}
bandIndexByBoundary[math.Inf(1)] = len(boundaries)
bucket := &qbtypes.AggregationBucket{
Index: 0,
Alias: "__result_0",
Meta: qbtypes.AggregationMeta{Buckets: boundaries},
Series: make([]*qbtypes.TimeSeries, 0, len(a.seriesOrder)),
}
for _, labelsKey := range a.seriesOrder {
accumulated := a.seriesByKey[labelsKey]
timestamps := make([]int64, 0, len(accumulated.counts))
for ts := range accumulated.counts {
timestamps = append(timestamps, ts)
}
slices.Sort(timestamps)
series := &qbtypes.TimeSeries{
Labels: accumulated.labels,
Values: make([]*qbtypes.TimeSeriesValue, 0, len(timestamps)),
}
for _, ts := range timestamps {
values := make([]float64, len(boundaries)+1)
for boundary, count := range accumulated.counts[ts] {
values[bandIndexByBoundary[boundary]] = count
}
series.Values = append(series.Values, &qbtypes.TimeSeriesValue{
Timestamp: ts,
Values: values,
Partial: isPartialValue(ts, queryWindow, stepMs),
})
}
bucket.Series = append(bucket.Series, series)
}
return &qbtypes.TimeSeriesData{
QueryName: queryName,
Aggregations: []*qbtypes.AggregationBucket{bucket},
}
}
// readAsHeatmap folds one row per cell — (timestamp, group labels, bucket upper
// boundary, count) — into one series per group.
func readAsHeatmap(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbtypes.Step, queryName string) (*qbtypes.TimeSeriesData, error) {
colTypes := rows.ColumnTypes()
colNames := rows.Columns()
slots := make([]any, len(colTypes))
for i, ct := range colTypes {
slots[i] = reflect.New(ct.ScanType()).Interface()
}
stepMs := uint64(step.Milliseconds())
accumulator := newHeatmapAccumulator()
// every column that is not the timestamp, the boundary or the count is a label
lblValsCapacity := len(colNames) - 3
if lblValsCapacity < 0 {
lblValsCapacity = 0
}
for rows.Next() {
if err := rows.Scan(slots...); err != nil {
return nil, err
}
var (
ts int64
boundary float64
count float64
hasCell bool
lblVals = make([]string, 0, lblValsCapacity)
lblObjs = make([]*qbtypes.Label, 0, lblValsCapacity)
)
for idx, ptr := range slots {
name := stripKeyAlias(colNames[idx])
value := derefValue(ptr)
if t, ok := value.(time.Time); ok {
ts = t.UnixMilli()
continue
}
switch name {
case qbtypes.HeatmapBucketColumn, legacyHeatmapBucketColumn:
boundary = numericAsFloat(value)
hasCell = true
default:
if aggRe.MatchString(name) || slices.Contains(legacyReservedColumnTargetAliases, name) {
count = numericAsFloat(value)
continue
}
// a nullable label column comes back as a nil any, which would
// otherwise key the series on the literal "<nil>"
if value == nil {
value = ""
}
lblVals = append(lblVals, fmt.Sprint(value))
lblObjs = append(lblObjs, &qbtypes.Label{
Key: telemetrytypes.TelemetryFieldKey{Name: name},
Value: value,
})
}
}
if ts == 0 || !hasCell || math.IsNaN(boundary) || math.IsInf(boundary, -1) {
continue
}
if math.IsNaN(count) || math.IsInf(count, 0) {
continue
}
sort.Strings(lblVals)
labelsKey := strings.Join(lblVals, ",")
accumulator.addCell(labelsKey, lblObjs, ts, boundary, count)
}
if err := rows.Err(); err != nil {
return nil, err
}
return accumulator.foldSeries(queryWindow, stepMs, queryName), nil
}
// isPartialValue reports whether the step interval starting at timestamp is only
// partly covered by the query window, which happens when the window boundaries
// are not step-aligned.
func isPartialValue(timestamp int64, queryWindow *qbtypes.TimeRange, stepMs uint64) bool {
if stepMs == 0 || queryWindow == nil {
return false
}
timestampMs := uint64(timestamp)
// For the first interval, check if query start is misaligned
// The first complete interval starts at the first timestamp >= queryWindow.From that is aligned to step
firstCompleteInterval := queryWindow.From
if queryWindow.From%stepMs != 0 {
// Round up to next step boundary
firstCompleteInterval = ((queryWindow.From / stepMs) + 1) * stepMs
}
// If timestamp is before the first complete interval, it's partial
if timestampMs < firstCompleteInterval {
return true
}
// For the last interval, check if it would extend beyond query end
if timestampMs+stepMs > queryWindow.To {
return queryWindow.To%stepMs != 0
}
return false
}
func isNumericKind(t reflect.Type) bool {
if t == nil {
return false

411
pkg/querier/heatmap_test.go Normal file
View File

@@ -0,0 +1,411 @@
package querier
import (
"math"
"reflect"
"testing"
"time"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeColumnType is the minimum of driver.ColumnType that readAsHeatmap reads:
// the scan type it allocates a slot from.
type fakeColumnType struct {
name string
scanType reflect.Type
}
func (c fakeColumnType) Name() string { return c.name }
func (c fakeColumnType) Nullable() bool { return false }
func (c fakeColumnType) ScanType() reflect.Type { return c.scanType }
func (c fakeColumnType) DatabaseTypeName() string { return c.scanType.String() }
// fakeRows replays a fixed set of rows, each holding one value per column in
// the order the columns are declared.
type fakeRows struct {
columns []fakeColumnType
rows [][]any
cursor int
}
func (r *fakeRows) Next() bool {
r.cursor++
return r.cursor <= len(r.rows)
}
func (r *fakeRows) Scan(dest ...any) error {
row := r.rows[r.cursor-1]
for i, value := range row {
reflect.ValueOf(dest[i]).Elem().Set(reflect.ValueOf(value))
}
return nil
}
func (r *fakeRows) ScanStruct(any) error { return nil }
func (r *fakeRows) ColumnTypes() []driver.ColumnType {
types := make([]driver.ColumnType, len(r.columns))
for i, column := range r.columns {
types[i] = column
}
return types
}
func (r *fakeRows) Totals(...any) error { return nil }
func (r *fakeRows) Columns() []string {
names := make([]string, len(r.columns))
for i, column := range r.columns {
names[i] = column.name
}
return names
}
func (r *fakeRows) HasData() bool { return len(r.rows) > 0 }
func (r *fakeRows) Close() error { return nil }
func (r *fakeRows) Err() error { return nil }
var _ driver.Rows = (*fakeRows)(nil)
func TestReadAsHeatmapBuildsSharedBucketAxis(t *testing.T) {
first := time.UnixMilli(1710000000000)
second := time.UnixMilli(1710000060000)
rows := &fakeRows{
columns: []fakeColumnType{
{name: "ts", scanType: reflect.TypeOf(time.Time{})},
{name: "__GROUP_BY_KEY_0_service.name", scanType: reflect.TypeOf("")},
{name: "__bucket", scanType: reflect.TypeOf(float64(0))},
{name: "__result_0", scanType: reflect.TypeOf(float64(0))},
},
rows: [][]any{
{first, "cart", 5.0, 3.0},
{first, "cart", 10.0, 7.0},
{first, "cart", math.Inf(1), 1.0},
{first, "pay", 10.0, 2.0},
{second, "cart", 5.0, 4.0},
{second, "pay", math.Inf(1), 6.0},
},
}
data, err := readAsHeatmap(rows, &qbtypes.TimeRange{From: 1710000000000, To: 1710000120000}, qbtypes.Step{Duration: time.Minute}, "A")
require.NoError(t, err)
require.Len(t, data.Aggregations, 1)
aggregation := data.Aggregations[0]
// +Inf is not a boundary; it is the slot past the last one
assert.Equal(t, []float64{5, 10}, aggregation.Meta.Buckets)
require.Len(t, aggregation.Series, 2)
cart := aggregation.Series[0]
require.Len(t, cart.Labels, 1)
assert.Equal(t, "cart", cart.Labels[0].Value)
require.Len(t, cart.Values, 2)
assert.Equal(t, int64(1710000000000), cart.Values[0].Timestamp)
assert.Equal(t, []float64{3, 7, 1}, cart.Values[0].Values)
assert.Equal(t, []float64{4, 0, 0}, cart.Values[1].Values)
pay := aggregation.Series[1]
assert.Equal(t, "pay", pay.Labels[0].Value)
assert.Equal(t, []float64{0, 2, 0}, pay.Values[0].Values)
assert.Equal(t, []float64{0, 0, 6}, pay.Values[1].Values)
}
func TestReadAsHeatmapWithoutGroupBy(t *testing.T) {
at := time.UnixMilli(1710000000000)
rows := &fakeRows{
columns: []fakeColumnType{
{name: "ts", scanType: reflect.TypeOf(time.Time{})},
{name: "__bucket", scanType: reflect.TypeOf(float64(0))},
{name: "__result_0", scanType: reflect.TypeOf(float64(0))},
},
rows: [][]any{
{at, 2.5, 9.0},
{at, 5.0, 4.0},
},
}
data, err := readAsHeatmap(rows, nil, qbtypes.Step{Duration: time.Minute}, "A")
require.NoError(t, err)
require.Len(t, data.Aggregations, 1)
aggregation := data.Aggregations[0]
assert.Equal(t, []float64{2.5, 5}, aggregation.Meta.Buckets)
require.Len(t, aggregation.Series, 1)
assert.Empty(t, aggregation.Series[0].Labels)
// no +Inf row, so the overflow slot is present but empty
assert.Equal(t, []float64{9, 4, 0}, aggregation.Series[0].Values[0].Values)
}
func TestReadAsHeatmapWithoutRows(t *testing.T) {
rows := &fakeRows{
columns: []fakeColumnType{
{name: "ts", scanType: reflect.TypeOf(time.Time{})},
{name: "__bucket", scanType: reflect.TypeOf(float64(0))},
{name: "__result_0", scanType: reflect.TypeOf(float64(0))},
},
}
data, err := readAsHeatmap(rows, nil, qbtypes.Step{Duration: time.Minute}, "A")
require.NoError(t, err)
assert.Equal(t, "A", data.QueryName)
assert.Empty(t, data.Aggregations)
}
func TestReadAsHeatmapMarksPartialTimestamps(t *testing.T) {
misaligned := time.UnixMilli(1710000000000)
aligned := time.UnixMilli(1710000060000)
rows := &fakeRows{
columns: []fakeColumnType{
{name: "ts", scanType: reflect.TypeOf(time.Time{})},
{name: "__bucket", scanType: reflect.TypeOf(float64(0))},
{name: "__result_0", scanType: reflect.TypeOf(float64(0))},
},
rows: [][]any{
{misaligned, 5.0, 1.0},
{aligned, 5.0, 2.0},
},
}
// The window starts mid-step, so the step the first row falls in is only
// partly covered by it.
data, err := readAsHeatmap(rows, &qbtypes.TimeRange{From: 1710000030000, To: 1710000120000}, qbtypes.Step{Duration: time.Minute}, "A")
require.NoError(t, err)
values := data.Aggregations[0].Series[0].Values
require.Len(t, values, 2)
assert.True(t, values[0].Partial)
assert.False(t, values[1].Partial)
}
func TestMergeTimeSeriesResultsUnionsHeatmapAxes(t *testing.T) {
// a log axis holds whichever bands the data reached, so a wide cached range
// and a narrow fresh one routinely disagree on which bands exist
cached := &qbtypes.TimeSeriesData{
QueryName: "A",
Aggregations: []*qbtypes.AggregationBucket{{
Index: 0,
Meta: qbtypes.AggregationMeta{Buckets: []float64{1, 4, 16}},
Series: []*qbtypes.TimeSeries{{
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host.name"}, Value: "node-1"}},
Values: []*qbtypes.TimeSeriesValue{{Timestamp: 1710000000000, Values: []float64{1, 2, 3, 4}}},
}},
}},
}
fresh := []*qbtypes.Result{{
Value: &qbtypes.TimeSeriesData{
QueryName: "A",
Aggregations: []*qbtypes.AggregationBucket{{
Index: 0,
Meta: qbtypes.AggregationMeta{Buckets: []float64{2, 4}},
Series: []*qbtypes.TimeSeries{{
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host.name"}, Value: "node-1"}},
Values: []*qbtypes.TimeSeriesValue{{Timestamp: 1710000060000, Values: []float64{5, 6, 7}}},
}},
}},
},
}}
merged := (&querier{}).mergeTimeSeriesResults(cached, fresh)
require.Len(t, merged.Aggregations, 1)
aggBucket := merged.Aggregations[0]
assert.Equal(t, []float64{1, 2, 4, 16}, aggBucket.Meta.Buckets)
require.Len(t, aggBucket.Series, 1)
require.Len(t, aggBucket.Series[0].Values, 2)
// the cached 16 band survives even though the fresh range never reached it
assert.Equal(t, []float64{1, 0, 2, 3, 4}, aggBucket.Series[0].Values[0].Values)
// and the fresh 2 band survives even though the cached range never had it
assert.Equal(t, []float64{0, 5, 6, 0, 7}, aggBucket.Series[0].Values[1].Values)
}
func TestReadAsHeatmapAcceptsHandWrittenColumnAliases(t *testing.T) {
at := time.UnixMilli(1710000000000)
// the aliases a user written clickhouse query would reach for, rather than
// the __bucket / __result_0 the statement builder emits
rows := &fakeRows{
columns: []fakeColumnType{
{name: "ts", scanType: reflect.TypeOf(time.Time{})},
{name: "service.name", scanType: reflect.TypeOf("")},
{name: "bucket", scanType: reflect.TypeOf(float64(0))},
{name: "value", scanType: reflect.TypeOf(float64(0))},
},
rows: [][]any{
{at, "cart", 5.0, 3.0},
{at, "cart", 10.0, 7.0},
},
}
data, err := readAsHeatmap(rows, nil, qbtypes.Step{Duration: time.Minute}, "A")
require.NoError(t, err)
require.Len(t, data.Aggregations, 1)
aggregation := data.Aggregations[0]
assert.Equal(t, []float64{5, 10}, aggregation.Meta.Buckets)
require.Len(t, aggregation.Series, 1)
require.Len(t, aggregation.Series[0].Labels, 1)
assert.Equal(t, "cart", aggregation.Series[0].Labels[0].Value)
assert.Equal(t, []float64{3, 7, 0}, aggregation.Series[0].Values[0].Values)
}
func TestApplyFormulasBucketsTheFormulaOutput(t *testing.T) {
q := &querier{logger: instrumentationtest.New().Logger()}
seriesAt := func(labelValue string, values ...float64) *qbtypes.TimeSeries {
points := make([]*qbtypes.TimeSeriesValue, 0, len(values))
for index, value := range values {
points = append(points, &qbtypes.TimeSeriesValue{
Timestamp: 1710000000000 + int64(index)*60000,
Value: value,
})
}
return &qbtypes.TimeSeries{
Labels: []*qbtypes.Label{{
Key: telemetrytypes.TelemetryFieldKey{Name: "host.name"},
Value: labelValue,
}},
Values: points,
}
}
results := map[string]*qbtypes.Result{
"A": {Value: &qbtypes.TimeSeriesData{
QueryName: "A",
Aggregations: []*qbtypes.AggregationBucket{{Index: 0, Series: []*qbtypes.TimeSeries{seriesAt("h1", 8, 64)}}},
}},
"B": {Value: &qbtypes.TimeSeriesData{
QueryName: "B",
Aggregations: []*qbtypes.AggregationBucket{{Index: 0, Series: []*qbtypes.TimeSeries{seriesAt("h1", 4, 16)}}},
}},
}
req := &qbtypes.QueryRangeRequest{
RequestType: qbtypes.RequestTypeHeatmap,
BucketOptions: &qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLog, Spec: qbtypes.LogBucketsSpec{}},
CompositeQuery: qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{
{Type: qbtypes.QueryTypeBuilder, Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{Name: "A", Disabled: true}},
{Type: qbtypes.QueryTypeBuilder, Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{Name: "B", Disabled: true}},
{Type: qbtypes.QueryTypeFormula, Spec: qbtypes.QueryBuilderFormula{Name: "F1", Expression: "A / B"}},
}},
}
results = q.applyFormulas(t.Context(), results, req)
formula, ok := results["F1"]
require.True(t, ok, "formula produced no result")
tsData, ok := formula.Value.(*qbtypes.TimeSeriesData)
require.True(t, ok)
require.Len(t, tsData.Aggregations, 1)
// 8/4 is 2 and 64/16 is 4, a doubling apart, so the filled axis carries
// every band from 2 to 4 inclusive and the two points sit at its ends
aggBucket := tsData.Aggregations[0]
require.Len(t, aggBucket.Meta.Buckets, 17)
assert.Equal(t, math.Exp2(1), aggBucket.Meta.Buckets[0])
assert.Equal(t, math.Exp2(2), aggBucket.Meta.Buckets[16])
require.Len(t, aggBucket.Series, 1)
points := aggBucket.Series[0].Values
require.Len(t, points, 2)
assert.Equal(t, float64(1), points[0].Values[0])
assert.Equal(t, float64(1), points[1].Values[16])
for index, point := range points {
require.Len(t, point.Values, 18, "point %d", index)
var total float64
for _, count := range point.Values {
total += count
}
assert.Equal(t, float64(1), total, "point %d counts the one series it came from", index)
}
}
func TestApplyFormulasCoarsensTheFormulaAxis(t *testing.T) {
q := &querier{logger: instrumentationtest.New().Logger()}
scale := 0
req := &qbtypes.QueryRangeRequest{
RequestType: qbtypes.RequestTypeHeatmap,
BucketOptions: &qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLog, Spec: qbtypes.LogBucketsSpec{Scale: &scale}},
CompositeQuery: qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{
{Type: qbtypes.QueryTypeBuilder, Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{Name: "A", Disabled: true}},
{Type: qbtypes.QueryTypeFormula, Spec: qbtypes.QueryBuilderFormula{Name: "F1", Expression: "A * 2"}},
}},
}
results := map[string]*qbtypes.Result{
"A": {Value: &qbtypes.TimeSeriesData{
QueryName: "A",
Aggregations: []*qbtypes.AggregationBucket{{Index: 0, Series: []*qbtypes.TimeSeries{{
Values: []*qbtypes.TimeSeriesValue{
{Timestamp: 1710000000000, Value: 1.5},
{Timestamp: 1710000060000, Value: 2},
},
}}}},
}},
}
results = q.applyFormulas(t.Context(), results, req)
tsData := results["F1"].Value.(*qbtypes.TimeSeriesData)
aggBucket := tsData.Aggregations[0]
// 3 and 4 sit in different bands at scale 4 but the same doubling at scale 0
assert.Equal(t, []float64{math.Exp2(2)}, aggBucket.Meta.Buckets)
assert.Equal(t, []float64{1, 0}, aggBucket.Series[0].Values[0].Values)
assert.Equal(t, []float64{1, 0}, aggBucket.Series[0].Values[1].Values)
}
func TestTrimResultToFluxBoundaryKeepsTheHeatmapAxis(t *testing.T) {
cache := &bucketCache{logger: instrumentationtest.New().Logger()}
result := &qbtypes.Result{
Type: qbtypes.RequestTypeHeatmap,
Value: &qbtypes.TimeSeriesData{
Aggregations: []*qbtypes.AggregationBucket{{
Index: 0,
Alias: "__result_0",
Meta: qbtypes.AggregationMeta{Unit: "By", Buckets: []float64{1, 2, 4}},
Series: []*qbtypes.TimeSeries{{
Values: []*qbtypes.TimeSeriesValue{
{Timestamp: 1710000000000, Values: []float64{1, 2, 3, 4}},
},
}},
}},
},
}
trimmed := cache.trimResultToFluxBoundary(result, 1710000060000)
tsData, ok := trimmed.Value.(*qbtypes.TimeSeriesData)
require.True(t, ok)
require.Len(t, tsData.Aggregations, 1)
// the counts are positional against the axis, so a cached bucket that lost
// Meta.Buckets would be realigned from an empty axis and collapse into the
// overflow slot on the way back out
aggBucket := tsData.Aggregations[0]
assert.Equal(t, []float64{1, 2, 4}, aggBucket.Meta.Buckets)
assert.Equal(t, "By", aggBucket.Meta.Unit)
assert.Equal(t, "__result_0", aggBucket.Alias)
}
func TestRealignFromAnEmptyAxisCollapsesIntoTheOverflow(t *testing.T) {
// pins the behaviour the trim bug exposed: with no axis to read the counts
// against, everything lands in the overflow slot
series := []*qbtypes.TimeSeries{{
Values: []*qbtypes.TimeSeriesValue{{Timestamp: 1710000000000, Values: []float64{7, 8, 9, 10}}},
}}
qbtypes.RealignHeatmapValues(series, nil, []float64{1, 2, 4})
assert.Equal(t, []float64{0, 0, 0, 7}, series[0].Values[0].Values)
}

View File

@@ -195,6 +195,17 @@ func postProcessBuilderQuery[T any](
return result
}
// resolveHeatmapAxis brings a heatmap axis to the resolution the caller asked
// for. Coarsening runs before the fill so the empty bands land at the resolution
// being returned rather than the one ClickHouse bucketed at.
func resolveHeatmapAxis(tsData *qbtypes.TimeSeriesData, bucketing qbtypes.HeatmapBucketing, requestedScale int) {
if bucketing.Kind == qbtypes.BucketsKindLog && requestedScale < bucketing.LogScale {
qbtypes.DownscaleHeatmapAxis(tsData, bucketing.LogScale, requestedScale)
bucketing.LogScale = requestedScale
}
qbtypes.DensifyHeatmapAxis(tsData, bucketing)
}
// postProcessMetricQuery applies postprocessing to a metric query result.
func postProcessMetricQuery(
q *querier,
@@ -216,6 +227,12 @@ func postProcessMetricQuery(
}
}
if req.RequestType == qbtypes.RequestTypeHeatmap && config.HeatmapBucketing != nil {
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
resolveHeatmapAxis(tsData, *config.HeatmapBucketing, req.BucketOptions.ResolveLogScale())
}
}
result = q.applySeriesLimit(result, query.Limit, query.Order)
if len(query.Functions) > 0 {
@@ -342,6 +359,19 @@ func (q *querier) applyFormulas(ctx context.Context, results map[string]*qbtypes
result = q.applySeriesLimit(result, formula.Limit, formula.Order)
results[name] = result
}
case qbtypes.RequestTypeHeatmap:
// The queries a formula reads were run as time series, so what
// arrives here is one value per group per timestamp.
result := q.processTimeSeriesFormula(ctx, results, formula, req)
if result != nil {
if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
bucketing := req.BucketOptions.ResolveBucketOptions()
qbtypes.BucketTimeSeriesValues(tsData, bucketing)
resolveHeatmapAxis(tsData, bucketing, req.BucketOptions.ResolveLogScale())
}
result = q.applySeriesLimit(result, formula.Limit, formula.Order)
results[name] = result
}
case qbtypes.RequestTypeScalar:
result := q.processScalarFormula(ctx, results, formula, req)
// For scalar results, apply limit by processScalarFormula itself since it needs to be applied before converting back to scalar format
@@ -494,7 +524,7 @@ func (q *querier) processScalarFormula(
bucket := &qbtypes.AggregationBucket{
Index: aggIdx,
Alias: scalarData.Columns[colIdx].Name,
Meta: scalarData.Columns[colIdx].Meta,
Meta: qbtypes.AggregationMeta{Unit: scalarData.Columns[colIdx].Meta.Unit},
Series: make([]*qbtypes.TimeSeries, 0),
}
@@ -667,13 +697,14 @@ func convertTimeSeriesDataToScalar(tsData *qbtypes.TimeSeriesData, queryName str
if name == "" {
name = fmt.Sprintf("__result_%d", agg.Index)
}
columns = append(columns, &qbtypes.ColumnDescriptor{
column := &qbtypes.ColumnDescriptor{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: name},
QueryName: queryName,
AggregationIndex: int64(agg.Index),
Meta: agg.Meta,
Type: qbtypes.ColumnTypeAggregation,
})
}
column.Meta.Unit = agg.Meta.Unit
columns = append(columns, column)
}
// Build rows.

View File

@@ -50,7 +50,7 @@ func (q *querier) QueryRangePreview(
env := []qbtypes.QueryEnvelope{req.CompositeQuery.Queries[idx]}
ps.Warnings = append(ps.Warnings, q.adjustStepInterval(env, req.Start, req.End)...)
missingMetricQueries, metricWarnings, mErr := q.resolveMetricMetadata(ctx, orgID, env, req.Start, req.End)
missingMetricQueries, metricWarnings, mErr := q.resolveMetricMetadata(ctx, orgID, env, req.Start, req.End, req.RequestType, req.BucketOptions)
if mErr != nil {
// Report this query's error but keep previewing the rest.
ps.Error = mErr

View File

@@ -0,0 +1,145 @@
package querier
import (
"fmt"
"math"
"slices"
"sort"
"strconv"
"strings"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
"github.com/SigNoz/signoz/pkg/errors"
qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// promHistogramBucketLabel is the label a classic histogram carries its
// cumulative upper bound on, in PromQL as in the metric itself.
const promHistogramBucketLabel = "le"
// promHeatmapGroup accumulates one group's cumulative counts. `le` series are
// separate series in a matrix, so a group is assembled across several of them
// and the differencing can only run once they have all been read.
type promHeatmapGroup struct {
labels []*qbv5.Label
labelsKey string
cumulative map[int64]map[float64]float64
}
// foldMatrixAsHeatmap reads a classic histogram matrix as heatmap cells: one
// series per (group, `le`) carrying the cumulative count at that boundary,
// folded into one series per group whose points hold a count per band.
//
// This is readAsHeatmap's counterpart for a result the builder did not produce.
// buildHistogramHeatmapFinalSelect differences along `le` in SQL with
// lagInFrame; there is no statement here to attach that to, so it runs below
// against the same rules.
//
// Whether the expression kept `le` can only be seen in the result, so a matrix
// carrying data but no `le` anywhere is refused rather than drawn as one
// meaningless band.
func foldMatrixAsHeatmap(matrix promql.Matrix, queryWindow *qbv5.TimeRange, stepMs uint64, queryName string) (*qbv5.TimeSeriesData, error) {
groups := map[string]*promHeatmapGroup{}
groupOrder := []string{}
sawBucketLabel := false
for _, promSeries := range matrix {
boundary, ok := extractBucketBoundary(promSeries.Metric)
if !ok {
continue
}
sawBucketLabel = true
lbls, labelsKey := extractHeatmapGroup(promSeries.Metric)
group, ok := groups[labelsKey]
if !ok {
group = &promHeatmapGroup{labels: lbls, labelsKey: labelsKey, cumulative: map[int64]map[float64]float64{}}
groups[labelsKey] = group
groupOrder = append(groupOrder, labelsKey)
}
for _, point := range promSeries.Floats {
// A non-finite cumulative count has nothing to difference against.
// Skipping the point leaves the band above it differenced against
// the next boundary that does have one, which is what lagInFrame
// does with an absent row on the builder path.
if math.IsNaN(point.F) || math.IsInf(point.F, 0) {
continue
}
if group.cumulative[point.T] == nil {
group.cumulative[point.T] = map[float64]float64{}
}
group.cumulative[point.T][boundary] = point.F
}
}
if len(matrix) > 0 && !sawBucketLabel {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput,
"promql heatmap needs a %q label to draw its bucket axis from, and %q returned none: keep it in the result, as in `sum by (%s) (increase(metric_bucket[5m]))`",
promHistogramBucketLabel, queryName, promHistogramBucketLabel)
}
accumulator := newHeatmapAccumulator()
for _, labelsKey := range groupOrder {
group := groups[labelsKey]
for ts, cumulative := range group.cumulative {
boundaries := make([]float64, 0, len(cumulative))
for boundary := range cumulative {
boundaries = append(boundaries, boundary)
}
slices.Sort(boundaries)
previous := float64(0)
for _, boundary := range boundaries {
accumulator.addCell(labelsKey, group.labels, ts, boundary, math.Max(cumulative[boundary]-previous, 0))
previous = cumulative[boundary]
}
}
}
return accumulator.foldSeries(queryWindow, stepMs, queryName), nil
}
// extractBucketBoundary reads the `le` label as a boundary. The label is a
// string, so `+Inf` arrives as one and parses to the overflow boundary. A -Inf
// or NaN label bounds nothing and is reported as absent.
func extractBucketBoundary(metric labels.Labels) (float64, bool) {
raw := metric.Get(promHistogramBucketLabel)
if raw == "" {
return 0, false
}
boundary, err := strconv.ParseFloat(raw, 64)
if err != nil || math.IsNaN(boundary) || math.IsInf(boundary, -1) {
return 0, false
}
return boundary, true
}
// extractHeatmapGroup returns the labels identifying a series' group — every
// label except `le`, which becomes the Y axis — and a key for it.
//
// The key holds names as well as values, unlike the row reader's, because two
// matrix series can carry different label sets where two rows of one result
// cannot, and values alone would collide across them.
func extractHeatmapGroup(metric labels.Labels) ([]*qbv5.Label, string) {
lbls := make([]*qbv5.Label, 0, metric.Len())
pairs := make([]string, 0, metric.Len())
metric.Range(func(l labels.Label) {
if l.Name == promHistogramBucketLabel || excludePromLabel(l.Name) {
return
}
lbls = append(lbls, &qbv5.Label{
Key: telemetrytypes.TelemetryFieldKey{Name: l.Name},
Value: l.Value,
})
pairs = append(pairs, fmt.Sprintf("%s=%s", l.Name, l.Value))
})
sort.Strings(pairs)
return lbls, strings.Join(pairs, ",")
}

View File

@@ -0,0 +1,231 @@
package querier
import (
"log/slog"
"math"
"sync"
"testing"
"time"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFoldMatrixAsHeatmapDifferencesAlongTheBucketLabel(t *testing.T) {
firstTimestamp := int64(1710000000000)
secondTimestamp := int64(1710000060000)
matrix := promql.Matrix{
{
Metric: labels.FromStrings("service.name", "cart", "le", "5"),
Floats: []promql.FPoint{{T: firstTimestamp, F: 3}, {T: secondTimestamp, F: 4}},
},
{
Metric: labels.FromStrings("service.name", "cart", "le", "10"),
Floats: []promql.FPoint{{T: firstTimestamp, F: 10}, {T: secondTimestamp, F: 4}},
},
{
Metric: labels.FromStrings("service.name", "cart", "le", "+Inf"),
Floats: []promql.FPoint{{T: firstTimestamp, F: 11}, {T: secondTimestamp, F: 4}},
},
{
Metric: labels.FromStrings("service.name", "pay", "le", "5"),
Floats: []promql.FPoint{{T: firstTimestamp, F: 0}, {T: secondTimestamp, F: 0}},
},
{
Metric: labels.FromStrings("service.name", "pay", "le", "10"),
Floats: []promql.FPoint{{T: firstTimestamp, F: 2}, {T: secondTimestamp, F: 0}},
},
{
Metric: labels.FromStrings("service.name", "pay", "le", "+Inf"),
Floats: []promql.FPoint{{T: firstTimestamp, F: 2}, {T: secondTimestamp, F: 6}},
},
}
data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000120000}, uint64(time.Minute.Milliseconds()), "A")
require.NoError(t, err)
require.Len(t, data.Aggregations, 1)
aggregation := data.Aggregations[0]
// +Inf is not a boundary; it is the slot past the last one
assert.Equal(t, []float64{5, 10}, aggregation.Meta.Buckets)
require.Len(t, aggregation.Series, 2)
cart := aggregation.Series[0]
require.Len(t, cart.Labels, 1)
assert.Equal(t, "service.name", cart.Labels[0].Key.Name)
assert.Equal(t, "cart", cart.Labels[0].Value)
require.Len(t, cart.Values, 2)
assert.Equal(t, firstTimestamp, cart.Values[0].Timestamp)
assert.Equal(t, []float64{3, 7, 1}, cart.Values[0].Values)
assert.Equal(t, []float64{4, 0, 0}, cart.Values[1].Values)
pay := aggregation.Series[1]
assert.Equal(t, "pay", pay.Labels[0].Value)
assert.Equal(t, []float64{0, 2, 0}, pay.Values[0].Values)
assert.Equal(t, []float64{0, 0, 6}, pay.Values[1].Values)
}
func TestToResultShapesAHeatmapRequestAsCells(t *testing.T) {
at := int64(1710000000000)
q := &promqlQuery{
query: qbv5.PromQuery{Name: "A", Step: qbv5.Step{Duration: time.Minute}},
tr: qbv5.TimeRange{From: 1710000000000, To: 1710000060000},
requestType: qbv5.RequestTypeHeatmap,
}
matrix := promql.Matrix{
{Metric: labels.FromStrings("le", "5"), Floats: []promql.FPoint{{T: at, F: 3}}},
{Metric: labels.FromStrings("le", "+Inf"), Floats: []promql.FPoint{{T: at, F: 8}}},
}
var mu sync.Mutex
var rows, bytes uint64
result, err := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes)
require.NoError(t, err)
assert.Equal(t, qbv5.RequestTypeHeatmap, result.Type)
tsData, ok := result.Value.(*qbv5.TimeSeriesData)
require.True(t, ok)
require.Len(t, tsData.Aggregations, 1)
assert.Equal(t, []float64{5}, tsData.Aggregations[0].Meta.Buckets)
point := tsData.Aggregations[0].Series[0].Values[0]
// counts, not a single value: the +Inf series becomes the overflow slot
assert.Equal(t, []float64{3, 5}, point.Values)
assert.Zero(t, point.Value)
}
func TestToResultRefusesAHeatmapRequestWithoutTheBucketLabel(t *testing.T) {
q := &promqlQuery{
query: qbv5.PromQuery{Name: "A", Step: qbv5.Step{Duration: time.Minute}},
tr: qbv5.TimeRange{From: 1710000000000, To: 1710000060000},
requestType: qbv5.RequestTypeHeatmap,
}
matrix := promql.Matrix{
{Metric: labels.FromStrings("service.name", "cart"), Floats: []promql.FPoint{{T: 1710000000000, F: 3}}},
}
var mu sync.Mutex
var rows, bytes uint64
result, err := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes)
require.Error(t, err)
assert.Nil(t, result)
}
// The cache key is the fingerprint alone, so two request types over one
// expression must not produce the same one — a time series payload served to a
// heatmap request has no axis and reads back as a single collapsed band.
func TestFingerprintSeparatesHeatmapFromTimeSeries(t *testing.T) {
fingerprintFor := func(requestType qbv5.RequestType) string {
q := &promqlQuery{
logger: slog.New(slog.DiscardHandler),
query: qbv5.PromQuery{Name: "A", Query: "sum by (le) (increase(signoz_latency_bucket[5m]))", Step: qbv5.Step{Duration: time.Minute}},
tr: qbv5.TimeRange{From: 1710000000000, To: 1710003600000},
requestType: requestType,
}
return q.Fingerprint()
}
heatmap := fingerprintFor(qbv5.RequestTypeHeatmap)
timeSeries := fingerprintFor(qbv5.RequestTypeTimeSeries)
assert.NotEmpty(t, heatmap, "a heatmap decomposes into time buckets like a time series")
assert.NotEqual(t, timeSeries, heatmap)
assert.Empty(t, fingerprintFor(qbv5.RequestTypeScalar), "a scalar result is its window's last point")
}
func TestFoldMatrixAsHeatmapRefusesAMatrixWithoutTheBucketLabel(t *testing.T) {
matrix := promql.Matrix{
{
Metric: labels.FromStrings("service.name", "cart"),
Floats: []promql.FPoint{{T: 1710000000000, F: 3}},
},
}
data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")
require.Error(t, err)
assert.Nil(t, data)
assert.Contains(t, err.Error(), `"le"`)
}
func TestFoldMatrixAsHeatmapAcceptsAnEmptyMatrix(t *testing.T) {
data, err := foldMatrixAsHeatmap(promql.Matrix{}, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")
require.NoError(t, err)
assert.Equal(t, "A", data.QueryName)
assert.Empty(t, data.Aggregations)
}
func TestFoldMatrixAsHeatmapClampsADecreasingCumulativeCount(t *testing.T) {
at := int64(1710000000000)
matrix := promql.Matrix{
{
Metric: labels.FromStrings("le", "5"),
Floats: []promql.FPoint{{T: at, F: 10}},
},
{
Metric: labels.FromStrings("le", "10"),
Floats: []promql.FPoint{{T: at, F: 4}},
},
}
data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")
require.NoError(t, err)
require.Len(t, data.Aggregations, 1)
// a cumulative count that went backwards would difference to -6
assert.Equal(t, []float64{10, 0, 0}, data.Aggregations[0].Series[0].Values[0].Values)
}
func TestFoldMatrixAsHeatmapWidensTheBandOverAMissingBoundary(t *testing.T) {
at := int64(1710000000000)
matrix := promql.Matrix{
{
Metric: labels.FromStrings("le", "5"),
Floats: []promql.FPoint{{T: at, F: 3}},
},
{
Metric: labels.FromStrings("le", "10"),
Floats: []promql.FPoint{{T: at, F: math.NaN()}},
},
{
Metric: labels.FromStrings("le", "20"),
Floats: []promql.FPoint{{T: at, F: 30}},
},
}
data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")
require.NoError(t, err)
require.Len(t, data.Aggregations, 1)
aggregation := data.Aggregations[0]
// 10 carried nothing to difference against, so it is not on the axis at all
// and 20 differences against 5, holding what (5,10] and (10,20] would split
assert.Equal(t, []float64{5, 20}, aggregation.Meta.Buckets)
assert.Equal(t, []float64{3, 27, 0}, aggregation.Series[0].Values[0].Values)
}
func TestFoldMatrixAsHeatmapHidesInternalLabels(t *testing.T) {
at := int64(1710000000000)
matrix := promql.Matrix{
{
Metric: labels.FromStrings("__temporality__", "delta", "__resource.host.name", "h1", "service.name", "cart", "le", "5"),
Floats: []promql.FPoint{{T: at, F: 3}},
},
}
data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")
require.NoError(t, err)
require.Len(t, data.Aggregations, 1)
require.Len(t, data.Aggregations[0].Series, 1)
series := data.Aggregations[0].Series[0]
require.Len(t, series.Labels, 1)
assert.Equal(t, "service.name", series.Labels[0].Key.Name)
}

View File

@@ -155,7 +155,12 @@ func (q *promqlQuery) Fingerprint() string {
if q.opts.serve != nil {
return ""
}
if q.requestType != qbv5.RequestTypeTimeSeries {
// Only a result that is one value per timestamp, or one vector of counts
// per timestamp, decomposes into cacheable time buckets. A scalar result is
// its window's last point, which says nothing about any sub-range of it.
switch q.requestType {
case qbv5.RequestTypeTimeSeries, qbv5.RequestTypeHeatmap:
default:
return ""
}
@@ -166,6 +171,10 @@ func (q *promqlQuery) Fingerprint() string {
}
parts := []string{
"promql",
// the cache key is the fingerprint alone, and a heatmap and a time
// series query over one expression return different shapes, so the
// request type has to separate their entries
fmt.Sprintf("requestType=%s", q.requestType.StringValue()),
query,
q.query.Step.String(),
}
@@ -369,7 +378,7 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
}
return nil, err
}
return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned), nil
return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned)
}
// When the serving provider has the RangeExecutor capability
@@ -385,7 +394,7 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
return nil, err
}
if served {
return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned), nil
return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned)
}
}
@@ -446,20 +455,48 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
}
warnings, _ := res.Warnings.AsStrings(query, 10, 0)
return q.toResult(matrix, warnings, began, &statsMu, &rowsScanned, &bytesScanned), nil
return q.toResult(matrix, warnings, began, &statsMu, &rowsScanned, &bytesScanned)
}
// excludePromLabel hides only known SigNoz storage keys: label names are user
// data and may legitimately start with "__" (e.g. __address__), so a blanket
// dunder strip mangles user labelsets. The __scope./__resource. prefixes cover
// every exporter version's keys.
func excludePromLabel(labelName string) bool {
return labelName == "__temporality__" ||
strings.HasPrefix(labelName, "__scope.") ||
strings.HasPrefix(labelName, "__resource.")
}
// collectExecStats snapshots the scan counters a query accumulated. Callers take
// it at the point they are done with the matrix, so the duration covers the
// shaping they did.
func collectExecStats(began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) qbv5.ExecStats {
statsMu.Lock()
defer statsMu.Unlock()
return qbv5.ExecStats{
RowsScanned: *rowsScanned,
BytesScanned: *bytesScanned,
DurationMS: uint64(time.Since(began).Milliseconds()),
}
}
// toResult converts an evaluated matrix into the v5 result shape, attaching
// the ClickHouse scan stats accumulated during evaluation.
func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) *qbv5.Result {
// Hide only known SigNoz storage keys: label names are user data and may
// legitimately start with "__" (e.g. __address__), so a blanket dunder
// strip mangles user labelsets. The __scope./__resource. prefixes cover
// every exporter version's keys.
excludeLabel := func(labelName string) bool {
return labelName == "__temporality__" ||
strings.HasPrefix(labelName, "__scope.") ||
strings.HasPrefix(labelName, "__resource.")
func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) (*qbv5.Result, error) {
// A heatmap reads one label as its Y axis and returns a count per band, so
// the per-series copy below cannot produce it.
if q.requestType == qbv5.RequestTypeHeatmap {
tsData, err := foldMatrixAsHeatmap(matrix, &q.tr, uint64(q.query.Step.Milliseconds()), q.query.Name)
if err != nil {
return nil, err
}
return &qbv5.Result{
Type: q.requestType,
Value: tsData,
Warnings: warnings,
Stats: collectExecStats(began, statsMu, rowsScanned, bytesScanned),
}, nil
}
var series []*qbv5.TimeSeries
@@ -467,7 +504,7 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti
var s qbv5.TimeSeries
lbls := make([]*qbv5.Label, 0, v.Metric.Len())
v.Metric.Range(func(l labels.Label) {
if excludeLabel(l.Name) {
if excludePromLabel(l.Name) {
return
}
lbls = append(lbls, &qbv5.Label{
@@ -495,13 +532,7 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti
series = append(series, &s)
}
statsMu.Lock()
stats := qbv5.ExecStats{
RowsScanned: *rowsScanned,
BytesScanned: *bytesScanned,
DurationMS: uint64(time.Since(began).Milliseconds()),
}
statsMu.Unlock()
stats := collectExecStats(began, statsMu, rowsScanned, bytesScanned)
tsData := &qbv5.TimeSeriesData{QueryName: q.query.Name}
// No bucket at all when nothing survived: a bucket holding no series reads
@@ -534,5 +565,5 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti
Value: payload,
Warnings: warnings,
Stats: stats,
}
}, nil
}

View File

@@ -495,7 +495,8 @@ func TestToResultDropsNonFiniteValues(t *testing.T) {
var mu sync.Mutex
var rows, bytes uint64
result := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes)
result, err := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes)
require.NoError(t, err)
tsData, ok := result.Value.(*qbv5.TimeSeriesData)
require.True(t, ok)
@@ -526,7 +527,9 @@ func TestToResultDropsSeriesAndBucketLeftEmpty(t *testing.T) {
var mu sync.Mutex
var rows, bytes uint64
tsData, ok := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes).Value.(*qbv5.TimeSeriesData)
result, err := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes)
require.NoError(t, err)
tsData, ok := result.Value.(*qbv5.TimeSeriesData)
require.True(t, ok)
require.Len(t, tsData.Aggregations, 1)
require.Len(t, tsData.Aggregations[0].Series, 1, "the all-NaN series is gone")
@@ -535,7 +538,9 @@ func TestToResultDropsSeriesAndBucketLeftEmpty(t *testing.T) {
allNaN := promql.Matrix{
{Metric: labels.FromStrings("job_name", "idleJob"), Floats: []promql.FPoint{{T: 1000, F: math.NaN()}}},
}
tsData, ok = q.toResult(allNaN, nil, time.Now(), &mu, &rows, &bytes).Value.(*qbv5.TimeSeriesData)
result, err = q.toResult(allNaN, nil, time.Now(), &mu, &rows, &bytes)
require.NoError(t, err)
tsData, ok = result.Value.(*qbv5.TimeSeriesData)
require.True(t, ok)
assert.Empty(t, tsData.Aggregations)
}

View File

@@ -156,7 +156,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
// We need to set if it is unspecified or adjust it if value is not within recommended range
intervalWarnings := q.adjustStepInterval(req.CompositeQuery.Queries, req.Start, req.End)
missingMetricQueries, metricWarnings, err := q.resolveMetricMetadata(ctx, orgID, req.CompositeQuery.Queries, req.Start, req.End)
missingMetricQueries, metricWarnings, err := q.resolveMetricMetadata(ctx, orgID, req.CompositeQuery.Queries, req.Start, req.End, req.RequestType, req.BucketOptions)
if err != nil {
return nil, err
}
@@ -177,7 +177,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
preseededResults := make(map[string]any)
for _, name := range missingMetricQueries {
switch req.RequestType {
case qbtypes.RequestTypeTimeSeries:
case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
preseededResults[name] = &qbtypes.TimeSeriesData{QueryName: name}
case qbtypes.RequestTypeScalar:
preseededResults[name] = &qbtypes.ScalarData{QueryName: name}
@@ -334,15 +334,24 @@ func (q *querier) buildQueries(
if missingMetricQuerySet[spec.Name] {
continue
}
// A disabled query in a heatmap request is there to feed a
// formula, and the formula evaluator reads
// TimeSeriesValue.Value, which heatmap cells leave at zero in
// favour of Values. Its inputs therefore run as time series;
// applyFormulas buckets the formula's output into cells after.
requestType := req.RequestType
if requestType == qbtypes.RequestTypeHeatmap && spec.Disabled {
requestType = qbtypes.RequestTypeTimeSeries
}
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, requestType)
var bq *builderQuery[qbtypes.MetricAggregation]
if spec.Source == telemetrytypes.SourceMeter {
event.Source = telemetrytypes.SourceMeter.StringValue()
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, query.Type, spec, timeRange, requestType, tmplVars, builderConfig{})
} else {
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, query.Type, spec, timeRange, requestType, tmplVars, builderConfig{})
}
queries[spec.Name] = bq
@@ -415,7 +424,7 @@ func (q *querier) populateQBEvent(event *qbtypes.QBEvent, queries []qbtypes.Quer
// resolved: never-seen metrics and dormant metrics (seen but no data in
// the query window).
// - err: Internal when a metadata fetch fails.
func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID, queries []qbtypes.QueryEnvelope, start, end uint64) (missingMetricQueries []string, metricWarnings []string, err error) {
func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID, queries []qbtypes.QueryEnvelope, start, end uint64, requestType qbtypes.RequestType, bucketOptions *qbtypes.BucketOptions) (missingMetricQueries []string, metricWarnings []string, err error) {
metricNames := make([]string, 0)
for idx := range queries {
if queries[idx].Type != qbtypes.QueryTypeBuilder {
@@ -465,6 +474,15 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID,
spec.Aggregations[i].Type = foundMetricType
}
}
// Only the enabled query draws cells, so only it needs an axis and
// the metric-type refusals that come with one. A heatmap refuses an
// unresolved type outright rather than returning an empty result for
// it, so this has to run before the drop below.
if requestType == qbtypes.RequestTypeHeatmap && !spec.Disabled {
if err := spec.Aggregations[i].ResolveHeatmapBucketing(bucketOptions); err != nil {
return nil, nil, err
}
}
if spec.Aggregations[i].Type == metrictypes.UnspecifiedType {
missingMetrics = append(missingMetrics, spec.Aggregations[i].MetricName)
continue
@@ -1000,7 +1018,7 @@ func (q *querier) mergeResults(cached *qbtypes.Result, fresh []*qbtypes.Result)
// Merge all fresh results including the first one
switch merged.Type {
case qbtypes.RequestTypeTimeSeries:
case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
// Pass nil as cached value to ensure proper merging of all fresh results
merged.Value = q.mergeTimeSeriesResults(nil, fresh)
}
@@ -1023,7 +1041,7 @@ func (q *querier) mergeResults(cached *qbtypes.Result, fresh []*qbtypes.Result)
}
switch merged.Type {
case qbtypes.RequestTypeTimeSeries:
case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
merged.Value = q.mergeTimeSeriesResults(cached.Value.(*qbtypes.TimeSeriesData), fresh)
}
@@ -1052,12 +1070,23 @@ func (q *querier) mergeTimeSeriesResults(cachedValue *qbtypes.TimeSeriesData, fr
// Map to store aggregation bucket metadata
bucketMetadata := make(map[int]*qbtypes.AggregationBucket)
// Both halves are moved onto the union of their axes before being merged
// positionally, so a band one range never reached reads as zero there.
axes := make([]*qbtypes.TimeSeriesData, 0, len(freshResults)+1)
axes = append(axes, cachedValue)
for _, result := range freshResults {
freshTS, _ := result.Value.(*qbtypes.TimeSeriesData)
axes = append(axes, freshTS)
}
mergedBoundaries := qbtypes.MergeHeatmapAxes(axes...)
// Process cached data if available
if cachedValue != nil && cachedValue.Aggregations != nil {
for _, aggBucket := range cachedValue.Aggregations {
if seriesMap[aggBucket.Index] == nil {
seriesMap[aggBucket.Index] = make(map[string]*qbtypes.TimeSeries)
}
qbtypes.RealignHeatmapValues(aggBucket.Series, aggBucket.Meta.Buckets, mergedBoundaries[aggBucket.Index])
if bucketMetadata[aggBucket.Index] == nil {
bucketMetadata[aggBucket.Index] = aggBucket
}
@@ -1109,6 +1138,7 @@ func (q *querier) mergeTimeSeriesResults(cachedValue *qbtypes.TimeSeriesData, fr
}
for _, aggBucket := range freshTS.Aggregations {
qbtypes.RealignHeatmapValues(aggBucket.Series, aggBucket.Meta.Buckets, mergedBoundaries[aggBucket.Index])
for _, series := range aggBucket.Series {
key := qbtypes.GetUniqueSeriesKey(series.Labels)
@@ -1172,6 +1202,9 @@ func (q *querier) mergeTimeSeriesResults(cachedValue *qbtypes.TimeSeriesData, fr
bucket.Alias = metadata.Alias
bucket.Meta = metadata.Meta
}
if boundaries, ok := mergedBoundaries[index]; ok {
bucket.Meta.Buckets = boundaries
}
result.Aggregations = append(result.Aggregations, bucket)
}

View File

@@ -129,7 +129,7 @@ func (b *meterQueryStatementBuilder) buildPipelineStatement(
}
// final SELECT
return b.metricsStatementBuilder.BuildFinalSelect(cteFragments, cteArgs, query)
return b.metricsStatementBuilder.BuildFinalSelect(cteFragments, cteArgs, qbtypes.RequestTypeTimeSeries, query)
}
func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(

View File

@@ -4,9 +4,13 @@ import (
"context"
"fmt"
"log/slog"
"math"
"slices"
"strconv"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -113,7 +117,7 @@ func (b *StatementBuilder) Build(
orgID valuer.UUID,
start uint64,
end uint64,
_ qbtypes.RequestType,
requestType qbtypes.RequestType,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
@@ -125,13 +129,14 @@ func (b *StatementBuilder) Build(
start, end = querybuilder.AdjustedMetricTimeRange(start, end, uint64(query.StepInterval.Seconds()), query)
return b.buildPipelineStatement(ctx, orgID, start, end, query, keys, variables)
return b.buildPipelineStatement(ctx, orgID, start, end, requestType, query, keys, variables)
}
func (b *StatementBuilder) buildPipelineStatement(
ctx context.Context,
orgID valuer.UUID,
start, end uint64,
requestType qbtypes.RequestType,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
@@ -144,7 +149,7 @@ func (b *StatementBuilder) buildPipelineStatement(
cteQuery := query
if query.Aggregations[0].Type == metrictypes.HistogramType {
query.GroupBy = slices.DeleteFunc(slices.Clone(query.GroupBy), isHistogramBucket)
cteQuery = histogramCTEQuery(query)
cteQuery = histogramCTEQuery(requestType, query)
}
agg := cteQuery.Aggregations[0]
@@ -216,7 +221,7 @@ func (b *StatementBuilder) buildPipelineStatement(
}
}
mainStmt, err := b.BuildFinalSelect(cteFragments, cteArgs, query)
mainStmt, err := b.BuildFinalSelect(cteFragments, cteArgs, requestType, query)
if err != nil {
return nil, err
}
@@ -224,7 +229,7 @@ func (b *StatementBuilder) buildPipelineStatement(
if reducedFragments == nil {
return mainStmt, nil
}
reducedStmt, err := b.BuildFinalSelect(reducedFragments, reducedArgs, query)
reducedStmt, err := b.BuildFinalSelect(reducedFragments, reducedArgs, requestType, query)
if err != nil {
return nil, err
}
@@ -758,11 +763,9 @@ func (b *StatementBuilder) buildSpatialAggregationCTE(
func (b *StatementBuilder) BuildFinalSelect(
cteFragments []string,
cteArgs [][]any,
requestType qbtypes.RequestType,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
) (*qbtypes.Statement, error) {
metricType := query.Aggregations[0].Type
spaceAgg := query.Aggregations[0].SpaceAggregation
combined := querybuilder.CombineCTEs(cteFragments)
var args []any
@@ -770,6 +773,22 @@ func (b *StatementBuilder) BuildFinalSelect(
args = append(args, a...)
}
if requestType == qbtypes.RequestTypeHeatmap {
return buildHeatmapFinalSelect(combined, args, query)
}
return buildAggregationFinalSelect(combined, args, query)
}
// buildAggregationFinalSelect reads __spatial_aggregation_cte as one value per
// (group, timestamp), which is what every request type but heatmap wants.
func buildAggregationFinalSelect(
combined string,
args []any,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
) (*qbtypes.Statement, error) {
metricType := query.Aggregations[0].Type
spaceAgg := query.Aggregations[0].SpaceAggregation
sb := sqlbuilder.NewSelectBuilder()
if metricType == metrictypes.HistogramType && spaceAgg.IsPercentile() {
@@ -842,17 +861,160 @@ func (b *StatementBuilder) BuildFinalSelect(
return &qbtypes.Statement{Query: combined + q, Args: append(args, a...)}, nil
}
const histogramBucketKey = "le"
const (
histogramBucketKey = "le"
heatmapValueAlias = "__result_0"
heatmapWindow = "__heatmap_window"
)
func isHistogramBucket(k qbtypes.GroupByKey) bool { return k.Name == histogramBucketKey }
func histogramCTEQuery(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation] {
// buildHeatmapFinalSelect turns __spatial_aggregation_cte into one row per
// heatmap cell: (ts, group labels..., bucket upper boundary, count). Histograms
// already carry their boundaries as `le` labels; every other metric type has its
// axis derived from the aggregated value itself.
func buildHeatmapFinalSelect(
combined string,
args []any,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
) (*qbtypes.Statement, error) {
if query.Aggregations[0].Type == metrictypes.HistogramType {
return buildHistogramHeatmapFinalSelect(combined, args, query)
}
return buildValueHeatmapFinalSelect(combined, args, query)
}
// buildHistogramHeatmapFinalSelect differences the cumulative per-`le` counts in
// __spatial_aggregation_cte into a count per band.
//
// `le` labels are cumulative upper bounds, so a bucket's own count is the
// difference against the next-smallest `le` in the same (group, timestamp).
// The boundary reported is the `le` itself, which leaves the `le=+Inf` row
// carrying an infinite boundary for the reader to turn into the open-above
// overflow band.
func buildHistogramHeatmapFinalSelect(
combined string,
args []any,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
) (*qbtypes.Statement, error) {
groupAliases := GroupByAliases(query.GroupBy)
partitionBy := append(append([]string{}, groupAliases...), "ts")
sb := sqlbuilder.NewSelectBuilder()
sb.Select("ts")
sb.SelectMore(groupAliases...)
sb.SelectMore(fmt.Sprintf("toFloat64(%s) AS %s", histogramBucketKey, qbtypes.HeatmapBucketColumn))
// Counts across `le` should rise monotonically; partial scrapes can break
// that, and a negative cell count has no meaning on a heatmap.
sb.SelectMore(fmt.Sprintf(
"greatest(value - lagInFrame(value, 1, 0) OVER %s, 0) AS %s",
heatmapWindow, heatmapValueAlias,
))
// sqlbuilder has no WINDOW clause, and the fragment has to land between FROM
// and ORDER BY. Heatmap statements never carry a WHERE or GROUP BY here, so
// appending it to FROM puts it in the right place.
sb.From(fmt.Sprintf(
"__spatial_aggregation_cte WINDOW %s AS (PARTITION BY %s ORDER BY toFloat64(%s))",
heatmapWindow, strings.Join(partitionBy, ", "), histogramBucketKey,
))
sb.OrderBy(groupAliases...)
sb.OrderBy("ts", fmt.Sprintf("toFloat64(%s)", histogramBucketKey))
q, a := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
return &qbtypes.Statement{Query: combined + q, Args: append(args, a...)}, nil
}
// buildValueHeatmapFinalSelect places each spatially aggregated value in a band
// of the requested axis. __spatial_aggregation_cte holds one row per (group,
// timestamp), so a cell counts the one group it came from; the panel sums
// across the series it is showing, which is what lets the legend select among
// them.
func buildValueHeatmapFinalSelect(
combined string,
args []any,
query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
) (*qbtypes.Statement, error) {
bucketing := query.Aggregations[0].HeatmapBucketing
if bucketing == nil {
return nil, errors.NewInternalf(errors.CodeInternal,
"heatmap over a %s metric reached the statement builder without a resolved bucket axis",
query.Aggregations[0].Type.StringValue())
}
boundary, err := heatmapBoundaryExpr(*bucketing)
if err != nil {
return nil, err
}
groupAliases := GroupByAliases(query.GroupBy)
sb := sqlbuilder.NewSelectBuilder()
sb.Select("ts")
sb.SelectMore(groupAliases...)
sb.SelectMore(fmt.Sprintf("%s AS %s", boundary, qbtypes.HeatmapBucketColumn))
sb.SelectMore(fmt.Sprintf("toFloat64(1) AS %s", heatmapValueAlias))
sb.From("__spatial_aggregation_cte")
sb.OrderBy(groupAliases...)
sb.OrderBy("ts", qbtypes.HeatmapBucketColumn)
q, a := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
return &qbtypes.Statement{Query: combined + q, Args: append(args, a...)}, nil
}
// heatmapBoundaryExpr renders the upper bound of the band `value` falls in.
// Values at or below zero have no log band of their own and no linear band below
// the first, so both scalings report them at the axis's lowest boundary rather
// than dropping the row: an upper bound still describes them truthfully.
func heatmapBoundaryExpr(bucketing qbtypes.HeatmapBucketing) (string, error) {
switch bucketing.Kind {
case qbtypes.BucketsKindLinear:
maxValue := formatFloat(bucketing.MaxValue)
numBuckets := strconv.Itoa(bucketing.NumBuckets)
// Indexing on value*numBuckets/maxValue rather than on a precomputed
// width keeps the top boundary exactly maxValue instead of a rounded
// multiple of that width.
return fmt.Sprintf(
"multiIf(value > %s, toFloat64('+Inf'), least(greatest(ceil(value * %s / %s), 1), %s) * %s / %s)",
maxValue, numBuckets, maxValue, numBuckets, maxValue, numBuckets,
), nil
case qbtypes.BucketsKindLog:
// The exponential histogram mapping at a fixed scale: 2^LogScale bands
// per doubling makes a band's index a pure function of the value, so the
// data never has to be scanned to decide where the boundaries go.
//
// The two clamps keep the axis finite. Without the lower one a single
// value approaching zero runs the band index off to -inf, and filling
// the empty bands below it would then cost thousands of slots per point.
bandsPerDoubling := formatFloat(math.Exp2(float64(bucketing.LogScale)))
lowest := formatFloat(qbtypes.LowestLogBoundary)
highest := formatFloat(qbtypes.HighestLogBoundary)
return fmt.Sprintf(
"multiIf(value <= 0, toFloat64(0), value <= %s, %s, value > %s, toFloat64('+Inf'), pow(2, ceil(log2(value) * %s) / %s))",
lowest, lowest, highest, bandsPerDoubling, bandsPerDoubling,
), nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
"unsupported bucketsScaling %q for heatmap requests", bucketing.Kind.StringValue())
}
}
// formatFloat renders a float64 as the shortest literal that reads back as the
// same value, so a boundary computed from it is identical on every row.
func formatFloat(v float64) string {
return strconv.FormatFloat(v, 'g', -1, 64)
}
func histogramCTEQuery(requestType qbtypes.RequestType, query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation] {
query.GroupBy = append(slices.Clone(query.GroupBy), qbtypes.GroupByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: histogramBucketKey},
})
query.Aggregations = slices.Clone(query.Aggregations)
if query.Aggregations[0].SpaceAggregation.IsPercentile() {
// A heatmap cell is an observation count whatever space aggregation was
// asked for, since the axis is the `le` labels rather than anything the
// space aggregation picks out. Rates would scale every cell by the step.
if query.Aggregations[0].SpaceAggregation.IsPercentile() && requestType != qbtypes.RequestTypeHeatmap {
query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationRate
} else {
query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationIncrease

View File

@@ -284,6 +284,133 @@ func TestStatementBuilder(t *testing.T) {
},
expectedErr: nil,
},
{
name: "test_histogram_heatmap_sum",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "signoz_latency",
Type: metrictypes.HistogramType,
Temporality: metrictypes.Delta,
TimeAggregation: metrictypes.TimeAggregationIncrease,
SpaceAggregation: metrictypes.SpaceAggregationSum,
},
},
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value) AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, toFloat64(le) AS __bucket, greatest(value - lagInFrame(value, 1, 0) OVER __heatmap_window, 0) AS __result_0 FROM __spatial_aggregation_cte WINDOW __heatmap_window AS (PARTITION BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY toFloat64(le)) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts, toFloat64(le)",
Args: []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "signoz_latency", uint64(1747947360000), uint64(1747983420000)},
},
expectedErr: nil,
},
{
name: "test_histogram_heatmap_percentile",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "signoz_latency",
Type: metrictypes.HistogramType,
Temporality: metrictypes.Delta,
TimeAggregation: metrictypes.TimeAggregationRate,
SpaceAggregation: metrictypes.SpaceAggregationPercentile95,
},
},
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value) AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, toFloat64(le) AS __bucket, greatest(value - lagInFrame(value, 1, 0) OVER __heatmap_window, 0) AS __result_0 FROM __spatial_aggregation_cte WINDOW __heatmap_window AS (PARTITION BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY toFloat64(le)) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts, toFloat64(le)",
Args: []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "signoz_latency", uint64(1747947360000), uint64(1747983420000)},
},
expectedErr: nil,
},
{
name: "test_gauge_heatmap_log",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
Temporality: metrictypes.Unspecified,
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
HeatmapBucketing: &qbtypes.HeatmapBucketing{
Kind: qbtypes.BucketsKindLog,
LogScale: qbtypes.MaxLogScale,
NumBuckets: qbtypes.DefaultNumBuckets,
},
},
},
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "host.name",
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `__GROUP_BY_KEY_0_host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_host.name`) SELECT ts, `__GROUP_BY_KEY_0_host.name`, multiIf(value <= 0, toFloat64(0), value <= 2.3283064365386963e-10, 2.3283064365386963e-10, value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(value) * 16) / 16)) AS __bucket, toFloat64(1) AS __result_0 FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_host.name`, ts, __bucket",
Args: []any{"system.memory.usage", uint64(1747936800000), uint64(1747983420000), "unspecified", "system.memory.usage", uint64(1747947360000), uint64(1747983420000), 0},
},
expectedErr: nil,
},
{
name: "test_gauge_heatmap_linear",
requestType: qbtypes.RequestTypeHeatmap,
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "system.memory.usage",
Type: metrictypes.GaugeType,
Temporality: metrictypes.Unspecified,
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
HeatmapBucketing: &qbtypes.HeatmapBucketing{
Kind: qbtypes.BucketsKindLinear,
LogScale: qbtypes.MaxLogScale,
MaxValue: 500,
NumBuckets: 25,
},
},
},
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "host.name",
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `__GROUP_BY_KEY_0_host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_host.name`) SELECT ts, `__GROUP_BY_KEY_0_host.name`, multiIf(value > 500, toFloat64('+Inf'), least(greatest(ceil(value * 25 / 500), 1), 25) * 500 / 25) AS __bucket, toFloat64(1) AS __result_0 FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_host.name`, ts, __bucket",
Args: []any{"system.memory.usage", uint64(1747936800000), uint64(1747983420000), "unspecified", "system.memory.usage", uint64(1747947360000), uint64(1747983420000), 0},
},
expectedErr: nil,
},
{
name: "test_gauge_avg_sum",
requestType: qbtypes.RequestTypeTimeSeries,

View File

@@ -35,6 +35,7 @@ func (PanelPlugin) PrepareJSONSchema(s *jsonschema.Schema) error {
string(PanelKindTable): schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTablePanelSpec"),
string(PanelKindHistogram): schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec"),
string(PanelKindList): schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec"),
string(PanelKindHeatmap): schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec"),
})
}
@@ -65,6 +66,7 @@ func (PanelPlugin) JSONSchemaOneOf() []any {
PanelPluginVariant[TablePanelSpec]{Kind: string(PanelKindTable)},
PanelPluginVariant[HistogramPanelSpec]{Kind: string(PanelKindHistogram)},
PanelPluginVariant[ListPanelSpec]{Kind: string(PanelKindList)},
PanelPluginVariant[HeatmapPanelSpec]{Kind: string(PanelKindHeatmap)},
}
}
@@ -228,6 +230,7 @@ var (
PanelKindTable: func() any { return new(TablePanelSpec) },
PanelKindHistogram: func() any { return new(HistogramPanelSpec) },
PanelKindList: func() any { return new(ListPanelSpec) },
PanelKindHeatmap: func() any { return new(HeatmapPanelSpec) },
}
queryPluginSpecs = map[QueryPluginKind]func() any{
QueryKindBuilder: func() any { return new(BuilderQuerySpec) },
@@ -250,6 +253,7 @@ var (
PanelKindPieChart: {QueryKindBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
PanelKindTable: {QueryKindBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
PanelKindList: {QueryKindBuilder},
PanelKindHeatmap: {QueryKindBuilder},
}
)

View File

@@ -173,10 +173,11 @@ const (
PanelKindTable PanelPluginKind = "signoz/TablePanel"
PanelKindHistogram PanelPluginKind = "signoz/HistogramPanel"
PanelKindList PanelPluginKind = "signoz/ListPanel"
PanelKindHeatmap PanelPluginKind = "signoz/HeatmapPanel"
)
func (PanelPluginKind) Enum() []any {
return []any{PanelKindTimeSeries, PanelKindBarChart, PanelKindNumber, PanelKindPieChart, PanelKindTable, PanelKindHistogram, PanelKindList}
return []any{PanelKindTimeSeries, PanelKindBarChart, PanelKindNumber, PanelKindPieChart, PanelKindTable, PanelKindHistogram, PanelKindList, PanelKindHeatmap}
}
type TimeSeriesPanelSpec struct {
@@ -237,6 +238,51 @@ type ListPanelSpec struct {
SelectFields []telemetrytypes.TelemetryFieldKey `json:"selectFields,omitzero" validate:"dive"`
}
type HeatmapPanelSpec struct {
Visualization HeatmapVisualization `json:"visualization"`
Formatting PanelFormatting `json:"formatting"`
Legend Legend `json:"legend"`
ShowOverflow bool `json:"showOverflow"`
Colors HeatmapColors `json:"colors"`
}
type HeatmapVisualization struct {
BasicVisualization
ShowVisualMap bool `json:"showVisualMap"`
}
type HeatmapColors struct {
Mode HeatmapColorMode `json:"mode"`
Scale HeatmapColorScale `json:"scale"`
// Min and Max clamp the colour scale; nil means derive from the data.
Min *float64 `json:"min"`
Max *float64 `json:"max"`
// Scheme, Steps and Reverse apply in scheme mode.
Scheme string `json:"scheme"`
Steps int `json:"steps" validate:"omitempty,min=2,max=128"`
Reverse bool `json:"reverse"`
// Fill applies in opacity mode; empty means the selected group's legend colour.
Fill string `json:"fill"`
}
func (c *HeatmapColors) UnmarshalJSON(data []byte) error {
type alias HeatmapColors
var tmp alias
if err := json.Unmarshal(data, &tmp); err != nil {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid heatmap colors")
}
*c = HeatmapColors(tmp)
return c.validate()
}
func (c HeatmapColors) validate() error {
if c.Min != nil && c.Max != nil && *c.Min > *c.Max {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput,
"heatmap colors.min %v is greater than colors.max %v", *c.Min, *c.Max)
}
return nil
}
// ══════════════════════════════════════════════
// Panel common types
// ══════════════════════════════════════════════
@@ -709,3 +755,78 @@ func (p *PrecisionOption) UnmarshalJSON(data []byte) error {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid precision option %q: must be `0`, `1`, `2`, `3`, `4`, or `full`", v)
}
}
type HeatmapColorMode struct{ valuer.String }
var (
HeatmapColorModeScheme = HeatmapColorMode{valuer.NewString("scheme")} // default
HeatmapColorModeOpacity = HeatmapColorMode{valuer.NewString("opacity")}
)
func (HeatmapColorMode) Enum() []any {
return []any{HeatmapColorModeScheme, HeatmapColorModeOpacity}
}
func (m HeatmapColorMode) ValueOrDefault() string {
if m.IsZero() {
return HeatmapColorModeScheme.StringValue()
}
return m.StringValue()
}
func (m HeatmapColorMode) MarshalJSON() ([]byte, error) {
return json.Marshal(m.ValueOrDefault())
}
func (m *HeatmapColorMode) UnmarshalJSON(data []byte) error {
var v string
if err := json.Unmarshal(data, &v); err != nil {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid heatmap color mode: must be a string, one of `scheme` or `opacity`")
}
mode := HeatmapColorMode{valuer.NewString(v)}
switch mode {
case HeatmapColorModeScheme, HeatmapColorModeOpacity:
*m = mode
return nil
default:
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid heatmap color mode %q: must be `scheme` or `opacity`", v)
}
}
type HeatmapColorScale struct{ valuer.String }
var (
HeatmapColorScaleLog = HeatmapColorScale{valuer.NewString("log")} // default
HeatmapColorScaleSqrt = HeatmapColorScale{valuer.NewString("sqrt")}
HeatmapColorScaleLinear = HeatmapColorScale{valuer.NewString("linear")}
)
func (HeatmapColorScale) Enum() []any {
return []any{HeatmapColorScaleLog, HeatmapColorScaleSqrt, HeatmapColorScaleLinear}
}
func (s HeatmapColorScale) ValueOrDefault() string {
if s.IsZero() {
return HeatmapColorScaleLog.StringValue()
}
return s.StringValue()
}
func (s HeatmapColorScale) MarshalJSON() ([]byte, error) {
return json.Marshal(s.ValueOrDefault())
}
func (s *HeatmapColorScale) UnmarshalJSON(data []byte) error {
var v string
if err := json.Unmarshal(data, &v); err != nil {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid heatmap color scale: must be a string, one of `log`, `sqrt`, or `linear`")
}
scale := HeatmapColorScale{valuer.NewString(v)}
switch scale {
case HeatmapColorScaleLog, HeatmapColorScaleSqrt, HeatmapColorScaleLinear:
*s = scale
return nil
default:
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid heatmap color scale %q: must be `log`, `sqrt`, or `linear`", v)
}
}

View File

@@ -540,6 +540,8 @@ type MetricAggregation struct {
// reduce to operator for metric scalar requests
ReduceTo ReduceTo `json:"reduceTo,omitzero"`
HeatmapBucketing *HeatmapBucketing `json:"-"`
Reduced bool `json:"-"`
}
@@ -554,6 +556,10 @@ func (m MetricAggregation) Copy() MetricAggregation {
valueFilterCopy := *m.ValueFilter
c.ValueFilter = &valueFilterCopy
}
if m.HeatmapBucketing != nil {
bucketingCopy := *m.HeatmapBucketing
c.HeatmapBucketing = &bucketingCopy
}
return c
}

View File

@@ -0,0 +1,405 @@
package querybuildertypesv5
import (
"maps"
"math"
"slices"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
)
const (
// A positive value approaching zero runs its band index off to -inf, so
// without a clamp one near-zero sample would stretch the axis by thousands
// of bands once DensifyHeatmapAxis fills the empty ones in.
MinLogBandIndex = -512 // 2^-32, about 2.3e-10
MaxLogBandIndex = 1024 // 2^64, about 1.8e19
)
// LowestLogBoundary and HighestLogBoundary are the ends the log axis is clamped
// to. They do not vary with the requested scale.
var (
LowestLogBoundary = math.Exp2(float64(MinLogBandIndex) / math.Exp2(MaxLogScale))
HighestLogBoundary = math.Exp2(float64(MaxLogBandIndex) / math.Exp2(MaxLogScale))
)
// HeatmapBucketing is the bucket axis a heatmap statement builds in ClickHouse,
// resolved from BucketOptions once the metric type is known. It stays nil for
// histograms, whose boundaries come from their own `le` labels.
type HeatmapBucketing struct {
Kind BucketsKind
// LogScale is always MaxLogScale; LogBucketsSpec.Scale coarsens the result
// afterwards rather than changing this.
LogScale int
// MaxValue and NumBuckets are linear only.
MaxValue float64
NumBuckets int
}
// ResolveBucketOptions fills in what the caller left unset. An absent
// BucketOptions resolves to the finest log axis, the one kind that needs nothing
// from the caller.
func (b *BucketOptions) ResolveBucketOptions() HeatmapBucketing {
resolved := HeatmapBucketing{
Kind: BucketsKindLog,
LogScale: MaxLogScale,
NumBuckets: DefaultNumBuckets,
}
if b == nil {
return resolved
}
if spec, ok := b.Spec.(LinearBucketsSpec); ok {
resolved.Kind = BucketsKindLinear
resolved.MaxValue = spec.MaxValue
if spec.NumBuckets > 0 {
resolved.NumBuckets = spec.NumBuckets
}
}
return resolved
}
// ResolveLogScale returns the axis resolution the caller wants back, which
// postprocessing folds the MaxLogScale axis down to.
func (b *BucketOptions) ResolveLogScale() int {
if b == nil {
return MaxLogScale
}
if spec, ok := b.Spec.(LogBucketsSpec); ok && spec.Scale != nil {
return *spec.Scale
}
return MaxLogScale
}
// ResolveHeatmapBucketing sets a.HeatmapBucketing to the axis a heatmap draws its
// rows from, and refuses the metric types that cannot produce one. It cannot live
// in validateHeatmap: a.Type is resolved from metadata after that has run.
func (a *MetricAggregation) ResolveHeatmapBucketing(bucketOptions *BucketOptions) error {
switch a.Type {
case metrictypes.HistogramType:
if bucketOptions != nil {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"bucketOptions are not supported for histogram metrics: %q takes its bucket axis from its own `le` labels, so nothing in the spec would be applied", a.MetricName)
}
a.HeatmapBucketing = nil
return nil
// A summary carries no boundaries of its own either, and its samples reach
// the final select the same way a gauge's do, so it buckets identically.
case metrictypes.GaugeType, metrictypes.SumType, metrictypes.SummaryType:
bucketing := bucketOptions.ResolveBucketOptions()
a.HeatmapBucketing = &bucketing
return nil
case metrictypes.UnspecifiedType:
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"heatmaps need a metric whose type is known: no type is recorded for %q, so its bucket axis cannot be chosen", a.MetricName)
case metrictypes.ExpHistogramType:
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"heatmaps are not supported for exponential histograms yet: %q keeps its bucket counts in a sketch column, which needs its own reader", a.MetricName)
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"heatmaps are not supported for %s metrics", a.Type.StringValue())
}
}
// MergeHeatmapAxes collects, per aggregation index, every bucket boundary any of
// tsData reached, so that halves holding different bands can be merged onto one
// axis. Only heatmap results carry boundaries, so it comes back empty for
// everything else and the realignment it feeds is a no-op.
func MergeHeatmapAxes(tsData ...*TimeSeriesData) map[int][]float64 {
reached := map[int]map[float64]struct{}{}
for _, data := range tsData {
if data == nil {
continue
}
for _, aggBucket := range data.Aggregations {
if len(aggBucket.Meta.Buckets) == 0 {
continue
}
if reached[aggBucket.Index] == nil {
reached[aggBucket.Index] = map[float64]struct{}{}
}
for _, boundary := range aggBucket.Meta.Buckets {
reached[aggBucket.Index][boundary] = struct{}{}
}
}
}
merged := make(map[int][]float64, len(reached))
for index, boundarySet := range reached {
merged[index] = slices.Sorted(maps.Keys(boundarySet))
}
return merged
}
// regroupAxis rewrites the aggregation onto boundaries, moving the count held in
// band i to targetBandIndexes[i] and summing where several bands land together.
// A band past the end of targetBandIndexes is the overflow, which stays the
// overflow on any axis.
func regroupAxis(aggBucket *AggregationBucket, boundaries []float64, targetBandIndexes []int) {
for _, series := range aggBucket.Series {
for _, point := range series.Values {
if len(point.Values) == 0 {
continue
}
regrouped := make([]float64, len(boundaries)+1)
for band, count := range point.Values {
if band >= len(targetBandIndexes) {
regrouped[len(boundaries)] += count
continue
}
regrouped[targetBandIndexes[band]] += count
}
point.Values = regrouped
}
}
aggBucket.Meta.Buckets = boundaries
}
// RealignHeatmapValues moves every point's per-bucket counts from the axis they
// were read against onto onto, matching on boundary rather than position. Two
// ranges of one query disagree on their axes when a histogram's `le` labels
// change partway through a window, or when one range's data never reached a
// band the other did.
func RealignHeatmapValues(series []*TimeSeries, from, onto []float64) {
if len(onto) == 0 || slices.Equal(from, onto) {
return
}
bandIndexByBoundary := make(map[float64]int, len(onto))
for band, boundary := range onto {
bandIndexByBoundary[boundary] = band
}
for _, s := range series {
for _, point := range s.Values {
if len(point.Values) == 0 {
continue
}
realigned := make([]float64, len(onto)+1)
for band, count := range point.Values {
if band >= len(from) {
realigned[len(onto)] = count
break
}
if targetBand, ok := bandIndexByBoundary[from[band]]; ok {
realigned[targetBand] = count
}
}
point.Values = realigned
}
}
}
// DownscaleHeatmapAxis folds a log axis bucketed at fromScale down to toScale,
// merging every 2^(fromScale-toScale) adjacent bands into one. The coarser
// boundaries are a subset of the finer ones, so the fold is exact.
func DownscaleHeatmapAxis(tsData *TimeSeriesData, fromScale, toScale int) {
if tsData == nil || toScale >= fromScale {
return
}
for _, aggBucket := range tsData.Aggregations {
downscaleAggregationAxis(aggBucket, fromScale, toScale)
}
}
func downscaleAggregationAxis(aggBucket *AggregationBucket, fromScale, toScale int) {
if aggBucket == nil || len(aggBucket.Meta.Buckets) == 0 {
return
}
factor := int(math.Exp2(float64(fromScale - toScale)))
// Bands merge by their index in the exponential mapping, not by position in
// Meta.Buckets, which lists only the boundaries some series reached.
coarse := make([]float64, 0, len(aggBucket.Meta.Buckets))
targetBandIndexes := make([]int, len(aggBucket.Meta.Buckets))
seen := make(map[float64]int, len(aggBucket.Meta.Buckets))
for band, boundary := range aggBucket.Meta.Buckets {
merged := coarsenHeatmapBoundary(boundary, fromScale, toScale, factor)
coarseBandIndex, ok := seen[merged]
if !ok {
coarseBandIndex = len(coarse)
coarse = append(coarse, merged)
seen[merged] = coarseBandIndex
}
targetBandIndexes[band] = coarseBandIndex
}
regroupAxis(aggBucket, coarse, targetBandIndexes)
}
// coarsenHeatmapBoundary moves a boundary from the fromScale exponential axis
// onto the toScale one. The zero band has no exponent to rescale and stays put.
func coarsenHeatmapBoundary(boundary float64, fromScale, toScale, factor int) float64 {
if boundary <= 0 || math.IsInf(boundary, 0) || math.IsNaN(boundary) {
return boundary
}
index := int(math.Round(math.Log2(boundary) * math.Exp2(float64(fromScale))))
merged := int(math.Ceil(float64(index) / float64(factor)))
return math.Exp2(float64(merged) / math.Exp2(float64(toScale)))
}
// DensifyHeatmapAxis fills in the bands no series reached, which are left out of
// Meta.Buckets entirely and would otherwise render with the two sides of a gap
// touching.
//
// Only a value-derived axis can be densified: its boundaries come from an index
// that is a pure function of the value, so the ones in between are known without
// having seen them. Nothing says what sits between two `le` labels.
func DensifyHeatmapAxis(tsData *TimeSeriesData, bucketing HeatmapBucketing) {
if tsData == nil {
return
}
for _, aggBucket := range tsData.Aggregations {
densifyAggregationAxis(aggBucket, bucketing)
}
}
func densifyAggregationAxis(aggBucket *AggregationBucket, bucketing HeatmapBucketing) {
if aggBucket == nil || len(aggBucket.Meta.Buckets) == 0 {
return
}
// The zero band holds everything at or below zero. It has no index on either
// axis and sits below every other boundary, so it keeps band 0 and the fill
// runs over the rest.
offset := 0
if aggBucket.Meta.Buckets[0] <= 0 {
offset = 1
}
positive := aggBucket.Meta.Buckets[offset:]
if len(positive) == 0 {
return
}
// Only finite boundaries have a band index, and the fill sizes a slice from
// one. Nothing should put +Inf or NaN on the axis, but bail if it happens.
indexes := make([]int, len(positive))
for i, boundary := range positive {
if math.IsInf(boundary, 0) || math.IsNaN(boundary) {
return
}
indexes[i] = bucketing.calculateBandIndex(boundary)
}
lowest, highest := slices.Min(indexes), slices.Max(indexes)
dense := append([]float64{}, aggBucket.Meta.Buckets[:offset]...)
for index := lowest; index <= highest; index++ {
dense = append(dense, bucketing.calculateBandBoundary(index))
}
if len(dense) == len(aggBucket.Meta.Buckets) {
return
}
// Bands map through their index rather than by matching boundaries, so a
// regenerated boundary differing from ClickHouse's in its last bit still
// lands on the band it came from.
targetBandIndexes := make([]int, len(aggBucket.Meta.Buckets))
for i, index := range indexes {
targetBandIndexes[i+offset] = index - lowest + offset
}
regroupAxis(aggBucket, dense, targetBandIndexes)
}
// calculateBandIndex and calculateBandBoundary are inverses, and match the
// expressions the statement builder renders: k * maxValue / numBuckets on a
// linear axis, 2^(k / 2^scale) on a log one.
func (h HeatmapBucketing) calculateBandIndex(boundary float64) int {
if h.Kind == BucketsKindLinear {
return int(math.Round(boundary * float64(h.NumBuckets) / h.MaxValue))
}
return int(math.Round(math.Log2(boundary) * math.Exp2(float64(h.LogScale))))
}
func (h HeatmapBucketing) calculateBandBoundary(index int) float64 {
if h.Kind == BucketsKindLinear {
return float64(index) * h.MaxValue / float64(h.NumBuckets)
}
return math.Exp2(float64(index) / math.Exp2(float64(h.LogScale)))
}
// BucketTimeSeriesValues turns one value per (series, timestamp) into heatmap
// cells on the axis bucketing describes, which is what ClickHouse does for a
// gauge or sum. A formula has no statement to carry the boundary expression, so
// its output is bucketed here instead. Every value counts the one series it came
// from, so a point ends up with a single occupied cell.
func BucketTimeSeriesValues(tsData *TimeSeriesData, bucketing HeatmapBucketing) {
if tsData == nil {
return
}
for _, aggBucket := range tsData.Aggregations {
bucketAggregationValues(aggBucket, bucketing)
}
}
func bucketAggregationValues(aggBucket *AggregationBucket, bucketing HeatmapBucketing) {
if aggBucket == nil {
return
}
// +Inf is the open-above overflow rather than a boundary of its own, and a
// NaN value has no band at all
boundarySet := map[float64]struct{}{}
for _, series := range aggBucket.Series {
for _, point := range series.Values {
boundary := bucketing.calculateValueBoundary(point.Value)
if !math.IsNaN(boundary) && !math.IsInf(boundary, 0) {
boundarySet[boundary] = struct{}{}
}
}
}
boundaries := slices.Sorted(maps.Keys(boundarySet))
bandIndexByBoundary := make(map[float64]int, len(boundaries))
for band, boundary := range boundaries {
bandIndexByBoundary[boundary] = band
}
for _, series := range aggBucket.Series {
for _, point := range series.Values {
boundary := bucketing.calculateValueBoundary(point.Value)
point.Values = make([]float64, len(boundaries)+1)
point.Value = 0
switch {
case math.IsNaN(boundary):
case math.IsInf(boundary, 1):
point.Values[len(boundaries)] = 1
default:
point.Values[bandIndexByBoundary[boundary]] = 1
}
}
}
aggBucket.Meta.Buckets = boundaries
}
// calculateValueBoundary renders the upper bound of the band value falls in. It
// is the Go side of the expression the statement builder emits and has to stay
// identical to it: a formula heatmap and a metric heatmap that disagreed here
// would put their bands in different places.
func (h HeatmapBucketing) calculateValueBoundary(value float64) float64 {
if h.Kind == BucketsKindLinear {
if value > h.MaxValue {
return math.Inf(1)
}
numBuckets := float64(h.NumBuckets)
index := math.Min(math.Max(math.Ceil(value*numBuckets/h.MaxValue), 1), numBuckets)
return index * h.MaxValue / numBuckets
}
if value <= 0 {
return 0
}
if value <= LowestLogBoundary {
return LowestLogBoundary
}
if value > HighestLogBoundary {
return math.Inf(1)
}
bandsPerDoubling := math.Exp2(float64(h.LogScale))
return math.Exp2(math.Ceil(math.Log2(value)*bandsPerDoubling) / bandsPerDoubling)
}

View File

@@ -0,0 +1,550 @@
package querybuildertypesv5
import (
"math"
"testing"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRealignHeatmapValues(t *testing.T) {
testCases := []struct {
description string
from []float64
onto []float64
values []float64
expectedValues []float64
}{
{
description: "an unchanged axis is left alone",
from: []float64{5, 10},
onto: []float64{5, 10},
values: []float64{1, 2, 3},
expectedValues: []float64{1, 2, 3},
},
{
description: "an inserted bucket shifts the counts above it",
from: []float64{5, 10},
onto: []float64{2, 5, 10},
values: []float64{1, 2, 3},
expectedValues: []float64{0, 1, 2, 3},
},
{
description: "a dropped bucket loses its counts but the overflow survives",
from: []float64{5, 10, 25},
onto: []float64{5, 25},
values: []float64{1, 2, 3, 4},
expectedValues: []float64{1, 3, 4},
},
{
description: "an axis with nothing in common keeps only the overflow",
from: []float64{5, 10},
onto: []float64{100, 200},
values: []float64{1, 2, 3},
expectedValues: []float64{0, 0, 3},
},
{
description: "counts beyond the axis they were read against are dropped",
from: []float64{5},
onto: []float64{5, 10},
values: []float64{1, 2, 3},
expectedValues: []float64{1, 0, 2},
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
series := []*TimeSeries{{
Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Values: testCase.values}},
}}
RealignHeatmapValues(series, testCase.from, testCase.onto)
require.Len(t, series[0].Values, 1)
assert.Equal(t, testCase.expectedValues, series[0].Values[0].Values)
})
}
}
func TestRealignHeatmapValuesLeavesNonHeatmapPointsAlone(t *testing.T) {
series := []*TimeSeries{{
Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Value: 42}},
}}
RealignHeatmapValues(series, nil, []float64{5, 10})
assert.Equal(t, float64(42), series[0].Values[0].Value)
assert.Empty(t, series[0].Values[0].Values)
}
func TestRealignHeatmapValuesWithoutTargetAxis(t *testing.T) {
series := []*TimeSeries{{
Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Values: []float64{1, 2}}},
}}
RealignHeatmapValues(series, []float64{5, 10}, nil)
assert.Equal(t, []float64{1, 2}, series[0].Values[0].Values)
}
func TestDownscaleHeatmapAxis(t *testing.T) {
testCases := []struct {
description string
fromScale int
toScale int
buckets []float64
values []float64
expectedBuckets []float64
expectedValues []float64
}{
{
description: "four scale-4 bands merge into one scale-2 band",
fromScale: 4,
toScale: 2,
buckets: []float64{
math.Exp2(0),
math.Exp2(1.0 / 16),
math.Exp2(2.0 / 16),
math.Exp2(3.0 / 16),
math.Exp2(4.0 / 16),
},
values: []float64{1, 2, 3, 4, 5, 6},
expectedBuckets: []float64{math.Exp2(0), math.Exp2(1.0 / 4)},
expectedValues: []float64{1, 14, 6},
},
{
description: "bands below 1 fold onto the same coarse boundary",
fromScale: 4,
toScale: 2,
buckets: []float64{math.Exp2(-3.0 / 16), math.Exp2(-2.0 / 16), math.Exp2(-1.0 / 16)},
values: []float64{1, 2, 3, 4},
expectedBuckets: []float64{math.Exp2(0)},
expectedValues: []float64{6, 4},
},
{
description: "the zero band keeps its own slot",
fromScale: 4,
toScale: 2,
buckets: []float64{0, math.Exp2(1.0 / 16), math.Exp2(4.0 / 16)},
values: []float64{7, 1, 2, 3},
expectedBuckets: []float64{0, math.Exp2(1.0 / 4)},
expectedValues: []float64{7, 3, 3},
},
{
// at scale 0 the whole doubling above 1 is a single band, and 2^(16/16)
// is its upper bound rather than the start of the next one
description: "a doubling's worth of bands collapses into one at scale 0",
fromScale: 4,
toScale: 0,
buckets: []float64{math.Exp2(1.0 / 16), math.Exp2(8.0 / 16), math.Exp2(16.0 / 16)},
values: []float64{1, 2, 3, 4},
expectedBuckets: []float64{math.Exp2(1)},
expectedValues: []float64{6, 4},
},
{
description: "the finest scale is left alone",
fromScale: 4,
toScale: 4,
buckets: []float64{math.Exp2(1.0 / 16), math.Exp2(2.0 / 16)},
values: []float64{1, 2, 3},
expectedBuckets: []float64{math.Exp2(1.0 / 16), math.Exp2(2.0 / 16)},
expectedValues: []float64{1, 2, 3},
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Meta: AggregationMeta{Buckets: testCase.buckets},
Series: []*TimeSeries{{
Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Values: testCase.values}},
}},
}},
}
DownscaleHeatmapAxis(tsData, testCase.fromScale, testCase.toScale)
aggBucket := tsData.Aggregations[0]
assert.Equal(t, testCase.expectedBuckets, aggBucket.Meta.Buckets)
assert.Equal(t, testCase.expectedValues, aggBucket.Series[0].Values[0].Values)
})
}
}
func TestDownscaleHeatmapAxisKeepsTheTotalCount(t *testing.T) {
buckets := make([]float64, 0, 64)
values := make([]float64, 0, 65)
for index := range 64 {
buckets = append(buckets, math.Exp2(float64(index)/16))
values = append(values, float64(index))
}
values = append(values, 100)
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Meta: AggregationMeta{Buckets: buckets},
Series: []*TimeSeries{{
Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Values: values}},
}},
}},
}
var before float64
for _, count := range values {
before += count
}
DownscaleHeatmapAxis(tsData, MaxLogScale, 1)
aggBucket := tsData.Aggregations[0]
// bands 0..63 fold onto ceil(k/8), so 0..8: the boundary at 2^0 keeps a band
// of its own and the four doublings above it take two each
assert.Len(t, aggBucket.Meta.Buckets, 9)
assert.Len(t, aggBucket.Series[0].Values[0].Values, 10)
var after float64
for _, count := range aggBucket.Series[0].Values[0].Values {
after += count
}
assert.Equal(t, before, after)
}
func TestDensifyHeatmapAxis(t *testing.T) {
testCases := []struct {
description string
bucketing HeatmapBucketing
buckets []float64
values []float64
expectedBuckets []float64
expectedValues []float64
}{
{
description: "an already contiguous log axis is left alone",
bucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: 4},
buckets: []float64{math.Exp2(1.0 / 16), math.Exp2(2.0 / 16), math.Exp2(3.0 / 16)},
values: []float64{1, 2, 3, 4},
expectedBuckets: []float64{
math.Exp2(1.0 / 16),
math.Exp2(2.0 / 16),
math.Exp2(3.0 / 16),
},
expectedValues: []float64{1, 2, 3, 4},
},
{
description: "log bands nothing reached are filled in with zero",
bucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: 4},
buckets: []float64{math.Exp2(1.0 / 16), math.Exp2(4.0 / 16)},
values: []float64{5, 7, 9},
expectedBuckets: []float64{
math.Exp2(1.0 / 16),
math.Exp2(2.0 / 16),
math.Exp2(3.0 / 16),
math.Exp2(4.0 / 16),
},
expectedValues: []float64{5, 0, 0, 7, 9},
},
{
description: "the zero band keeps the lowest slot and the fill starts above it",
bucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: 4},
buckets: []float64{0, math.Exp2(1.0 / 16), math.Exp2(3.0 / 16)},
values: []float64{4, 5, 6, 7},
expectedBuckets: []float64{0, math.Exp2(1.0 / 16), math.Exp2(2.0 / 16), math.Exp2(3.0 / 16)},
expectedValues: []float64{4, 5, 0, 6, 7},
},
{
description: "a log axis spanning a doubling gets every band between",
bucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: 1},
buckets: []float64{math.Exp2(0), math.Exp2(1)},
values: []float64{1, 2, 3},
expectedBuckets: []float64{math.Exp2(0), math.Exp2(0.5), math.Exp2(1)},
expectedValues: []float64{1, 0, 2, 3},
},
{
description: "linear bands nothing reached are filled in with zero",
bucketing: HeatmapBucketing{Kind: BucketsKindLinear, MaxValue: 500, NumBuckets: 25},
buckets: []float64{20, 100},
values: []float64{3, 4, 5},
expectedBuckets: []float64{20, 40, 60, 80, 100},
expectedValues: []float64{3, 0, 0, 0, 4, 5},
},
{
description: "a single band has nothing to fill in",
bucketing: HeatmapBucketing{Kind: BucketsKindLinear, MaxValue: 500, NumBuckets: 25},
buckets: []float64{100},
values: []float64{1, 2},
expectedBuckets: []float64{100},
expectedValues: []float64{1, 2},
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Meta: AggregationMeta{Buckets: testCase.buckets},
Series: []*TimeSeries{{
Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Values: testCase.values}},
}},
}},
}
DensifyHeatmapAxis(tsData, testCase.bucketing)
aggBucket := tsData.Aggregations[0]
assert.Equal(t, testCase.expectedBuckets, aggBucket.Meta.Buckets)
assert.Equal(t, testCase.expectedValues, aggBucket.Series[0].Values[0].Values)
})
}
}
func TestDensifyHeatmapAxisKeepsTheTotalCount(t *testing.T) {
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Meta: AggregationMeta{Buckets: []float64{0, math.Exp2(2.0 / 16), math.Exp2(37.0 / 16)}},
Series: []*TimeSeries{{
Values: []*TimeSeriesValue{
{Timestamp: 1710000000000, Values: []float64{2, 3, 5, 7}},
{Timestamp: 1710000060000, Values: []float64{11, 13, 17, 19}},
},
}},
}},
}
DensifyHeatmapAxis(tsData, HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale})
aggBucket := tsData.Aggregations[0]
// the zero band plus every band from index 2 to index 37
assert.Len(t, aggBucket.Meta.Buckets, 37)
for _, point := range aggBucket.Series[0].Values {
assert.Len(t, point.Values, 38)
}
assert.Equal(t, float64(2+3+5+7), sumHeatmapCounts(aggBucket.Series[0].Values[0].Values))
assert.Equal(t, float64(11+13+17+19), sumHeatmapCounts(aggBucket.Series[0].Values[1].Values))
}
func sumHeatmapCounts(values []float64) float64 {
var total float64
for _, count := range values {
total += count
}
return total
}
func TestBucketTimeSeriesValues(t *testing.T) {
testCases := []struct {
description string
bucketing HeatmapBucketing
values []float64
expectedBuckets []float64
expectedValues [][]float64
}{
{
description: "log values land on the band above them",
bucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale},
values: []float64{1, 2, 3},
expectedBuckets: []float64{
math.Exp2(0),
math.Exp2(1),
math.Exp2(26.0 / 16),
},
expectedValues: [][]float64{
{1, 0, 0, 0},
{0, 1, 0, 0},
{0, 0, 1, 0},
},
},
{
// the log axis has no band below zero, so both report the boundary
// that means "everything at or below zero"
description: "zero and negative values share the lowest log band",
bucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale},
values: []float64{-5, 0, 1},
expectedBuckets: []float64{0, 1},
expectedValues: [][]float64{
{1, 0, 0},
{1, 0, 0},
{0, 1, 0},
},
},
{
description: "a linear value above maxValue lands in the overflow",
bucketing: HeatmapBucketing{Kind: BucketsKindLinear, MaxValue: 100, NumBuckets: 4},
values: []float64{30, 100, 150, 0},
expectedBuckets: []float64{25, 50, 100},
expectedValues: [][]float64{
{0, 1, 0, 0},
{0, 0, 1, 0},
{0, 0, 0, 1},
{1, 0, 0, 0},
},
},
{
description: "a value with no band occupies no cell",
bucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale},
values: []float64{math.NaN(), 1},
expectedBuckets: []float64{1},
expectedValues: [][]float64{
{0, 0},
{1, 0},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
points := make([]*TimeSeriesValue, 0, len(testCase.values))
for index, value := range testCase.values {
points = append(points, &TimeSeriesValue{
Timestamp: 1710000000000 + int64(index)*60000,
Value: value,
})
}
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Series: []*TimeSeries{{Values: points}},
}},
}
BucketTimeSeriesValues(tsData, testCase.bucketing)
aggBucket := tsData.Aggregations[0]
assert.Equal(t, testCase.expectedBuckets, aggBucket.Meta.Buckets)
for index, point := range aggBucket.Series[0].Values {
assert.Equal(t, testCase.expectedValues[index], point.Values, "point %d", index)
assert.Zero(t, point.Value, "point %d keeps its scalar value", index)
}
})
}
}
func TestBucketTimeSeriesValuesSharesOneAxisAcrossSeries(t *testing.T) {
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Series: []*TimeSeries{
{
Labels: []*Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host.name"}, Value: "a"}},
Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Value: 1}},
},
{
Labels: []*Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host.name"}, Value: "b"}},
Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Value: 4}},
},
},
}},
}
BucketTimeSeriesValues(tsData, HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale})
aggBucket := tsData.Aggregations[0]
assert.Equal(t, []float64{math.Exp2(0), math.Exp2(2)}, aggBucket.Meta.Buckets)
// each series counts itself, and the panel adds up whichever are selected
assert.Equal(t, []float64{1, 0, 0}, aggBucket.Series[0].Values[0].Values)
assert.Equal(t, []float64{0, 1, 0}, aggBucket.Series[1].Values[0].Values)
}
func TestBucketTimeSeriesValuesMatchesTheStatementBuilderBoundaries(t *testing.T) {
// the same expressions the statement builder renders, evaluated in Go:
// multiIf(value <= 0, 0, pow(2, ceil(log2(value) * 16) / 16)) and
// multiIf(value > max, +Inf, least(greatest(ceil(value * n / max), 1), n) * max / n)
logBucketing := HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale}
assert.Equal(t, math.Exp2(math.Ceil(math.Log2(37)*16)/16), logBucketing.calculateValueBoundary(37))
assert.Equal(t, 0.0, logBucketing.calculateValueBoundary(-1))
linearBucketing := HeatmapBucketing{Kind: BucketsKindLinear, MaxValue: 500, NumBuckets: 25}
assert.Equal(t, math.Ceil(37.0*25/500)*500/25, linearBucketing.calculateValueBoundary(37))
assert.Equal(t, 1*500.0/25, linearBucketing.calculateValueBoundary(0))
assert.True(t, math.IsInf(linearBucketing.calculateValueBoundary(501), 1))
}
func TestHeatmapBoundaryForClampsTheLogAxis(t *testing.T) {
bucketing := HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale}
// without the clamp the band index runs off to -inf as a positive value
// approaches zero, and the axis fill follows it
assert.Equal(t, LowestLogBoundary, bucketing.calculateValueBoundary(1e-30))
assert.Equal(t, LowestLogBoundary, bucketing.calculateValueBoundary(math.SmallestNonzeroFloat64))
assert.Equal(t, LowestLogBoundary, bucketing.calculateValueBoundary(LowestLogBoundary))
assert.True(t, math.IsInf(bucketing.calculateValueBoundary(1e30), 1))
assert.True(t, math.IsInf(bucketing.calculateValueBoundary(math.MaxFloat64), 1))
assert.Equal(t, HighestLogBoundary, bucketing.calculateValueBoundary(HighestLogBoundary))
// zero and negatives keep their own band below the floor
assert.Equal(t, 0.0, bucketing.calculateValueBoundary(0))
assert.Equal(t, 0.0, bucketing.calculateValueBoundary(-5))
// anything in between is untouched
assert.Equal(t, math.Exp2(math.Ceil(math.Log2(37)*16)/16), bucketing.calculateValueBoundary(37))
}
func TestLogAxisClampsStayOnTheGridAtEveryScale(t *testing.T) {
// a coarser fold must land the clamped ends on real boundaries, which holds
// because both indexes are powers of two
for scale := MinLogScale; scale <= MaxLogScale; scale++ {
bandsPerDoubling := math.Exp2(float64(scale))
for _, boundary := range []float64{LowestLogBoundary, HighestLogBoundary} {
index := math.Log2(boundary) * bandsPerDoubling
assert.Equal(t, math.Trunc(index), index, "scale %d, boundary %g", scale, boundary)
}
}
}
func TestDensifyHeatmapAxisIsBoundedByTheFloor(t *testing.T) {
bucketing := HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale}
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Series: []*TimeSeries{{
Values: []*TimeSeriesValue{
{Timestamp: 1710000000000, Value: 1e-30},
{Timestamp: 1710000060000, Value: 1000},
},
}},
}},
}
BucketTimeSeriesValues(tsData, bucketing)
DensifyHeatmapAxis(tsData, bucketing)
// 1e-30 clamps to the floor, so the fill spans MinLogBandIndex upwards
// rather than chasing that value's own index near -1594
buckets := tsData.Aggregations[0].Meta.Buckets
highest := bucketing.calculateBandIndex(bucketing.calculateValueBoundary(1000))
assert.Equal(t, LowestLogBoundary, buckets[0])
assert.Len(t, buckets, highest-MinLogBandIndex+1)
}
func TestDensifyHeatmapAxisSkipsANonFiniteBoundary(t *testing.T) {
// the overflow is the slot past the axis, never a boundary on it; a bad one
// would otherwise size the fill from a garbage band index
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Meta: AggregationMeta{Buckets: []float64{1, math.Inf(1)}},
Series: []*TimeSeries{{Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Values: []float64{1, 1, 0}}}}},
}},
}
DensifyHeatmapAxis(tsData, HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale})
assert.Equal(t, []float64{1, math.Inf(1)}, tsData.Aggregations[0].Meta.Buckets)
}
func TestDensifyHeatmapAxisWorstCaseSpan(t *testing.T) {
bucketing := HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale}
tsData := &TimeSeriesData{
Aggregations: []*AggregationBucket{{
Meta: AggregationMeta{Buckets: []float64{LowestLogBoundary, HighestLogBoundary}},
Series: []*TimeSeries{{Values: []*TimeSeriesValue{{Timestamp: 1710000000000, Values: []float64{1, 1, 0}}}}},
}},
}
DensifyHeatmapAxis(tsData, bucketing)
// the widest axis the bucketing can produce, whatever the data does
assert.Len(t, tsData.Aggregations[0].Meta.Buckets, MaxLogBandIndex-MinLogBandIndex+1)
}

View File

@@ -0,0 +1,656 @@
package querybuildertypesv5
import (
"encoding/json"
"math"
"testing"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidateHeatmapRequest(t *testing.T) {
testCases := []struct {
description string
request QueryRangeRequest
expectedErrContains string
}{
{
description: "a single metrics builder query with increase and sum is accepted",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "http.server.request.duration",
TimeAggregation: metrictypes.TimeAggregationIncrease,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
},
}}},
},
},
{
// the axis comes from the `le` labels, so the space aggregation has
// nothing left to pick out and a percentile draws the same heatmap a
// count would
description: "percentile space aggregation is accepted",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "http.server.request.duration",
TimeAggregation: metrictypes.TimeAggregationRate,
SpaceAggregation: metrictypes.SpaceAggregationPercentile95,
}},
},
}}},
},
},
{
// the statement builder strips `le` from a histogram's groupBy before
// re-adding it for the bucket CTE, the same as any other histogram
// query, so it needs no heatmap rule of its own
description: "le in groupBy is accepted",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "http.server.request.duration",
TimeAggregation: metrictypes.TimeAggregationIncrease,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
GroupBy: []GroupByKey{{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "le"},
}},
},
}}},
},
},
{
description: "having is refused",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "http.server.request.duration",
TimeAggregation: metrictypes.TimeAggregationIncrease,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
Having: &Having{Expression: "sum(http.server.request.duration) > 10"},
},
}}},
},
expectedErrContains: "having is not supported",
},
{
description: "functions are refused",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "http.server.request.duration",
TimeAggregation: metrictypes.TimeAggregationIncrease,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
Functions: []Function{{Name: FunctionNameAbsolute}},
},
}}},
},
expectedErrContains: "functions are not supported",
},
{
// a promql histogram carries `le` through the matrix as an ordinary
// label, which is the same axis the builder's histogram path reads
description: "a promql query is accepted",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypePromQL,
Spec: PromQuery{Name: "A", Query: "sum by (le) (increase(signoz_latency_bucket[5m]))"},
}}},
},
},
{
description: "bucket options alongside a promql query are refused",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
BucketOptions: &BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{}},
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypePromQL,
Spec: PromQuery{Name: "A", Query: "sum by (le) (increase(signoz_latency_bucket[5m]))"},
}}},
},
expectedErrContains: "bucketOptions are not supported for promql heatmap requests",
},
{
// a clickhouse query's rows are read by request type like any other,
// so one shaped as heatmap cells renders without the builder
description: "a clickhouse query is accepted",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypeClickHouseSQL,
Spec: ClickHouseQuery{Name: "A", Query: "SELECT ts, bucket, value FROM cells"},
}}},
},
},
{
description: "a formula over disabled builder queries is accepted",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{
{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Disabled: true,
Aggregations: []MetricAggregation{{
MetricName: "system.memory.usage",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
},
},
{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "B",
Signal: telemetrytypes.SignalMetrics,
Disabled: true,
Aggregations: []MetricAggregation{{
MetricName: "system.memory.limit",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
},
},
{
Type: QueryTypeFormula,
Spec: QueryBuilderFormula{Name: "F1", Expression: "A / B"},
},
}},
},
},
{
description: "a formula alongside an enabled query is refused",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{
{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "system.memory.usage",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
},
},
{
Type: QueryTypeFormula,
Spec: QueryBuilderFormula{Name: "F1", Expression: "A * 2"},
},
}},
},
expectedErrContains: "exactly one enabled query",
},
{
description: "functions on a formula are refused",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{
{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Disabled: true,
Aggregations: []MetricAggregation{{
MetricName: "system.memory.usage",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
},
},
{
Type: QueryTypeFormula,
Spec: QueryBuilderFormula{
Name: "F1",
Expression: "A * 2",
Functions: []Function{{Name: FunctionNameAbsolute}},
},
},
}},
},
expectedErrContains: "functions are not supported",
},
{
// a disabled query is a formula input, so its functions still reach
// the cells the heatmap draws
description: "functions on a disabled formula input are refused",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{
{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Disabled: true,
Aggregations: []MetricAggregation{{
MetricName: "system.memory.usage",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
Functions: []Function{{Name: FunctionNameAbsolute}},
},
},
{
Type: QueryTypeFormula,
Spec: QueryBuilderFormula{Name: "F1", Expression: "A * 2"},
},
}},
},
expectedErrContains: "functions are not supported",
},
{
description: "a disabled clickhouse query leaves nothing to draw",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypeClickHouseSQL,
Spec: ClickHouseQuery{Name: "A", Query: "SELECT 1", Disabled: true},
}}},
},
expectedErrContains: "exactly one enabled query",
},
{
description: "two enabled queries are refused",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{
{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "http.server.request.duration",
TimeAggregation: metrictypes.TimeAggregationIncrease,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
},
},
{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "B",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "http.server.request.body.size",
TimeAggregation: metrictypes.TimeAggregationIncrease,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
},
},
}},
},
expectedErrContains: "exactly one enabled query",
},
{
description: "fillGaps is refused",
request: QueryRangeRequest{
Start: 1710000000000,
End: 1710003600000,
RequestType: RequestTypeHeatmap,
FormatOptions: &FormatOptions{FillGaps: true},
CompositeQuery: CompositeQuery{Queries: []QueryEnvelope{{
Type: QueryTypeBuilder,
Spec: QueryBuilderQuery[MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []MetricAggregation{{
MetricName: "http.server.request.duration",
TimeAggregation: metrictypes.TimeAggregationIncrease,
SpaceAggregation: metrictypes.SpaceAggregationSum,
}},
},
}}},
},
expectedErrContains: "fillGaps is not supported",
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
err := testCase.request.Validate()
if testCase.expectedErrContains == "" {
require.NoError(t, err)
return
}
require.Error(t, err)
assert.Contains(t, err.Error(), testCase.expectedErrContains)
})
}
}
func TestHeatmapRequestTypeIsAccepted(t *testing.T) {
var requestType RequestType
require.NoError(t, requestType.UnmarshalJSON([]byte(`"heatmap"`)))
assert.Equal(t, RequestTypeHeatmap, requestType)
assert.True(t, requestType.IsAggregation())
}
func TestResolveBucketOptions(t *testing.T) {
coarseScale := 2
testCases := []struct {
description string
options *BucketOptions
expectedBucketing HeatmapBucketing
expectedLogScale int
}{
{
description: "an absent config defaults to the finest log axis",
options: nil,
expectedBucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale, NumBuckets: DefaultNumBuckets},
expectedLogScale: MaxLogScale,
},
{
description: "a linear spec carries its cap and count through",
options: &BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{MaxValue: 1024, NumBuckets: 20}},
expectedBucketing: HeatmapBucketing{Kind: BucketsKindLinear, LogScale: MaxLogScale, MaxValue: 1024, NumBuckets: 20},
expectedLogScale: MaxLogScale,
},
{
description: "a linear spec without a count takes the default",
options: &BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{MaxValue: 1024}},
expectedBucketing: HeatmapBucketing{Kind: BucketsKindLinear, LogScale: MaxLogScale, MaxValue: 1024, NumBuckets: DefaultNumBuckets},
expectedLogScale: MaxLogScale,
},
{
description: "a coarser scale is kept out of the axis clickhouse builds",
options: &BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{Scale: &coarseScale}},
expectedBucketing: HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale, NumBuckets: DefaultNumBuckets},
expectedLogScale: 2,
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
assert.Equal(t, testCase.expectedBucketing, testCase.options.ResolveBucketOptions())
assert.Equal(t, testCase.expectedLogScale, testCase.options.ResolveLogScale())
})
}
}
func TestUnmarshalBucketOptions(t *testing.T) {
scale := 2
testCases := []struct {
description string
body string
expectedOptions BucketOptions
expectedErrContains string
}{
{
description: "a linear kind decodes its own spec",
body: `{"kind":"linear","spec":{"maxValue":500,"numBuckets":25}}`,
expectedOptions: BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{MaxValue: 500, NumBuckets: 25}},
},
{
description: "a log kind decodes its own spec",
body: `{"kind":"log","spec":{"scale":2}}`,
expectedOptions: BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{Scale: &scale}},
},
{
description: "an empty log spec asks for the defaults",
body: `{"kind":"log","spec":{}}`,
expectedOptions: BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{}},
},
{
description: "a kind with no spec beside it is refused",
body: `{"kind":"log"}`,
expectedErrContains: "bucketOptions spec is required",
},
{
description: "an unknown kind is refused",
body: `{"kind":"quadratic","spec":{}}`,
expectedErrContains: "invalid bucketOptions kind",
},
{
description: "a missing kind is refused",
body: `{"spec":{"maxValue":500}}`,
expectedErrContains: "invalid bucketOptions kind",
},
{
// the kind picks the spec, so a field belonging to the other one is a
// typo rather than something to quietly drop
description: "a log field under a linear kind is refused",
body: `{"kind":"linear","spec":{"maxValue":500,"scale":2}}`,
expectedErrContains: "scale",
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
var options BucketOptions
err := json.Unmarshal([]byte(testCase.body), &options)
if testCase.expectedErrContains != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), testCase.expectedErrContains)
return
}
require.NoError(t, err)
assert.Equal(t, testCase.expectedOptions, options)
})
}
}
func TestValidateBucketOptions(t *testing.T) {
tooFine := MaxLogScale + 1
tooCoarse := MinLogScale - 1
coarseScale := 2
testCases := []struct {
description string
options *BucketOptions
expectedErrContains string
}{
{
description: "an absent config is accepted",
options: nil,
},
{
description: "a full linear spec is accepted",
options: &BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{MaxValue: 1024, NumBuckets: 32}},
},
{
description: "a log spec with a coarser scale is accepted",
options: &BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{Scale: &coarseScale}},
},
{
description: "an empty log spec is accepted",
options: &BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{}},
},
{
description: "a bucket count above the cap is refused",
options: &BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{MaxValue: 1024, NumBuckets: MaxNumBuckets + 1}},
expectedErrContains: "numBuckets must be between",
},
{
description: "a kind with no spec behind it is refused",
options: &BucketOptions{Kind: BucketsKind{valuer.NewString("quadratic")}},
expectedErrContains: "invalid bucketOptions kind",
},
{
description: "a non-finite maxValue is refused",
options: &BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{MaxValue: math.NaN()}},
expectedErrContains: "finite maxValue greater than 0",
},
{
// a linear spec that omits maxValue decodes to zero, which is the
// same refusal
description: "a maxValue at zero is refused",
options: &BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{}},
expectedErrContains: "finite maxValue greater than 0",
},
{
description: "a scale finer than clickhouse buckets at is refused",
options: &BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{Scale: &tooFine}},
expectedErrContains: "scale must be between",
},
{
description: "a scale below the coarsest axis is refused",
options: &BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{Scale: &tooCoarse}},
expectedErrContains: "scale must be between",
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
err := testCase.options.validateBucketOptions()
if testCase.expectedErrContains == "" {
require.NoError(t, err)
return
}
require.Error(t, err)
assert.Contains(t, err.Error(), testCase.expectedErrContains)
})
}
}
func TestResolveHeatmapBucketing(t *testing.T) {
coarseScale := 1
testCases := []struct {
description string
aggregation MetricAggregation
bucketOptions *BucketOptions
expectedBucketing *HeatmapBucketing
expectedErrContains string
}{
{
description: "a histogram buckets on its own le labels",
aggregation: MetricAggregation{MetricName: "http.server.request.duration", Type: metrictypes.HistogramType},
bucketOptions: nil,
expectedBucketing: nil,
},
{
description: "bucketOptions alongside a histogram are refused",
aggregation: MetricAggregation{MetricName: "http.server.request.duration", Type: metrictypes.HistogramType},
bucketOptions: &BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{MaxValue: 500}},
expectedErrContains: "bucketOptions are not supported for histogram metrics",
},
{
description: "a gauge with no options gets the default log axis",
aggregation: MetricAggregation{MetricName: "system.memory.usage", Type: metrictypes.GaugeType},
bucketOptions: nil,
expectedBucketing: &HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale, NumBuckets: DefaultNumBuckets},
},
{
description: "a sum takes the requested linear axis",
aggregation: MetricAggregation{MetricName: "http.server.request.count", Type: metrictypes.SumType},
bucketOptions: &BucketOptions{Kind: BucketsKindLinear, Spec: LinearBucketsSpec{MaxValue: 500, NumBuckets: 25}},
expectedBucketing: &HeatmapBucketing{Kind: BucketsKindLinear, LogScale: MaxLogScale, MaxValue: 500, NumBuckets: 25},
},
{
description: "a coarser scale does not change the axis clickhouse builds",
aggregation: MetricAggregation{MetricName: "system.memory.usage", Type: metrictypes.GaugeType},
bucketOptions: &BucketOptions{Kind: BucketsKindLog, Spec: LogBucketsSpec{Scale: &coarseScale}},
expectedBucketing: &HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale, NumBuckets: DefaultNumBuckets},
},
{
description: "an unresolved type is refused",
aggregation: MetricAggregation{MetricName: "never.seen", Type: metrictypes.UnspecifiedType},
expectedErrContains: "no type is recorded",
},
{
description: "an exponential histogram is refused",
aggregation: MetricAggregation{MetricName: "http.server.request.duration", Type: metrictypes.ExpHistogramType},
expectedErrContains: "keeps its bucket counts in a sketch column",
},
{
description: "a summary buckets like a gauge",
aggregation: MetricAggregation{MetricName: "go.gc.duration", Type: metrictypes.SummaryType},
bucketOptions: nil,
expectedBucketing: &HeatmapBucketing{Kind: BucketsKindLog, LogScale: MaxLogScale, NumBuckets: DefaultNumBuckets},
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
err := testCase.aggregation.ResolveHeatmapBucketing(testCase.bucketOptions)
if testCase.expectedErrContains != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), testCase.expectedErrContains)
assert.Contains(t, err.Error(), testCase.aggregation.MetricName)
return
}
require.NoError(t, err)
assert.Equal(t, testCase.expectedBucketing, testCase.aggregation.HeatmapBucketing)
})
}
}

View File

@@ -397,6 +397,147 @@ type QueryRangeRequest struct {
PromQLProvider string `json:"-"`
FormatOptions *FormatOptions `json:"formatOptions,omitempty"`
// BucketOptions shapes the bucket axis for heatmap requests, and is refused
// rather than ignored for the metrics that bucket on their own `le` labels.
BucketOptions *BucketOptions `json:"bucketOptions,omitempty"`
}
// BucketOptions configures how a value range is divided into heatmap buckets.
type BucketOptions struct {
Kind BucketsKind `json:"kind"`
// Spec holds the LinearBucketsSpec or LogBucketsSpec for Kind.
Spec any `json:"spec"`
}
const (
DefaultNumBuckets = 60
MaxNumBuckets = 512
// MaxLogScale is the resolution ClickHouse buckets every log heatmap at:
// 2^MaxLogScale bands per doubling. It is both the default and the finest
// available, since a coarser LogBucketsSpec.Scale folds down from it.
MaxLogScale = 4
// MinLogScale is one band per 16x, the coarsest axis worth rendering.
MinLogScale = -4
)
type BucketsKind struct {
valuer.String
}
var (
BucketsKindLinear = BucketsKind{valuer.NewString("linear")}
BucketsKindLog = BucketsKind{valuer.NewString("log")}
)
// Enum implements jsonschema.Enum.
func (BucketsKind) Enum() []any {
return []any{
BucketsKindLinear,
BucketsKindLog,
}
}
// LinearBucketsSpec divides (0, MaxValue] into NumBuckets equal bands.
type LinearBucketsSpec struct {
// Everything above MaxValue is counted in the trailing overflow band. Evenly
// spaced boundaries have no top to divide without it, so it is required.
MaxValue float64 `json:"maxValue" required:"true"`
// DefaultNumBuckets applies when unset.
NumBuckets int `json:"numBuckets,omitempty"`
}
// LogBucketsSpec spaces boundaries at 2^Scale bands per doubling, the mapping
// an exponential histogram uses.
type LogBucketsSpec struct {
// ClickHouse always buckets at MaxLogScale and the surplus is folded away
// afterwards, so every Scale reads the same cache entry. MaxLogScale applies
// when unset.
Scale *int `json:"scale,omitempty"`
}
// bucketOptionsLinear and bucketOptionsLog are the OpenAPI schemas for the two
// BucketOptions variants. They have to be named types: the reflector turns an
// anonymous one into an inline subschema, leaving the discriminator mapping in
// PrepareJSONSchema pointing at components that were never emitted. `kind` is
// required:"true" on both so oapi-codegen renders the discriminator non-pointer.
type bucketOptionsLinear struct {
Kind BucketsKind `json:"kind" required:"true" description:"How the boundaries are spaced."`
Spec LinearBucketsSpec `json:"spec" required:"true" description:"The evenly spaced bucket specification."`
}
type bucketOptionsLog struct {
Kind BucketsKind `json:"kind" required:"true" description:"How the boundaries are spaced."`
Spec LogBucketsSpec `json:"spec" required:"true" description:"The logarithmic bucket specification."`
}
var _ jsonschema.OneOfExposer = BucketOptions{}
func (BucketOptions) JSONSchemaOneOf() []any {
return []any{
bucketOptionsLinear{},
bucketOptionsLog{},
}
}
var _ jsonschema.Preparer = BucketOptions{}
// PrepareJSONSchema marks the options as a `kind`-discriminated union;
// signoz.attachDiscriminators promotes it and strips the base properties.
func (BucketOptions) PrepareJSONSchema(s *jsonschema.Schema) error {
if s.ExtraProperties == nil {
s.ExtraProperties = map[string]any{}
}
s.ExtraProperties["x-signoz-discriminator"] = map[string]any{
"propertyName": "kind",
"mapping": map[string]string{
BucketsKindLinear.StringValue(): "#/components/schemas/Querybuildertypesv5BucketOptionsLinear",
BucketsKindLog.StringValue(): "#/components/schemas/Querybuildertypesv5BucketOptionsLog",
},
}
return nil
}
func (b *BucketOptions) UnmarshalJSON(data []byte) error {
var shadow struct {
Kind BucketsKind `json:"kind"`
Spec json.RawMessage `json:"spec"`
}
if err := binding.JSON.BindBody(bytes.NewReader(data), &shadow, binding.WithDisallowUnknownFields(true)); err != nil {
return err
}
b.Kind = shadow.Kind
// An absent spec is a malformed pair rather than a request for defaults;
// `"spec": {}` asks for those.
if len(shadow.Spec) == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"bucketOptions spec is required, use an empty object for the kind's defaults")
}
switch shadow.Kind {
case BucketsKindLinear:
var spec LinearBucketsSpec
if err := binding.JSON.BindBody(bytes.NewReader(shadow.Spec), &spec, binding.WithDisallowUnknownFields(true), binding.WithUnknownFieldContext("linear buckets spec")); err != nil {
return err
}
b.Spec = spec
case BucketsKindLog:
var spec LogBucketsSpec
if err := binding.JSON.BindBody(bytes.NewReader(shadow.Spec), &spec, binding.WithDisallowUnknownFields(true), binding.WithUnknownFieldContext("log buckets spec")); err != nil {
return err
}
b.Spec = spec
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"invalid bucketOptions kind %q, expected one of linear, log", shadow.Kind.StringValue())
}
return nil
}
// PrepareJSONSchema adds description to the QueryRangeRequest schema.

View File

@@ -19,11 +19,11 @@ func (r *RequestType) UnmarshalJSON(data []byte) error {
}
v := RequestType{valuer.NewString(s)}
switch v {
case RequestTypeScalar, RequestTypeTimeSeries, RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace, RequestTypeDistribution:
case RequestTypeScalar, RequestTypeTimeSeries, RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace, RequestTypeDistribution, RequestTypeHeatmap:
*r = v
return nil
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unknown request type %q; allowed values: %s", s, "`scalar`, `time_series`, `raw`, `raw_stream`, `trace`, `distribution`")
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unknown request type %q; allowed values: %s", s, "`scalar`, `time_series`, `raw`, `raw_stream`, `trace`, `distribution`, `heatmap`")
}
}
@@ -41,6 +41,9 @@ var (
RequestTypeTrace = RequestType{valuer.NewString("trace")}
// []Bucket (struct{Lower,Upper,Count float64}), example: histogram.
RequestTypeDistribution = RequestType{valuer.NewString("distribution")}
// TimeSeriesData carrying one count per histogram bucket at each timestamp,
// with the shared bucket boundaries on the aggregation's meta.
RequestTypeHeatmap = RequestType{valuer.NewString("heatmap")}
)
// IsAggregation returns true for request types that produce aggregated results
@@ -49,7 +52,7 @@ var (
// For non-aggregation types (raw, raw_stream, trace), those fields are ignored
// and don't need to be validated.
func (r RequestType) IsAggregation() bool {
return r == RequestTypeTimeSeries || r == RequestTypeScalar || r == RequestTypeDistribution
return r == RequestTypeTimeSeries || r == RequestTypeScalar || r == RequestTypeDistribution || r == RequestTypeHeatmap
}
// Enum implements jsonschema.Enum; returns the acceptable values for RequestType.
@@ -60,6 +63,7 @@ func (RequestType) Enum() []any {
RequestTypeRaw,
RequestTypeRawStream,
RequestTypeTrace,
RequestTypeHeatmap,
// RequestTypeDistribution,
}
}

View File

@@ -138,12 +138,10 @@ type TimeSeriesData struct {
}
type AggregationBucket struct {
Index int `json:"index"` // or string Alias
Alias string `json:"alias"`
Meta struct {
Unit string `json:"unit,omitempty"`
} `json:"meta,omitempty"`
Series []*TimeSeries `json:"series"` // no extra nesting
Index int `json:"index"` // or string Alias
Alias string `json:"alias"`
Meta AggregationMeta `json:"meta,omitempty"`
Series []*TimeSeries `json:"series"` // no extra nesting
PredictedSeries []*TimeSeries `json:"predictedSeries,omitempty"`
UpperBoundSeries []*TimeSeries `json:"upperBoundSeries,omitempty"`
@@ -151,6 +149,20 @@ type AggregationBucket struct {
AnomalyScores []*TimeSeries `json:"anomalyScores,omitempty"`
}
// HeatmapBucketColumn is the alias a heatmap statement gives the column holding
// a row's bucket boundary. Every other aggregation returns a single numeric
// column the reader treats as the value; this name tells the two apart.
const HeatmapBucketColumn = "__bucket"
type AggregationMeta struct {
Unit string `json:"unit,omitempty"`
// Buckets are the ascending bucket upper bounds shared by every series here.
// Set only for heatmap results, where each point's Values holds
// len(Buckets)+1 counts: one per bucket, then the open-above overflow, whose
// bound is `le=+Inf` and so cannot be listed as a JSON number.
Buckets []float64 `json:"buckets,omitempty"`
}
type TimeSeries struct {
Labels []*Label `json:"labels,omitempty"`
Values []*TimeSeriesValue `json:"values"`
@@ -254,13 +266,9 @@ type TimeSeriesValue struct {
// on the client side, these partial values are rendered differently.
Partial bool `json:"partial,omitempty"`
// for the heatmap type chart
// Values holds one count per histogram bucket for heatmap results, in the
// order of the aggregation's Meta.Buckets. Value is unused in that case.
Values []float64 `json:"values,omitempty"`
Bucket *Bucket `json:"bucket,omitempty"`
}
type Bucket struct {
Step float64 `json:"step"`
}
type ColumnType struct {

View File

@@ -127,7 +127,7 @@ func calculateSeriesValue(series *TimeSeries) float64 {
// For single-point series, return that value directly
if len(series.Values) == 1 {
value := series.Values[0].Value
value := calculatePointValue(series.Values[0])
if math.IsNaN(value) || math.IsInf(value, 0) {
return 0.0
}
@@ -139,10 +139,11 @@ func calculateSeriesValue(series *TimeSeries) float64 {
var count float64
for _, point := range series.Values {
if math.IsNaN(point.Value) || math.IsInf(point.Value, 0) {
value := calculatePointValue(point)
if math.IsNaN(value) || math.IsInf(value, 0) {
continue
}
sum += point.Value
sum += value
count++
}
@@ -154,6 +155,25 @@ func calculateSeriesValue(series *TimeSeries) float64 {
return sum / count
}
// calculatePointValue returns what a point contributes to its series' rank.
// Heatmap points carry one count per bucket in Values and leave Value at zero,
// so they rank on the total across buckets.
func calculatePointValue(point *TimeSeriesValue) float64 {
if len(point.Values) == 0 {
return point.Value
}
var total float64
for _, value := range point.Values {
if math.IsNaN(value) || math.IsInf(value, 0) {
continue
}
total += value
}
return total
}
// convertValueToString converts various types to string for comparison.
func convertValueToString(value any) string {
switch v := value.(type) {

View File

@@ -1,10 +1,12 @@
package querybuildertypesv5
import (
"math"
"testing"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestApplySeriesLimit(t *testing.T) {
@@ -232,3 +234,81 @@ func TestApplySeriesLimit(t *testing.T) {
assert.Equal(t, 40.0, result[2].Values[0].Value)
})
}
func TestApplySeriesLimitRanksHeatmapSeriesByBucketTotals(t *testing.T) {
// A reshaped heatmap point leaves Value at zero and holds one count per
// bucket in Values, so ranking has to sum the buckets to see any difference.
series := []*TimeSeries{
{
Labels: []*Label{{
Key: telemetrytypes.TelemetryFieldKey{Name: "service.name"},
Value: "quiet",
}},
Values: []*TimeSeriesValue{
{Timestamp: 1000, Values: []float64{1, 2, 0}},
{Timestamp: 1060, Values: []float64{0, 1, 0}},
},
},
{
Labels: []*Label{{
Key: telemetrytypes.TelemetryFieldKey{Name: "service.name"},
Value: "busy",
}},
Values: []*TimeSeriesValue{
{Timestamp: 1000, Values: []float64{40, 60, 5}},
{Timestamp: 1060, Values: []float64{30, 70, 5}},
},
},
{
Labels: []*Label{{
Key: telemetrytypes.TelemetryFieldKey{Name: "service.name"},
Value: "middling",
}},
Values: []*TimeSeriesValue{
{Timestamp: 1000, Values: []float64{5, 5, 0}},
{Timestamp: 1060, Values: []float64{4, 6, 0}},
},
},
}
result := ApplySeriesLimit(series, nil, 2)
require.Len(t, result, 2)
assert.Equal(t, "busy", result[0].Labels[0].Value)
assert.Equal(t, "middling", result[1].Labels[0].Value)
}
func TestCalculatePointValue(t *testing.T) {
testCases := []struct {
description string
point *TimeSeriesValue
expectedValue float64
}{
{
description: "a plain time series point ranks on its single value",
point: &TimeSeriesValue{Timestamp: 1000, Value: 7},
expectedValue: 7,
},
{
description: "a heatmap point ranks on the total across its buckets",
point: &TimeSeriesValue{Timestamp: 1000, Values: []float64{1, 12, 14, 3}},
expectedValue: 30,
},
{
description: "non-finite bucket counts are skipped",
point: &TimeSeriesValue{Timestamp: 1000, Values: []float64{2, math.NaN(), math.Inf(1), 3}},
expectedValue: 5,
},
{
description: "an empty bucket list falls back to the single value",
point: &TimeSeriesValue{Timestamp: 1000, Value: 4, Values: []float64{}},
expectedValue: 4,
},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
assert.Equal(t, testCase.expectedValue, calculatePointValue(testCase.point))
})
}
}

View File

@@ -2,6 +2,7 @@ package querybuildertypesv5
import (
"fmt"
"math"
"slices"
"strings"
@@ -581,7 +582,7 @@ func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
// Validate request type
switch r.RequestType {
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace, RequestTypeTimeSeries, RequestTypeScalar:
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace, RequestTypeTimeSeries, RequestTypeScalar, RequestTypeHeatmap:
opts = append(opts, GetValidationOptions(r.RequestType)...)
default:
return errors.NewInvalidInputf(
@@ -589,10 +590,14 @@ func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
"invalid request type: %s",
r.RequestType,
).WithAdditional(
"Valid request types are: raw, timeseries, scalar",
"Valid request types are: raw, timeseries, scalar, heatmap",
)
}
if err := r.validateHeatmap(); err != nil {
return err
}
// raw/trace request types don't support metric queries;
// metrics are always aggregated and there is no raw form.
if r.RequestType == RequestTypeRaw || r.RequestType == RequestTypeRawStream || r.RequestType == RequestTypeTrace {
@@ -630,11 +635,15 @@ func (r *QueryRangeRequest) ValidateRequestScope() ([]ValidationOption, error) {
var opts []ValidationOption
switch r.RequestType {
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace, RequestTypeTimeSeries, RequestTypeScalar:
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace, RequestTypeTimeSeries, RequestTypeScalar, RequestTypeHeatmap:
opts = GetValidationOptions(r.RequestType)
default:
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid request type: %s", r.RequestType).
WithAdditional("Valid request types are: raw, timeseries, scalar")
WithAdditional("Valid request types are: raw, timeseries, scalar, heatmap")
}
if err := r.validateHeatmap(); err != nil {
return nil, err
}
if r.RequestType == RequestTypeRaw || r.RequestType == RequestTypeRawStream || r.RequestType == RequestTypeTrace {
@@ -838,9 +847,129 @@ func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) err
}
}
// validateHeatmap refuses request shapes a heatmap cannot render. Metric type is
// deliberately not checked here: MetricAggregation.Type is resolved from metadata
// after validation runs, so gauge/sum/counter and exponential histograms have to
// be refused by the querier once that resolution has happened.
func (r *QueryRangeRequest) validateHeatmap() error {
if r.RequestType != RequestTypeHeatmap {
return nil
}
if r.FormatOptions != nil && r.FormatOptions.FillGaps {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"fillGaps is not supported for heatmap requests: an absent column means collection stopped, which a zero-filled column would hide")
}
if err := r.BucketOptions.validateBucketOptions(); err != nil {
return err
}
enabled := 0
for _, envelope := range r.CompositeQuery.Queries {
switch spec := envelope.Spec.(type) {
case QueryBuilderQuery[MetricAggregation]:
if err := validateHeatmapQuery(spec.Functions, spec.Having); err != nil {
return err
}
if spec.Disabled {
continue
}
enabled++
case QueryBuilderFormula:
if err := validateHeatmapQuery(spec.Functions, spec.Having); err != nil {
return err
}
if spec.Disabled {
continue
}
enabled++
case ClickHouseQuery:
// The rows a ClickHouse query returns are read by request type, the
// same as for any other request, so one shaped as heatmap cells
// renders without the builder having produced it.
if spec.Disabled {
continue
}
enabled++
case PromQuery:
// A PromQL heatmap is a classic histogram read through its `le`
// labels, the same axis the builder's histogram path uses.
if r.BucketOptions != nil {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"bucketOptions are not supported for promql heatmap requests: the bucket axis comes from the `le` labels the query returns, so nothing in the spec would be applied")
}
if spec.Disabled {
continue
}
enabled++
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"heatmap requests support one metrics builder query, one formula over them, one clickhouse query, or one promql query, got %q", envelope.Type.StringValue())
}
}
if enabled != 1 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"heatmap requests need exactly one enabled query, got %d", enabled)
}
return nil
}
func (b *BucketOptions) validateBucketOptions() error {
if b == nil {
return nil
}
switch spec := b.Spec.(type) {
case LinearBucketsSpec:
// Boundaries are placed at maxValue*i/numBuckets, so a cap at or below
// zero collapses every one of them onto the same point, and a non-finite
// one compares false against every value so nothing reaches the overflow.
if math.IsNaN(spec.MaxValue) || math.IsInf(spec.MaxValue, 0) || spec.MaxValue <= 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"linear buckets need a finite maxValue greater than 0, got %v", spec.MaxValue)
}
if spec.NumBuckets < 0 || spec.NumBuckets > MaxNumBuckets {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"numBuckets must be between 1 and %d, got %d", MaxNumBuckets, spec.NumBuckets)
}
case LogBucketsSpec:
if spec.Scale != nil && (*spec.Scale < MinLogScale || *spec.Scale > MaxLogScale) {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"scale must be between %d and %d, got %d", MinLogScale, MaxLogScale, *spec.Scale)
}
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"invalid bucketOptions kind %q, expected one of linear, log", b.Kind.StringValue())
}
return nil
}
// validateHeatmapQuery refuses the per-query settings that cannot mean anything
// on a heatmap. It runs on disabled queries too: a disabled query is a formula
// input, so whatever it does still reaches the cells.
func validateHeatmapQuery(functions []Function, having *Having) error {
if len(functions) > 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"functions are not supported for heatmap requests: a heatmap point is a count per bucket, not a single value")
}
if having != nil && having.Expression != "" {
return errors.NewInvalidInputf(errors.CodeInvalidInput,
"having is not supported for heatmap requests: it filters individual cells, which breaks the cumulative differencing")
}
return nil
}
func GetValidationOptions(requestType RequestType) []ValidationOption {
switch requestType {
case RequestTypeTimeSeries:
case RequestTypeTimeSeries, RequestTypeHeatmap:
return []ValidationOption{WithSkipSelectFieldValidation(), WithTimestampGroupByValidation()}
case RequestTypeScalar:
return []ValidationOption{WithSkipSelectFieldValidation(), WithReduceToValidation()}