Compare commits

..

2 Commits

Author SHA1 Message Date
srikanthccv
858ff3992e chore: generate openapi specs 2026-11-09 07:33:34 +05:30
srikanthccv
e4c381aa31 docs(querier): add query_range examples for every builder function
One request per function family, with the arguments, their values and
what the function does to the series.

Assisted-by: Claude Fable 5.1
2026-11-09 07:33:34 +05:30
11 changed files with 583 additions and 168 deletions

View File

@@ -32729,6 +32729,301 @@ paths:
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_anomaly_request_rate:
description: 'anomaly is the one function with named arguments. seasonality
says which past periods predict the current value: hourly, daily
or weekly, daily when absent. z_score_threshold is how many standard
deviations from the prediction count as an anomaly, 3 when absent.
The response carries, next to the values, a predicted series, an
upper and a lower bound series and an anomaly score series. The
scoring runs in the enterprise build; the community build returns
the series unchanged.'
summary: 'anomaly: request rate scored against its daily pattern'
value:
compositeQuery:
queries:
- spec:
aggregations:
- metricName: http.server.duration.count
spaceAggregation: sum
timeAggregation: rate
functions:
- args:
- name: seasonality
value: daily
- name: z_score_threshold
value: 3
name: anomaly
groupBy:
- fieldContext: resource
name: service.name
name: A
signal: metrics
stepInterval: 300s
type: builder_query
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_cumulativesum_errors_so_far:
description: cumulativeSum replaces each point with the sum of every
point up to it, so a per-step count becomes a running total that
starts at zero at the window start. It takes no arguments. The total
resets with the window, so a longer time range gives a larger final
value; it is a view of the window, not a counter.
summary: 'cumulativeSum: 5xx responses since the start of the window'
value:
compositeQuery:
queries:
- spec:
aggregations:
- alias: server_errors
expression: count()
filter:
expression: http.response.status_code >= 500
functions:
- name: cumulativeSum
groupBy:
- fieldContext: resource
name: service.name
name: A
signal: traces
stepInterval: 60s
type: builder_query
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_cutoffmin_hide_idle_hosts:
description: The four threshold functions take one number. cutOffMin
and cutOffMax remove the points below or above the threshold, so
an idle host disappears from the chart instead of drawing a line
along the bottom. clampMin and clampMax keep the points and replace
the value with the threshold, which caps spikes without hiding them.
system.cpu.utilization is a ratio, so 0.05 is 5%.
summary: 'cutOffMin: drop hosts below 5% CPU from the chart'
value:
compositeQuery:
queries:
- spec:
aggregations:
- metricName: system.cpu.utilization
spaceAggregation: avg
timeAggregation: avg
functions:
- args:
- value: 0.05
name: cutOffMin
groupBy:
- fieldContext: resource
name: host.name
name: A
signal: metrics
stepInterval: 60s
type: builder_query
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_ewma_smoothed_latency:
description: 'ewma3, ewma5 and ewma7 replace each point with an exponentially
weighted average of the point and the points before it. The number
in the name is the window the default weight is derived from: alpha
= 2 / (n + 1), so 0.5, 0.333 and 0.25. One optional argument overrides
alpha; a value close to 1 follows the raw series, a value close
to 0 smooths harder. 0.3 here smooths a little more than ewma5''s
default.'
summary: 'ewma5: smoothed p95 latency'
value:
compositeQuery:
queries:
- spec:
aggregations:
- metricName: http.server.duration.bucket
spaceAggregation: p95
functions:
- args:
- value: 0.3
name: ewma5
groupBy:
- fieldContext: resource
name: service.name
name: A
signal: metrics
stepInterval: 60s
type: builder_query
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_fillzero_sparse_errors:
description: A count over sparse events returns a point only for steps
that had a match, so a chart of failed spans has gaps where nothing
failed. fillZero adds a zero at every missing step. With no arguments
the server fills the request window at the query's step, which is
the common case. The three optional arguments are start, end and
step in milliseconds, for callers that want to fill a different
grid.
summary: 'fillZero: a zero at every step with no failed spans'
value:
compositeQuery:
queries:
- spec:
aggregations:
- alias: failed_spans
expression: count()
filter:
expression: has_error = true
functions:
- name: fillZero
groupBy:
- fieldContext: resource
name: service.name
name: A
signal: traces
stepInterval: 60s
type: builder_query
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_log10_network_bytes:
description: log2 and log10 replace each value with its logarithm,
so a loopback device moving kilobytes and an uplink moving gigabytes
fit on one chart. They take no arguments. Zero and negative values
have no logarithm and are not drawn. absolute, also without arguments,
is the usual companion for a signed series before a logarithm.
summary: 'log10: network throughput across devices that differ by
orders of magnitude'
value:
compositeQuery:
queries:
- spec:
aggregations:
- metricName: system.network.io
spaceAggregation: sum
timeAggregation: rate
filter:
expression: direction = 'transmit'
functions:
- name: log10
groupBy:
- fieldContext: attribute
name: device
name: A
signal: metrics
stepInterval: 60s
type: builder_query
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_median_despiked_request_rate:
description: median3, median5 and median7 replace each point with
the median of the surrounding 3, 5 or 7 points. A spike that lasts
one step vanishes, a shift that lasts longer than half the window
stays. They take no arguments. Use them when the raw series is noisy
and the chart is for a trend, not for catching the spike.
summary: 'median5: request rate without one-step spikes'
value:
compositeQuery:
queries:
- spec:
aggregations:
- metricName: http.server.duration.count
spaceAggregation: sum
timeAggregation: rate
functions:
- name: median5
groupBy:
- fieldContext: resource
name: service.name
name: A
signal: metrics
stepInterval: 60s
type: builder_query
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_runningdiff_disk_growth:
description: 'runningDiff turns a gauge into the change between consecutive
points, so a filesystem usage gauge becomes bytes written per step.
It takes no arguments and drops the first point of the window, since
that point has nothing to difference against; the server reads one
step before the window to keep the first interval. Functions apply
in order: clampMin with 0 runs on the differences and replaces the
negative value a cleanup or a volume reset produces with zero.'
summary: 'runningDiff then clampMin: bytes written to disk per step'
value:
compositeQuery:
queries:
- spec:
aggregations:
- metricName: system.filesystem.usage
spaceAggregation: sum
timeAggregation: latest
filter:
expression: state = 'used'
functions:
- name: runningDiff
- args:
- value: 0
name: clampMin
groupBy:
- fieldContext: resource
name: host.name
name: A
signal: metrics
stepInterval: 60s
type: builder_query
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
function_timeshift_week_over_week:
description: timeShift moves every point of its query forward by the
argument, in seconds, so a series read from the past lands on the
current timestamps. B here reads the hour exactly seven days (604800
s) before the requested window and comes back on this week's timestamps,
so it overlays A on a chart and the formula can subtract it point
by point. The argument is positive; it says how far back to read.
summary: 'timeShift: this week''s error logs next to last week''s'
value:
compositeQuery:
queries:
- spec:
aggregations:
- alias: errors
expression: count()
filter:
expression: severity_text = 'ERROR'
name: A
signal: logs
stepInterval: 60s
type: builder_query
- spec:
aggregations:
- alias: errors
expression: count()
filter:
expression: severity_text = 'ERROR'
functions:
- args:
- value: 604800
name: timeShift
name: B
signal: logs
stepInterval: 60s
type: builder_query
- spec:
expression: A - B
name: F1
type: builder_formula
end: 1640998800000
requestType: time_series
schemaVersion: v1
start: 1640995200000
logs_raw:
summary: 'Raw: fetch raw log records'
value:

View File

@@ -30,7 +30,7 @@ func (provider *provider) addQuerierRoutes(router *mux.Router) error {
Description: "Execute a composite query over a time range. Supports builder queries (traces, logs, metrics), formulas, trace operators, PromQL, and ClickHouse SQL.",
Request: new(qbtypes.QueryRangeRequest),
RequestContentType: "application/json",
RequestExamples: []handler.OpenAPIExample{
RequestExamples: append([]handler.OpenAPIExample{
{
Name: "traces_time_series",
Summary: "Time series: count spans grouped by service",
@@ -598,7 +598,7 @@ func (provider *provider) addQuerierRoutes(router *mux.Router) error {
},
},
},
},
}, queryRangeFunctionExamples()...),
Response: new(qbtypes.QueryRangeResponse),
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,

View File

@@ -0,0 +1,202 @@
package signozapiserver
import "github.com/SigNoz/signoz/pkg/http/handler"
// timeSeriesExample wraps builder query specs in a one hour time series
// request, the shape every function example shares.
func timeSeriesExample(name, summary, description string, queries ...map[string]any) handler.OpenAPIExample {
list := make([]any, 0, len(queries))
for _, query := range queries {
list = append(list, query)
}
return handler.OpenAPIExample{
Name: name,
Summary: summary,
Description: description,
Value: map[string]any{
"schemaVersion": "v1",
"start": 1640995200000,
"end": 1640998800000,
"requestType": "time_series",
"compositeQuery": map[string]any{"queries": list},
},
}
}
func builderQueryEnvelope(spec map[string]any) map[string]any {
return map[string]any{"type": "builder_query", "spec": spec}
}
// function builds a function entry with positional arguments; anomaly is the
// one function that takes named arguments and is written out in full.
func function(name string, args ...any) map[string]any {
list := make([]any, 0, len(args))
for _, arg := range args {
list = append(list, map[string]any{"value": arg})
}
entry := map[string]any{"name": name}
if len(list) > 0 {
entry["args"] = list
}
return entry
}
// queryRangeFunctionExamples shows every function a builder query can carry,
// what its arguments mean and the values they take. Functions run in the
// order listed, after the aggregation and before formulas.
func queryRangeFunctionExamples() []handler.OpenAPIExample {
return []handler.OpenAPIExample{
timeSeriesExample(
"function_timeshift_week_over_week",
"timeShift: this week's error logs next to last week's",
"timeShift moves every point of its query forward by the argument, in seconds, so a series read from the past lands on the current timestamps. B here reads the hour exactly seven days (604800 s) before the requested window and comes back on this week's timestamps, so it overlays A on a chart and the formula can subtract it point by point. The argument is positive; it says how far back to read.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "logs",
"aggregations": []any{map[string]any{"expression": "count()", "alias": "errors"}},
"stepInterval": "60s",
"filter": map[string]any{"expression": "severity_text = 'ERROR'"},
}),
builderQueryEnvelope(map[string]any{
"name": "B",
"signal": "logs",
"aggregations": []any{map[string]any{"expression": "count()", "alias": "errors"}},
"stepInterval": "60s",
"filter": map[string]any{"expression": "severity_text = 'ERROR'"},
"functions": []any{function("timeShift", 604800)},
}),
map[string]any{
"type": "builder_formula",
"spec": map[string]any{"name": "F1", "expression": "A - B"},
},
),
timeSeriesExample(
"function_fillzero_sparse_errors",
"fillZero: a zero at every step with no failed spans",
"A count over sparse events returns a point only for steps that had a match, so a chart of failed spans has gaps where nothing failed. fillZero adds a zero at every missing step. With no arguments the server fills the request window at the query's step, which is the common case. The three optional arguments are start, end and step in milliseconds, for callers that want to fill a different grid.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "traces",
"aggregations": []any{map[string]any{"expression": "count()", "alias": "failed_spans"}},
"stepInterval": "60s",
"filter": map[string]any{"expression": "has_error = true"},
"groupBy": []any{map[string]any{"name": "service.name", "fieldContext": "resource"}},
"functions": []any{function("fillZero")},
}),
),
timeSeriesExample(
"function_cutoffmin_hide_idle_hosts",
"cutOffMin: drop hosts below 5% CPU from the chart",
"The four threshold functions take one number. cutOffMin and cutOffMax remove the points below or above the threshold, so an idle host disappears from the chart instead of drawing a line along the bottom. clampMin and clampMax keep the points and replace the value with the threshold, which caps spikes without hiding them. system.cpu.utilization is a ratio, so 0.05 is 5%.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "metrics",
"aggregations": []any{
map[string]any{"metricName": "system.cpu.utilization", "timeAggregation": "avg", "spaceAggregation": "avg"},
},
"stepInterval": "60s",
"groupBy": []any{map[string]any{"name": "host.name", "fieldContext": "resource"}},
"functions": []any{function("cutOffMin", 0.05)},
}),
),
timeSeriesExample(
"function_runningdiff_disk_growth",
"runningDiff then clampMin: bytes written to disk per step",
"runningDiff turns a gauge into the change between consecutive points, so a filesystem usage gauge becomes bytes written per step. It takes no arguments and drops the first point of the window, since that point has nothing to difference against; the server reads one step before the window to keep the first interval. Functions apply in order: clampMin with 0 runs on the differences and replaces the negative value a cleanup or a volume reset produces with zero.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "metrics",
"aggregations": []any{
map[string]any{"metricName": "system.filesystem.usage", "timeAggregation": "latest", "spaceAggregation": "sum"},
},
"stepInterval": "60s",
"filter": map[string]any{"expression": "state = 'used'"},
"groupBy": []any{map[string]any{"name": "host.name", "fieldContext": "resource"}},
"functions": []any{function("runningDiff"), function("clampMin", 0)},
}),
),
timeSeriesExample(
"function_ewma_smoothed_latency",
"ewma5: smoothed p95 latency",
"ewma3, ewma5 and ewma7 replace each point with an exponentially weighted average of the point and the points before it. The number in the name is the window the default weight is derived from: alpha = 2 / (n + 1), so 0.5, 0.333 and 0.25. One optional argument overrides alpha; a value close to 1 follows the raw series, a value close to 0 smooths harder. 0.3 here smooths a little more than ewma5's default.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "metrics",
"aggregations": []any{
map[string]any{"metricName": "http.server.duration.bucket", "spaceAggregation": "p95"},
},
"stepInterval": "60s",
"groupBy": []any{map[string]any{"name": "service.name", "fieldContext": "resource"}},
"functions": []any{function("ewma5", 0.3)},
}),
),
timeSeriesExample(
"function_median_despiked_request_rate",
"median5: request rate without one-step spikes",
"median3, median5 and median7 replace each point with the median of the surrounding 3, 5 or 7 points. A spike that lasts one step vanishes, a shift that lasts longer than half the window stays. They take no arguments. Use them when the raw series is noisy and the chart is for a trend, not for catching the spike.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "metrics",
"aggregations": []any{
map[string]any{"metricName": "http.server.duration.count", "timeAggregation": "rate", "spaceAggregation": "sum"},
},
"stepInterval": "60s",
"groupBy": []any{map[string]any{"name": "service.name", "fieldContext": "resource"}},
"functions": []any{function("median5")},
}),
),
timeSeriesExample(
"function_cumulativesum_errors_so_far",
"cumulativeSum: 5xx responses since the start of the window",
"cumulativeSum replaces each point with the sum of every point up to it, so a per-step count becomes a running total that starts at zero at the window start. It takes no arguments. The total resets with the window, so a longer time range gives a larger final value; it is a view of the window, not a counter.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "traces",
"aggregations": []any{map[string]any{"expression": "count()", "alias": "server_errors"}},
"stepInterval": "60s",
"filter": map[string]any{"expression": "http.response.status_code >= 500"},
"groupBy": []any{map[string]any{"name": "service.name", "fieldContext": "resource"}},
"functions": []any{function("cumulativeSum")},
}),
),
timeSeriesExample(
"function_log10_network_bytes",
"log10: network throughput across devices that differ by orders of magnitude",
"log2 and log10 replace each value with its logarithm, so a loopback device moving kilobytes and an uplink moving gigabytes fit on one chart. They take no arguments. Zero and negative values have no logarithm and are not drawn. absolute, also without arguments, is the usual companion for a signed series before a logarithm.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "metrics",
"aggregations": []any{
map[string]any{"metricName": "system.network.io", "timeAggregation": "rate", "spaceAggregation": "sum"},
},
"stepInterval": "60s",
"filter": map[string]any{"expression": "direction = 'transmit'"},
"groupBy": []any{map[string]any{"name": "device", "fieldContext": "attribute"}},
"functions": []any{function("log10")},
}),
),
timeSeriesExample(
"function_anomaly_request_rate",
"anomaly: request rate scored against its daily pattern",
"anomaly is the one function with named arguments. seasonality says which past periods predict the current value: hourly, daily or weekly, daily when absent. z_score_threshold is how many standard deviations from the prediction count as an anomaly, 3 when absent. The response carries, next to the values, a predicted series, an upper and a lower bound series and an anomaly score series. The scoring runs in the enterprise build; the community build returns the series unchanged.",
builderQueryEnvelope(map[string]any{
"name": "A",
"signal": "metrics",
"aggregations": []any{
map[string]any{"metricName": "http.server.duration.count", "timeAggregation": "rate", "spaceAggregation": "sum"},
},
"stepInterval": "300s",
"groupBy": []any{map[string]any{"name": "service.name", "fieldContext": "resource"}},
"functions": []any{
map[string]any{
"name": "anomaly",
"args": []any{
map[string]any{"name": "seasonality", "value": "daily"},
map[string]any{"name": "z_score_threshold", "value": 3},
},
},
},
}),
),
}
}

View File

@@ -0,0 +1,21 @@
package signozapiserver
import (
"encoding/json"
"testing"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/stretchr/testify/require"
)
func TestQueryRangeFunctionExamplesAreValidRequests(t *testing.T) {
for _, ex := range queryRangeFunctionExamples() {
t.Run(ex.Name, func(t *testing.T) {
raw, err := json.Marshal(ex.Value)
require.NoError(t, err)
var req qbtypes.QueryRangeRequest
require.NoError(t, json.Unmarshal(raw, &req))
require.NoError(t, req.Validate())
})
}
}

View File

@@ -26,9 +26,8 @@ const bucketCacheCellSteps = 60
type BucketCache interface {
// cached portion + list of gaps to fetch
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
// store fresh buckets for future hits; only the ranges in the map were
// queried for fresh, the rest of it came from the cache
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats)
// store fresh buckets for future hits
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
}
// bucketCache implements the BucketCache interface.
@@ -117,7 +116,7 @@ func (bc *bucketCache) GetMissRanges(
}
// Merge buckets into a single result
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, cachedWarnings, startMs, endMs)
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, cachedWarnings)
// Filter the merged result to only include values within the requested time range
mergedResult = bc.filterResultToTimeRange(mergedResult, q, startMs, endMs, stepMs)
@@ -126,7 +125,7 @@ func (bc *bucketCache) GetMissRanges(
}
// Put stores fresh query results in the cache.
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) {
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result) {
// Get query window
startMs, endMs := q.Window()
@@ -189,7 +188,7 @@ func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Que
// Convert trimmed result to buckets, one per grid cell it spans
cellWidthMs := bucketCacheCellSteps * stepMs
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs, cellWidthMs, execStatsForQueriedTimeRanges)
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs, cellWidthMs)
cellKeys := make([]string, len(freshBuckets))
cellEntries := make([]cachetypes.Cacheable, len(freshBuckets))
@@ -211,7 +210,7 @@ func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Que
cellEntry = cellEntries[i].(*qbtypes.CachedData)
}
cellBuckets, changed := bc.mergeFreshBucketIntoCell(ctx, cellEntry.Buckets, freshBucket, execStatsForQueriedTimeRanges)
cellBuckets, changed := bc.mergeFreshBucketIntoCell(ctx, cellEntry.Buckets, freshBucket)
if !changed {
continue
}
@@ -464,7 +463,7 @@ func (bc *bucketCache) filterRelevantBuckets(buckets []*qbtypes.CachedBucket, st
}
// mergeBuckets combines multiple cached buckets into a single result.
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string, startMs, endMs uint64) *qbtypes.Result {
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string) *qbtypes.Result {
if len(buckets) == 0 {
return &qbtypes.Result{}
}
@@ -472,13 +471,12 @@ func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.Cach
// All buckets should have the same type
resultType := buckets[0].Type
// A bucket can reach past the window; only the part inside it is served
// Aggregate stats
var totalStats qbtypes.ExecStats
for _, bucket := range buckets {
servedStats := bucket.ProrateStatsToWindow(startMs, endMs)
totalStats.RowsScanned += servedStats.RowsScanned
totalStats.BytesScanned += servedStats.BytesScanned
totalStats.DurationMS += servedStats.DurationMS
totalStats.RowsScanned += bucket.Stats.RowsScanned
totalStats.BytesScanned += bucket.Stats.BytesScanned
totalStats.DurationMS += bucket.Stats.DurationMS
}
// Merge values based on type
@@ -675,7 +673,7 @@ func (bc *bucketCache) isEmptyResult(result *qbtypes.Result) (isEmpty bool, isFi
}
// resultToBuckets converts a query result into time-based buckets.
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs, cellWidthMs uint64, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) []*qbtypes.CachedBucket {
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs, cellWidthMs uint64) []*qbtypes.CachedBucket {
// Check if result is empty
isEmpty, isFiltered := bc.isEmptyResult(result)
@@ -687,7 +685,7 @@ func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Resu
}
// Empty filtered results still become buckets, so the range is not re-queried
perCellBuckets, err := result.SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs, execStatsForQueriedTimeRanges)
perCellBuckets, err := result.SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs)
if err != nil {
bc.logger.ErrorContext(ctx, "failed to convert result to buckets", errors.Attr(err))
return nil
@@ -699,7 +697,7 @@ func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Resu
// and reports whether the cell changed: a bucket already covering the fresh
// one leaves it as is, buckets inside the fresh one are replaced, buckets
// overlapping or edge to edge with it are unioned, and the rest are kept apart.
func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets []*qbtypes.CachedBucket, freshBucket *qbtypes.CachedBucket, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) ([]*qbtypes.CachedBucket, bool) {
func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets []*qbtypes.CachedBucket, freshBucket *qbtypes.CachedBucket) ([]*qbtypes.CachedBucket, bool) {
keptBuckets := make([]*qbtypes.CachedBucket, 0, len(cellBuckets)+1)
touchingBuckets := []*qbtypes.CachedBucket{}
for _, cellBucket := range cellBuckets {
@@ -712,7 +710,7 @@ func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets
keptBuckets = append(keptBuckets, cellBucket)
}
}
keptBuckets = append(keptBuckets, bc.unionBuckets(ctx, freshBucket, touchingBuckets, execStatsForQueriedTimeRanges))
keptBuckets = append(keptBuckets, bc.unionBuckets(ctx, freshBucket, touchingBuckets))
slices.SortFunc(keptBuckets, func(a, b *qbtypes.CachedBucket) int {
return cmp.Compare(a.StartMs, b.StartMs)
@@ -723,16 +721,7 @@ func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets
// unionBuckets widens the fresh bucket by the buckets it touches. Buckets
// inside it are replaced outright, so only a genuine partial overlap decodes
// anything.
func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.CachedBucket, touchingBuckets []*qbtypes.CachedBucket, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) *qbtypes.CachedBucket {
// Data queried again takes the new query's cost, which the fresh bucket
// carries; a touching bucket keeps its cost only for what was not queried again
for _, touchingBucket := range touchingBuckets {
keptStats := touchingBucket.ProrateStatsToUnqueried(execStatsForQueriedTimeRanges)
freshBucket.Stats.RowsScanned += keptStats.RowsScanned
freshBucket.Stats.BytesScanned += keptStats.BytesScanned
freshBucket.Stats.DurationMS += keptStats.DurationMS
}
func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.CachedBucket, touchingBuckets []*qbtypes.CachedBucket) *qbtypes.CachedBucket {
bucketsToMerge := []*qbtypes.CachedBucket{freshBucket}
for _, touchingBucket := range touchingBuckets {
if !freshBucket.Contains(touchingBucket) {
@@ -753,11 +742,14 @@ func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.Ca
EndMs: freshBucket.EndMs,
Type: freshBucket.Type,
Value: value,
Stats: freshBucket.Stats,
Stats: qbtypes.ExecStats{StepIntervals: freshBucket.Stats.StepIntervals},
}
for _, mergedBucket := range bucketsToMerge {
unionBucket.StartMs = min(unionBucket.StartMs, mergedBucket.StartMs)
unionBucket.EndMs = max(unionBucket.EndMs, mergedBucket.EndMs)
unionBucket.Stats.RowsScanned += mergedBucket.Stats.RowsScanned
unionBucket.Stats.BytesScanned += mergedBucket.Stats.BytesScanned
unionBucket.Stats.DurationMS += mergedBucket.Stats.DurationMS
}
return unionBucket
}

View File

@@ -30,7 +30,7 @@ func BenchmarkBucketCache_GetMissRanges(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
}
// Create test queries with varying cache hit patterns
@@ -121,7 +121,7 @@ func BenchmarkBucketCache_Put(b *testing.B) {
for i := 0; i < b.N; i++ {
for j := 0; j < tc.numQueries; j++ {
bc.Put(ctx, orgID, queries[j], qbtypes.Step{Duration: 1000 * time.Millisecond}, results[j], map[qbtypes.TimeRange]qbtypes.ExecStats{{From: queries[j].startMs, To: queries[j].endMs}: results[j].Stats})
bc.Put(ctx, orgID, queries[j], qbtypes.Step{Duration: 1000 * time.Millisecond}, results[j])
}
}
})
@@ -259,7 +259,7 @@ func BenchmarkBucketCache_ConcurrentOperations(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
}
b.ResetTimer()
@@ -284,7 +284,7 @@ func BenchmarkBucketCache_ConcurrentOperations(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
case 2: // Partial read
query := &mockQuery{
fingerprint: fmt.Sprintf("concurrent-query-%d", i%100),

View File

@@ -59,7 +59,7 @@ func TestBucketCacheStepAlignment(t *testing.T) {
}
// Put result in cache
bc.Put(ctx, orgID, query, step, result, nil)
bc.Put(ctx, orgID, query, step, result)
// Get cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
@@ -109,7 +109,7 @@ func TestBucketCacheNoStepInterval(t *testing.T) {
}
// Should cache the entire range when step is 0
bc.Put(ctx, orgID, query, step, result, nil)
bc.Put(ctx, orgID, query, step, result)
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
assert.NotNil(t, cached)

View File

@@ -156,7 +156,7 @@ func TestBucketCache_Put_And_Get(t *testing.T) {
}
// Store in cache
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Wait a bit for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -189,7 +189,7 @@ func TestBucketCache_PartialHit(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 3000, 1000),
}
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1, nil)
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1)
// Wait for cache write
time.Sleep(10 * time.Millisecond)
@@ -225,7 +225,7 @@ func TestBucketCache_MultipleBuckets(t *testing.T) {
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 100 * time.Millisecond}, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 2000, 100),
}, nil)
})
query2 := &mockQuery{
fingerprint: "test-query",
@@ -235,7 +235,7 @@ func TestBucketCache_MultipleBuckets(t *testing.T) {
bc.Put(context.Background(), valuer.UUID{}, query2, qbtypes.Step{Duration: 100 * time.Millisecond}, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 3000, 4000, 100),
}, nil)
})
// Wait for cache writes
time.Sleep(10 * time.Millisecond)
@@ -280,7 +280,7 @@ func TestBucketCache_FluxInterval(t *testing.T) {
}
// This should not be cached due to flux interval
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Wait a bit
time.Sleep(10 * time.Millisecond)
@@ -358,7 +358,7 @@ func TestBucketCache_MergeTimeSeriesResults(t *testing.T) {
{Series: []*qbtypes.TimeSeries{series1}},
},
},
}, nil)
})
// Cache second part
query2 := &mockQuery{
@@ -374,7 +374,7 @@ func TestBucketCache_MergeTimeSeriesResults(t *testing.T) {
{Series: []*qbtypes.TimeSeries{series2}},
},
},
}, nil)
})
// Wait for cache writes
time.Sleep(10 * time.Millisecond)
@@ -441,7 +441,7 @@ func TestBucketCache_RawData(t *testing.T) {
Value: rawData,
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -481,7 +481,7 @@ func TestBucketCache_ScalarData(t *testing.T) {
Value: scalarData,
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -509,7 +509,7 @@ func TestBucketCache_EmptyFingerprint(t *testing.T) {
Value: createTestTimeSeries("A", 1000, 5000, 1000),
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Should still be able to retrieve
@@ -564,7 +564,7 @@ func TestBucketCache_ConcurrentAccess(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries(fmt.Sprintf("Q%d", id), query.startMs, query.endMs, 100),
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 100 * time.Microsecond}, result, nil)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 100 * time.Microsecond}, result)
done <- true
}(i)
}
@@ -624,7 +624,7 @@ func TestBucketCache_GetMissRanges_FluxInterval(t *testing.T) {
},
}
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, cachedResult, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, cachedResult)
// Get miss ranges
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -686,7 +686,7 @@ func TestBucketCache_Put_FluxIntervalTrimming(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Retrieve cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -756,7 +756,7 @@ func TestBucketCache_Put_EntireRangeInFluxInterval(t *testing.T) {
}
// Put the result - should not cache anything
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Try to get cached data - should have no cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -862,7 +862,7 @@ func TestBucketCache_EmptyDataHandling(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, tt.result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, tt.result)
// Wait a bit for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -927,7 +927,7 @@ func TestBucketCache_PartialValues(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Wait for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -997,7 +997,7 @@ func TestBucketCache_AllPartialValues(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
// Wait for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -1058,7 +1058,7 @@ func TestBucketCache_FilteredCachedResults(t *testing.T) {
}
// Cache the wide range
bc.Put(ctx, orgID, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1, nil)
bc.Put(ctx, orgID, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1)
time.Sleep(10 * time.Millisecond)
// Now query for a smaller range (2000-3500ms)
@@ -1125,7 +1125,7 @@ func TestBucketCache_PromQLKeepsTheValueAtTheWindowEnd(t *testing.T) {
}},
}},
},
}, nil)
})
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
@@ -1291,7 +1291,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1345,7 +1345,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1397,7 +1397,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1454,7 +1454,7 @@ func TestBucketCache_NoCache(t *testing.T) {
}
// Put the result in cache
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
time.Sleep(10 * time.Millisecond)
// Verify data is cached
@@ -1475,12 +1475,11 @@ func TestBucketCachePutSlicesAtCellBoundaries(t *testing.T) {
// a 1s step gives 60s cells, so 30s to 150s spans three of them
query := &mockQuery{fingerprint: "sliced-query", startMs: 30000, endMs: 150000}
result := &qbtypes.Result{
bc.Put(ctx, orgID, query, step, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 30000, 150000, 1000),
Stats: qbtypes.ExecStats{RowsScanned: 120, BytesScanned: 1200, DurationMS: 12},
}
bc.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: 30000, To: 150000}: result.Stats})
})
type bucketShape struct {
cell, startMs, endMs, rowsScanned uint64
@@ -1540,7 +1539,7 @@ func TestBucketCachePutAbsorbsWritesWithinACell(t *testing.T) {
name: "OverlappingWrites_BecomeOne",
firstWindow: qbtypes.TimeRange{From: 1000, To: 4000},
secondWindow: qbtypes.TimeRange{From: 3000, To: 6000},
expectedBuckets: []bucketShape{{1000, 6000, 5}},
expectedBuckets: []bucketShape{{1000, 6000, 6}},
expectedMissingRanges: []*qbtypes.TimeRange{},
expectedValueCount: 5,
},
@@ -1571,13 +1570,12 @@ func TestBucketCachePutAbsorbsWritesWithinACell(t *testing.T) {
for _, window := range []qbtypes.TimeRange{testCase.firstWindow, testCase.secondWindow} {
query := &mockQuery{fingerprint: "absorb-query", startMs: window.From, endMs: window.To}
result := &qbtypes.Result{
bc.Put(ctx, orgID, query, step, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", window.From, window.To, 1000),
// one row per second, so a bucket's stats read as the seconds of data it holds
// one row per second, so a bucket's stats read as the seconds it was scanned for
Stats: qbtypes.ExecStats{RowsScanned: (window.To - window.From) / 1000},
}
bc.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{window: result.Stats})
})
}
query := &mockQuery{
@@ -1611,7 +1609,7 @@ func TestBucketCachePutLeavesAnUnchangedCellAlone(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 10000, 1000),
Warnings: []string{"from the wide write"},
}, nil)
})
// a write already covered by the cell's bucket must not rewrite the entry,
// which shows as its warnings never arriving
@@ -1620,7 +1618,7 @@ func TestBucketCachePutLeavesAnUnchangedCellAlone(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 3000, 5000, 1000),
Warnings: []string{"from the inside write"},
}, nil)
})
var cellEntry qbtypes.CachedData
require.NoError(t, bc.cache.Get(ctx, orgID, bc.generateCacheKey(wide, 0), &cellEntry))

View File

@@ -823,7 +823,7 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
return nil, err
}
// Store in cache for future use
q.bucketCache.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: startMs, To: endMs}: result.Stats})
q.bucketCache.Put(ctx, orgID, query, step, result)
return result, nil
}
}
@@ -879,19 +879,16 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
if err != nil {
return nil, err
}
startMs, endMs := query.Window()
q.bucketCache.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: startMs, To: endMs}: result.Stats})
q.bucketCache.Put(ctx, orgID, query, step, result)
return result, nil
}
}
// Calculate total stats and filter out nil results
validResults := make([]*qbtypes.Result, 0, len(freshResults))
execStatsForQueriedTimeRanges := make(map[qbtypes.TimeRange]qbtypes.ExecStats, len(freshResults))
for i, result := range freshResults {
for _, result := range freshResults {
if result != nil {
validResults = append(validResults, result)
execStatsForQueriedTimeRanges[*missingRanges[i]] = result.Stats
totalStats.RowsScanned += result.Stats.RowsScanned
totalStats.BytesScanned += result.Stats.BytesScanned
totalStats.DurationMS += result.Stats.DurationMS
@@ -906,7 +903,7 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
mergedResult.Stats.DurationMS += totalStats.DurationMS
// Store merged result in cache
q.bucketCache.Put(ctx, orgID, query, step, mergedResult, execStatsForQueriedTimeRanges)
q.bucketCache.Put(ctx, orgID, query, step, mergedResult)
return mergedResult, nil
}

View File

@@ -43,23 +43,6 @@ func (c *CachedBucket) Touches(other *CachedBucket) bool {
return c.StartMs <= other.EndMs && other.StartMs <= c.EndMs
}
// ProrateStatsToWindow returns the share of the stats for the part of the
// bucket inside [startMs, endMs).
func (c *CachedBucket) ProrateStatsToWindow(startMs, endMs uint64) ExecStats {
return c.Stats.prorate(overlapMs(c.StartMs, c.EndMs, startMs, endMs), c.EndMs-c.StartMs)
}
// ProrateStatsToUnqueried returns the share of the stats for the part of the
// bucket no queried range covers.
func (c *CachedBucket) ProrateStatsToUnqueried(execStatsForQueriedTimeRanges map[TimeRange]ExecStats) ExecStats {
widthMs := c.EndMs - c.StartMs
queriedMs := uint64(0)
for queried := range execStatsForQueriedTimeRanges {
queriedMs += overlapMs(c.StartMs, c.EndMs, queried.From, queried.To)
}
return c.Stats.prorate(widthMs-min(queriedMs, widthMs), widthMs)
}
// CachedData represents the full cached data for a query.
type CachedData struct {
Buckets []*CachedBucket `json:"buckets"`
@@ -105,7 +88,7 @@ func (c *CachedData) Cost() int64 {
return size
}
func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64, execStatsForQueriedTimeRanges map[TimeRange]ExecStats) ([]*CachedBucket, error) {
func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64) ([]*CachedBucket, error) {
tsData, ok := r.Value.(*TimeSeriesData)
if !ok || tsData == nil {
return nil, errors.NewInternalf(errors.CodeInternal, "result of type %s cannot be bucketed", r.Type.StringValue())
@@ -128,39 +111,18 @@ func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64, e
if err != nil {
return nil, err
}
// A slice costs what was queried inside it; the rest of the result came
// out of the cache and is costed there already
sliceStats := ExecStats{StepIntervals: maps.Clone(r.Stats.StepIntervals)}
for queried, stats := range execStatsForQueriedTimeRanges {
queriedMs := overlapMs(from, to, queried.From, queried.To)
if queriedMs == 0 {
continue
}
share := stats.prorate(queriedMs, queried.To-queried.From)
sliceStats.RowsScanned += share.RowsScanned
sliceStats.BytesScanned += share.BytesScanned
sliceStats.DurationMS += share.DurationMS
}
cellBuckets = append(cellBuckets, &CachedBucket{
StartMs: from,
EndMs: to,
Type: r.Type,
Value: value,
Stats: sliceStats,
Stats: r.Stats.prorate(to-from, endMs-startMs),
})
from = to
}
return cellBuckets, nil
}
// overlapMs is the length of [aFromMs, aToMs) ∩ [bFromMs, bToMs).
func overlapMs(aFromMs, aToMs, bFromMs, bToMs uint64) uint64 {
if fromMs, toMs := max(aFromMs, bFromMs), min(aToMs, bToMs); fromMs < toMs {
return toMs - fromMs
}
return 0
}
// prorate returns the share of the stats that partMs out of wholeMs of the
// scanned window accounts for.
func (e ExecStats) prorate(partMs, wholeMs uint64) ExecStats {

View File

@@ -442,55 +442,3 @@ def test_builder_refreshing_a_sliding_time_range(
assert uncached.status_code == HTTPStatus.OK, uncached.text
assert_results_equal(from_cache.json(), uncached.json(), "A", "the twentieth refresh")
def test_builder_refreshing_does_not_inflate_rows_scanned(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_metrics: Callable[[list[Metrics]], None],
) -> None:
metric_name = f"cache_rows_scanned_{uuid4().hex[:8]}"
# 90 minutes back so even the last refresh closes clear of the flux
# interval, which holds recent data out of the cache
start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=90)).timestamp()) // 60 * 60, tz=UTC)
start_time_ms = int(start_time.timestamp() * 1000)
query = [build_builder_query("A", metric_name, "max", "max")]
insert_metrics(
[
Metrics(
metric_name=metric_name,
labels={"service": "api"},
timestamp=start_time + timedelta(minutes=minute),
value=minute,
type_="Gauge",
is_monotonic=False,
)
for minute in range(70)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# a dashboard left open on a thirty minute range, re-running a minute later
# each time. rowsScanned is a cost indicator, so a refresh served mostly out
# of the cache cannot claim more than every read made so far: the first
# full read and then one new minute per refresh, measured with the cache off
uncached_rows_scanned_so_far = 0
for refresh in range(10):
refresh_start_ms = start_time_ms + refresh * MINUTE_MS
refresh_end_ms = refresh_start_ms + 30 * MINUTE_MS
uncached_start_ms = refresh_start_ms if refresh == 0 else refresh_end_ms - MINUTE_MS
uncached_response = make_query_request(signoz, token, uncached_start_ms, refresh_end_ms, query, no_cache=True)
assert uncached_response.status_code == HTTPStatus.OK, f"refresh {refresh}: uncached read failed: {uncached_response.text}"
uncached_rows_scanned_so_far += uncached_response.json()["data"]["meta"]["rowsScanned"]
assert uncached_rows_scanned_so_far > 0, f"refresh {refresh}: uncached read reported no rows scanned"
cached_response = make_query_request(signoz, token, refresh_start_ms, refresh_end_ms, query, no_cache=False)
assert cached_response.status_code == HTTPStatus.OK, f"refresh {refresh}: cached read failed: {cached_response.text}"
cached_rows_scanned = cached_response.json()["data"]["meta"]["rowsScanned"]
assert cached_rows_scanned <= uncached_rows_scanned_so_far, f"refresh {refresh}: cached read reported {cached_rows_scanned} rows scanned, more than the {uncached_rows_scanned_so_far} rows all uncached reads scanned so far"