Compare commits

...

6 Commits

Author SHA1 Message Date
Naman Verma
87ee5f99af chore: remove bucket cache change 2026-08-04 14:49:33 +05:30
Naman Verma
97093bf0e4 Merge branch 'main' into nv/promql-cache-nan 2026-08-04 14:36:50 +05:30
Naman Verma
f41f541d2f test: add integration test 2026-08-04 14:36:22 +05:30
Naman Verma
73d40051ac chore: rearrange methods 2026-08-04 13:29:30 +05:30
Naman Verma
025dccec69 Merge branch 'main' into nv/promql-cache-nan 2026-08-04 12:55:45 +05:30
Naman Verma
e7000bbaa6 chore: first draft of NaN cache fix 2026-08-03 23:06:09 +05:30
4 changed files with 308 additions and 0 deletions

View File

@@ -3,6 +3,7 @@ package querier
import (
"context"
"fmt"
"math"
"testing"
"time"
@@ -1401,3 +1402,61 @@ func TestBucketCache_NoCache(t *testing.T) {
// The actual NoCache logic is implemented in querier.run(), not in bucket cache
// This test verifies that the cache works normally and NoCache bypasses it at a higher level
}
// A promql ratio yields NaN wherever the denominator is zero. If those do not
// survive the cache, every good point in the same bucket is lost with them.
func TestBucketCacheServesBucketsHoldingNonFiniteValues(t *testing.T) {
ctx := context.Background()
orgID := valuer.GenerateUUID()
bc := NewBucketCache(instrumentationtest.New().ToProviderSettings(), createTestCache(t), cacheTTL, defaultFluxInterval)
step := qbtypes.Step{Duration: 300 * time.Second}
stepMs := uint64(step.Milliseconds())
end := (uint64(time.Now().UnixMilli()) - uint64(20*time.Minute.Milliseconds())) / stepMs * stepMs
start := end - uint64(36*time.Hour.Milliseconds())
series := &qbtypes.TimeSeries{
Labels: []*qbtypes.Label{{
Key: telemetrytypes.TelemetryFieldKey{Name: "job_name"},
Value: "dbBloatMonitorJob",
}},
}
finitePoints := 0
for ts := start; ts < end; ts += stepMs {
value := 11.524
if (ts/stepMs)%7 == 0 {
value = math.NaN()
} else {
finitePoints++
}
series.Values = append(series.Values, &qbtypes.TimeSeriesValue{Timestamp: int64(ts), Value: value})
}
q := &mockQuery{fingerprint: "promql&ratio&5m0s", startMs: start, endMs: end}
bc.Put(ctx, orgID, q, step, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: &qbtypes.TimeSeriesData{
QueryName: "A",
Aggregations: []*qbtypes.AggregationBucket{{Series: []*qbtypes.TimeSeries{series}}},
},
})
cached, missing := bc.GetMissRanges(ctx, orgID, q, step)
require.NotNil(t, cached)
servedFinite := 0
tsData, ok := cached.Value.(*qbtypes.TimeSeriesData)
require.True(t, ok)
for _, agg := range tsData.Aggregations {
for _, s := range agg.Series {
for _, v := range s.Values {
if !math.IsNaN(v.Value) {
servedFinite++
}
}
}
}
assert.Equal(t, finitePoints, servedFinite, "every finite point in the bucket is still served")
assert.Empty(t, missing, "and the covered span needs no re-query")
}

View File

@@ -492,6 +492,75 @@ func (t TimeSeriesValue) MarshalJSON() ([]byte, error) {
})
}
// UnmarshalJSON inverts MarshalJSON, which renders non-finite floats as the
// strings "NaN"/"Inf"/"-Inf". The bucket cache serializes through this type,
// so a value it cannot read back costs it the whole cached entry.
func (t *TimeSeriesValue) UnmarshalJSON(data []byte) error {
type Alias TimeSeriesValue
aux := &struct {
*Alias
Value json.RawMessage `json:"value"`
Values []json.RawMessage `json:"values,omitempty"`
}{
Alias: (*Alias)(t),
}
if err := json.Unmarshal(data, aux); err != nil {
return err
}
var err error
if t.Value, err = parseFloatOrNonFinite(aux.Value); err != nil {
return err
}
t.Values, err = parseFloatsOrNonFinite(aux.Values)
return err
}
// parseFloatOrNonFinite inverts sanitizeValue for one float: a JSON number, or
// a sentinel string standing in for a value JSON cannot represent.
func parseFloatOrNonFinite(raw json.RawMessage) (float64, error) {
if len(raw) == 0 || string(raw) == "null" {
return 0, nil
}
var f float64
if err := json.Unmarshal(raw, &f); err == nil {
return f, nil
}
var s string
if err := json.Unmarshal(raw, &s); err != nil {
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "value %s is neither a number nor a non-finite sentinel", raw)
}
switch s {
case "NaN":
return math.NaN(), nil
case "Inf", "+Inf":
return math.Inf(1), nil
case "-Inf":
return math.Inf(-1), nil
default:
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "unrecognized non-finite value %q", s)
}
}
// parseFloatsOrNonFinite does the same for Values, keeping the nil/empty
// distinction MarshalJSON preserves.
func parseFloatsOrNonFinite(raw []json.RawMessage) ([]float64, error) {
if raw == nil {
return nil, nil
}
values := make([]float64, len(raw))
for idx := range raw {
var err error
if values[idx], err = parseFloatOrNonFinite(raw[idx]); err != nil {
return nil, err
}
}
return values, nil
}
func (r RawData) MarshalJSON() ([]byte, error) {
type Alias RawData
return json.Marshal((*Alias)(&r))

View File

@@ -428,3 +428,111 @@ func TestRoundToNonZeroDecimals(t *testing.T) {
assert.Equal(t, math.Inf(1), roundToNonZeroDecimals(math.Inf(1), 3))
assert.Equal(t, math.Inf(-1), roundToNonZeroDecimals(math.Inf(-1), 3))
}
func TestTimeSeriesValueUnmarshalJSONNonFinite(t *testing.T) {
cases := []struct {
description string
encoded string
expectedValue float64
expectError bool
}{
{
description: "finite value decodes as a number",
encoded: `{"timestamp":1000,"value":1.5}`,
expectedValue: 1.5,
},
{
description: "NaN sentinel decodes back to NaN",
encoded: `{"timestamp":1000,"value":"NaN"}`,
expectedValue: math.NaN(),
},
{
description: "Inf sentinel decodes back to positive infinity",
encoded: `{"timestamp":1000,"value":"Inf"}`,
expectedValue: math.Inf(1),
},
{
description: "negative Inf sentinel decodes back to negative infinity",
encoded: `{"timestamp":1000,"value":"-Inf"}`,
expectedValue: math.Inf(-1),
},
{
description: "null decodes to zero",
encoded: `{"timestamp":1000,"value":null}`,
expectedValue: 0,
},
{
description: "an unrecognized string is still an error",
encoded: `{"timestamp":1000,"value":"banana"}`,
expectError: true,
},
}
for _, c := range cases {
t.Run(c.description, func(t *testing.T) {
var got TimeSeriesValue
err := json.Unmarshal([]byte(c.encoded), &got)
if c.expectError {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.EqualValues(t, 1000, got.Timestamp)
if math.IsNaN(c.expectedValue) {
assert.True(t, math.IsNaN(got.Value))
return
}
assert.Equal(t, c.expectedValue, got.Value)
})
}
}
func TestTimeSeriesValueRoundTripsNonFiniteValues(t *testing.T) {
original := &TimeSeries{
Labels: []*Label{
{Key: telemetrytypes.TelemetryFieldKey{Name: "job_name"}, Value: "dbBloatMonitorJob"},
},
Values: []*TimeSeriesValue{
{Timestamp: 1000, Value: 11.524},
{Timestamp: 2000, Value: math.NaN()},
{Timestamp: 3000, Value: math.Inf(1)},
{Timestamp: 4000, Value: math.Inf(-1)},
{Timestamp: 5000, Value: 456.7},
},
}
encoded, err := json.Marshal(original)
assert.NoError(t, err)
var decoded *TimeSeries
err = json.Unmarshal(encoded, &decoded)
assert.NoError(t, err, "a response containing non-finite values must survive the round trip")
assert.Len(t, decoded.Values, 5)
assert.Equal(t, 11.524, decoded.Values[0].Value)
assert.True(t, math.IsNaN(decoded.Values[1].Value))
assert.True(t, math.IsInf(decoded.Values[2].Value, 1))
assert.True(t, math.IsInf(decoded.Values[3].Value, -1))
assert.Equal(t, 456.7, decoded.Values[4].Value)
}
func TestTimeSeriesValueRoundTripsHeatmapValues(t *testing.T) {
original := &TimeSeriesValue{
Timestamp: 1000,
Value: 2.5,
Values: []float64{1.5, math.NaN(), 3.5},
Bucket: &Bucket{Step: 10},
}
encoded, err := json.Marshal(original)
assert.NoError(t, err)
var decoded TimeSeriesValue
err = json.Unmarshal(encoded, &decoded)
assert.NoError(t, err)
assert.Equal(t, 2.5, decoded.Value)
assert.Len(t, decoded.Values, 3)
assert.Equal(t, 1.5, decoded.Values[0])
assert.True(t, math.IsNaN(decoded.Values[1]))
assert.Equal(t, 3.5, decoded.Values[2])
assert.Equal(t, float64(10), decoded.Bucket.Step)
}

View File

@@ -0,0 +1,72 @@
"""
Regression test for caching PromQL results that contain non-finite values.
A ratio yields NaN where both sides are zero, and NaN marshals as the string
"NaN". Before the fix the cached bucket could not be read back, so a second
identical request returned only the window edges.
"""
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.metrics import Metrics
from fixtures.querier import get_all_series, make_query_request
SUM_METRIC = "job_duration_sum"
COUNT_METRIC = "job_duration_count"
HOUR_MS = 3_600_000
SAMPLE_INTERVAL_MS = 60_000
def test_cached_promql_result_with_nan_matches_uncached(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_metrics: Callable[[list[Metrics]], None],
) -> None:
# 12h ending on an hour boundary 15m ago — old enough to be cached.
end_ms = (int((datetime.now(tz=UTC) - timedelta(minutes=15)).timestamp() * 1000) // HOUR_MS) * HOUR_MS
start_ms = end_ms - 12 * HOUR_MS
# active_job divides finite; idle_job is 0/0, the NaN the cache must survive.
series = {"active_job": (100.0, 4.0), "idle_job": (0.0, 0.0)}
metrics: list[Metrics] = []
for job_name, (sum_value, count_value) in series.items():
for ts_ms in range(start_ms, end_ms + 1, SAMPLE_INTERVAL_MS):
timestamp = datetime.fromtimestamp(ts_ms / 1000, tz=UTC)
metrics.append(Metrics(metric_name=SUM_METRIC, labels={"job_name": job_name}, timestamp=timestamp, value=sum_value))
metrics.append(Metrics(metric_name=COUNT_METRIC, labels={"job_name": job_name}, timestamp=timestamp, value=count_value))
insert_metrics(metrics)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
promql = f"sum by (job_name) ({SUM_METRIC}) / sum by (job_name) ({COUNT_METRIC})"
def run() -> tuple[dict[str, dict[int, object]], int]:
query = {"type": "promql", "spec": {"name": "A", "query": promql}}
response = make_query_request(signoz, token, start_ms, end_ms, [query], no_cache=False)
assert response.status_code == HTTPStatus.OK, response.text[:300]
body = response.json()
out: dict[str, dict[int, object]] = {}
for entry in get_all_series(body, "A") or []:
labels = {l["key"]["name"]: str(l["value"]) for l in entry.get("labels") or []}
out[labels["job_name"]] = {v["timestamp"]: v["value"] for v in entry.get("values") or []}
return out, int(body["data"]["meta"]["stepIntervals"]["A"])
# First populates the cache, second must be served from it.
first, step_seconds = run()
second, _ = run()
expected_points = (end_ms - start_ms) // (step_seconds * 1000) + 1
assert set(first) == set(series), sorted(first)
assert set(first["idle_job"].values()) == {"NaN"}, sorted(set(first["idle_job"].values()))
assert set(first["active_job"].values()) == {25.0}, sorted(set(first["active_job"].values()))
assert len(first["idle_job"]) == expected_points, f"expected {expected_points} points, got {len(first['idle_job'])}"
# The cached read excludes end_ms, the one legitimate difference.
assert set(second) == set(first), sorted(second)
for job_name, points in first.items():
expected = {ts: value for ts, value in points.items() if ts < end_ms}
assert second[job_name] == expected, f"{job_name}: got {len(second[job_name])} of {len(expected)} points"