mirror of
https://github.com/SigNoz/signoz.git
synced 2026-08-04 12:10:43 +01:00
Compare commits
2 Commits
nv/promql-
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c70866e4b3 | ||
|
|
34041be308 |
@@ -114,6 +114,23 @@ describe('getCaretContext — stage detection', () => {
|
||||
expect(ctx.partial).toBe('');
|
||||
});
|
||||
|
||||
it('never replaces past the caret when it sits before the operator', () => {
|
||||
const ctx = getCaretContext("env = 'prod'", 4);
|
||||
expect(ctx.stage).toBe('operator');
|
||||
expect(ctx.partial).toBe('');
|
||||
expect(ctx.replaceStart).toBe(4);
|
||||
expect(ctx.replaceEnd).toBe(4);
|
||||
});
|
||||
|
||||
it('never replaces past the caret when it sits before the value', () => {
|
||||
const ctx = getCaretContext("env = 'prod'", 6);
|
||||
expect(ctx.stage).toBe('value');
|
||||
expect(ctx.operator).toBe('=');
|
||||
expect(ctx.partial).toBe('');
|
||||
expect(ctx.replaceStart).toBe(6);
|
||||
expect(ctx.replaceEnd).toBe(6);
|
||||
});
|
||||
|
||||
it('detects the stage of the term under a mid-string caret', () => {
|
||||
const q = "env = AND team = 'core'";
|
||||
// caret right after the first `env ` (index 4) is the operator stage
|
||||
@@ -142,6 +159,13 @@ describe('spliceAtCaret', () => {
|
||||
expect(next).toBe("env = 'prod'");
|
||||
});
|
||||
|
||||
it('inserts (without duplicating text) at a caret parked before a token', () => {
|
||||
const q = "env = 'prod'";
|
||||
const ctx = getCaretContext(q, 4);
|
||||
const { next } = spliceAtCaret(q, ctx, '!= ');
|
||||
expect(next).toBe("env != = 'prod'");
|
||||
});
|
||||
|
||||
it('preserves text after the caret', () => {
|
||||
const q = "env AND team = 'core'";
|
||||
const ctx = getCaretContext(q, 4); // operator gap after `env`
|
||||
|
||||
@@ -328,7 +328,7 @@ export const getCaretContext = (query: string, caret: number): CaretContext => {
|
||||
fieldKey: scan.key ? scan.key.text : '',
|
||||
operator: slot.operator,
|
||||
partial: slot.partial,
|
||||
replaceStart: term.start + slot.replaceStartRel,
|
||||
replaceStart: Math.min(term.start + slot.replaceStartRel, pos),
|
||||
replaceEnd: pos,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package implcloudintegration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
citypes "github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestServiceDefinitionsAreValid(t *testing.T) {
|
||||
store := NewServiceDefinitionStore()
|
||||
|
||||
for _, provider := range []citypes.CloudProviderType{
|
||||
citypes.CloudProviderTypeAWS,
|
||||
citypes.CloudProviderTypeAzure,
|
||||
citypes.CloudProviderTypeGCP,
|
||||
} {
|
||||
t.Run(provider.StringValue(), func(t *testing.T) {
|
||||
defs, err := store.List(context.Background(), provider)
|
||||
require.NoError(t, err, "all embedded definitions must load and validate")
|
||||
require.NotEmpty(t, defs, "provider should ship at least one service definition")
|
||||
|
||||
for _, def := range defs {
|
||||
assert.NotEmpty(t, def.ID, "service definition must have an id")
|
||||
assert.NotEmpty(t, def.Title, "service %q must have a title", def.ID)
|
||||
|
||||
// Get() must agree with List() for every service it advertises.
|
||||
serviceID, err := citypes.NewServiceID(provider, def.ID)
|
||||
if !assert.NoError(t, err, "service id %q must be registered in serviceid.go", def.ID) {
|
||||
continue
|
||||
}
|
||||
got, err := store.Get(context.Background(), provider, serviceID)
|
||||
require.NoError(t, err, "service %q listed but not gettable", def.ID)
|
||||
assert.Equal(t, def.ID, got.ID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -621,7 +621,7 @@
|
||||
{
|
||||
"metricName": "cloudsql.googleapis.com/database/postgresql/deadlock_count",
|
||||
"temporality": "",
|
||||
"timeAggregation": "max",
|
||||
"timeAggregation": "rate",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
@@ -882,7 +882,7 @@
|
||||
{
|
||||
"metricName": "cloudsql.googleapis.com/database/postgresql/transaction_count",
|
||||
"temporality": "",
|
||||
"timeAggregation": "max",
|
||||
"timeAggregation": "rate",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
|
||||
@@ -54,13 +54,13 @@
|
||||
{
|
||||
"name": "cloudsql.googleapis.com/database/postgresql/transaction_count",
|
||||
"unit": "Count",
|
||||
"type": "Gauge",
|
||||
"type": "Sum",
|
||||
"description": ""
|
||||
},
|
||||
{
|
||||
"name": "cloudsql.googleapis.com/database/postgresql/deadlock_count",
|
||||
"unit": "Count",
|
||||
"type": "Gauge",
|
||||
"type": "Sum",
|
||||
"description": ""
|
||||
},
|
||||
{
|
||||
|
||||
@@ -925,7 +925,7 @@
|
||||
"metricName": "compute.googleapis.com/instance/disk/average_io_latency",
|
||||
"temporality": "",
|
||||
"timeAggregation": "avg",
|
||||
"spaceAggregation": "sum",
|
||||
"spaceAggregation": "max",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
|
||||
@@ -1207,8 +1207,8 @@
|
||||
{
|
||||
"metricName": "kubernetes.io/container/restart_count",
|
||||
"temporality": "",
|
||||
"timeAggregation": "max",
|
||||
"spaceAggregation": "max",
|
||||
"timeAggregation": "increase",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
|
||||
@@ -601,8 +601,8 @@
|
||||
{
|
||||
"metricName": "redis.googleapis.com/stats/cache_hit_ratio",
|
||||
"temporality": "",
|
||||
"timeAggregation": "max",
|
||||
"spaceAggregation": "sum",
|
||||
"timeAggregation": "min",
|
||||
"spaceAggregation": "min",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
@@ -788,7 +788,7 @@
|
||||
"metricName": "redis.googleapis.com/commands/usec_per_call",
|
||||
"temporality": "",
|
||||
"timeAggregation": "max",
|
||||
"spaceAggregation": "sum",
|
||||
"spaceAggregation": "max",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
@@ -945,7 +945,7 @@
|
||||
{
|
||||
"metricName": "redis.googleapis.com/commands/calls",
|
||||
"temporality": "",
|
||||
"timeAggregation": "max",
|
||||
"timeAggregation": "rate",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
@@ -1044,8 +1044,8 @@
|
||||
{
|
||||
"metricName": "redis.googleapis.com/stats/reject_connections_count",
|
||||
"temporality": "",
|
||||
"timeAggregation": "avg",
|
||||
"spaceAggregation": "avg",
|
||||
"timeAggregation": "rate",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
@@ -1213,4 +1213,4 @@
|
||||
"refreshInterval": "",
|
||||
"links": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package querier
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -1402,61 +1401,3 @@ func TestBucketCache_NoCache(t *testing.T) {
|
||||
// The actual NoCache logic is implemented in querier.run(), not in bucket cache
|
||||
// This test verifies that the cache works normally and NoCache bypasses it at a higher level
|
||||
}
|
||||
|
||||
// A promql ratio yields NaN wherever the denominator is zero. If those do not
|
||||
// survive the cache, every good point in the same bucket is lost with them.
|
||||
func TestBucketCacheServesBucketsHoldingNonFiniteValues(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
orgID := valuer.GenerateUUID()
|
||||
bc := NewBucketCache(instrumentationtest.New().ToProviderSettings(), createTestCache(t), cacheTTL, defaultFluxInterval)
|
||||
|
||||
step := qbtypes.Step{Duration: 300 * time.Second}
|
||||
stepMs := uint64(step.Milliseconds())
|
||||
end := (uint64(time.Now().UnixMilli()) - uint64(20*time.Minute.Milliseconds())) / stepMs * stepMs
|
||||
start := end - uint64(36*time.Hour.Milliseconds())
|
||||
|
||||
series := &qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{
|
||||
Key: telemetrytypes.TelemetryFieldKey{Name: "job_name"},
|
||||
Value: "dbBloatMonitorJob",
|
||||
}},
|
||||
}
|
||||
finitePoints := 0
|
||||
for ts := start; ts < end; ts += stepMs {
|
||||
value := 11.524
|
||||
if (ts/stepMs)%7 == 0 {
|
||||
value = math.NaN()
|
||||
} else {
|
||||
finitePoints++
|
||||
}
|
||||
series.Values = append(series.Values, &qbtypes.TimeSeriesValue{Timestamp: int64(ts), Value: value})
|
||||
}
|
||||
|
||||
q := &mockQuery{fingerprint: "promql&ratio&5m0s", startMs: start, endMs: end}
|
||||
bc.Put(ctx, orgID, q, step, &qbtypes.Result{
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: &qbtypes.TimeSeriesData{
|
||||
QueryName: "A",
|
||||
Aggregations: []*qbtypes.AggregationBucket{{Series: []*qbtypes.TimeSeries{series}}},
|
||||
},
|
||||
})
|
||||
|
||||
cached, missing := bc.GetMissRanges(ctx, orgID, q, step)
|
||||
require.NotNil(t, cached)
|
||||
|
||||
servedFinite := 0
|
||||
tsData, ok := cached.Value.(*qbtypes.TimeSeriesData)
|
||||
require.True(t, ok)
|
||||
for _, agg := range tsData.Aggregations {
|
||||
for _, s := range agg.Series {
|
||||
for _, v := range s.Values {
|
||||
if !math.IsNaN(v.Value) {
|
||||
servedFinite++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, finitePoints, servedFinite, "every finite point in the bucket is still served")
|
||||
assert.Empty(t, missing, "and the covered span needs no re-query")
|
||||
}
|
||||
|
||||
@@ -492,75 +492,6 @@ func (t TimeSeriesValue) MarshalJSON() ([]byte, error) {
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalJSON inverts MarshalJSON, which renders non-finite floats as the
|
||||
// strings "NaN"/"Inf"/"-Inf". The bucket cache serializes through this type,
|
||||
// so a value it cannot read back costs it the whole cached entry.
|
||||
func (t *TimeSeriesValue) UnmarshalJSON(data []byte) error {
|
||||
type Alias TimeSeriesValue
|
||||
|
||||
aux := &struct {
|
||||
*Alias
|
||||
Value json.RawMessage `json:"value"`
|
||||
Values []json.RawMessage `json:"values,omitempty"`
|
||||
}{
|
||||
Alias: (*Alias)(t),
|
||||
}
|
||||
if err := json.Unmarshal(data, aux); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var err error
|
||||
if t.Value, err = parseFloatOrNonFinite(aux.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
t.Values, err = parseFloatsOrNonFinite(aux.Values)
|
||||
return err
|
||||
}
|
||||
|
||||
// parseFloatOrNonFinite inverts sanitizeValue for one float: a JSON number, or
|
||||
// a sentinel string standing in for a value JSON cannot represent.
|
||||
func parseFloatOrNonFinite(raw json.RawMessage) (float64, error) {
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var f float64
|
||||
if err := json.Unmarshal(raw, &f); err == nil {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "value %s is neither a number nor a non-finite sentinel", raw)
|
||||
}
|
||||
switch s {
|
||||
case "NaN":
|
||||
return math.NaN(), nil
|
||||
case "Inf", "+Inf":
|
||||
return math.Inf(1), nil
|
||||
case "-Inf":
|
||||
return math.Inf(-1), nil
|
||||
default:
|
||||
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "unrecognized non-finite value %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
// parseFloatsOrNonFinite does the same for Values, keeping the nil/empty
|
||||
// distinction MarshalJSON preserves.
|
||||
func parseFloatsOrNonFinite(raw []json.RawMessage) ([]float64, error) {
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
values := make([]float64, len(raw))
|
||||
for idx := range raw {
|
||||
var err error
|
||||
if values[idx], err = parseFloatOrNonFinite(raw[idx]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (r RawData) MarshalJSON() ([]byte, error) {
|
||||
type Alias RawData
|
||||
return json.Marshal((*Alias)(&r))
|
||||
|
||||
@@ -428,111 +428,3 @@ func TestRoundToNonZeroDecimals(t *testing.T) {
|
||||
assert.Equal(t, math.Inf(1), roundToNonZeroDecimals(math.Inf(1), 3))
|
||||
assert.Equal(t, math.Inf(-1), roundToNonZeroDecimals(math.Inf(-1), 3))
|
||||
}
|
||||
|
||||
func TestTimeSeriesValueUnmarshalJSONNonFinite(t *testing.T) {
|
||||
cases := []struct {
|
||||
description string
|
||||
encoded string
|
||||
expectedValue float64
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
description: "finite value decodes as a number",
|
||||
encoded: `{"timestamp":1000,"value":1.5}`,
|
||||
expectedValue: 1.5,
|
||||
},
|
||||
{
|
||||
description: "NaN sentinel decodes back to NaN",
|
||||
encoded: `{"timestamp":1000,"value":"NaN"}`,
|
||||
expectedValue: math.NaN(),
|
||||
},
|
||||
{
|
||||
description: "Inf sentinel decodes back to positive infinity",
|
||||
encoded: `{"timestamp":1000,"value":"Inf"}`,
|
||||
expectedValue: math.Inf(1),
|
||||
},
|
||||
{
|
||||
description: "negative Inf sentinel decodes back to negative infinity",
|
||||
encoded: `{"timestamp":1000,"value":"-Inf"}`,
|
||||
expectedValue: math.Inf(-1),
|
||||
},
|
||||
{
|
||||
description: "null decodes to zero",
|
||||
encoded: `{"timestamp":1000,"value":null}`,
|
||||
expectedValue: 0,
|
||||
},
|
||||
{
|
||||
description: "an unrecognized string is still an error",
|
||||
encoded: `{"timestamp":1000,"value":"banana"}`,
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.description, func(t *testing.T) {
|
||||
var got TimeSeriesValue
|
||||
err := json.Unmarshal([]byte(c.encoded), &got)
|
||||
if c.expectError {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
assert.NoError(t, err)
|
||||
assert.EqualValues(t, 1000, got.Timestamp)
|
||||
if math.IsNaN(c.expectedValue) {
|
||||
assert.True(t, math.IsNaN(got.Value))
|
||||
return
|
||||
}
|
||||
assert.Equal(t, c.expectedValue, got.Value)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimeSeriesValueRoundTripsNonFiniteValues(t *testing.T) {
|
||||
original := &TimeSeries{
|
||||
Labels: []*Label{
|
||||
{Key: telemetrytypes.TelemetryFieldKey{Name: "job_name"}, Value: "dbBloatMonitorJob"},
|
||||
},
|
||||
Values: []*TimeSeriesValue{
|
||||
{Timestamp: 1000, Value: 11.524},
|
||||
{Timestamp: 2000, Value: math.NaN()},
|
||||
{Timestamp: 3000, Value: math.Inf(1)},
|
||||
{Timestamp: 4000, Value: math.Inf(-1)},
|
||||
{Timestamp: 5000, Value: 456.7},
|
||||
},
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(original)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var decoded *TimeSeries
|
||||
err = json.Unmarshal(encoded, &decoded)
|
||||
assert.NoError(t, err, "a response containing non-finite values must survive the round trip")
|
||||
assert.Len(t, decoded.Values, 5)
|
||||
assert.Equal(t, 11.524, decoded.Values[0].Value)
|
||||
assert.True(t, math.IsNaN(decoded.Values[1].Value))
|
||||
assert.True(t, math.IsInf(decoded.Values[2].Value, 1))
|
||||
assert.True(t, math.IsInf(decoded.Values[3].Value, -1))
|
||||
assert.Equal(t, 456.7, decoded.Values[4].Value)
|
||||
}
|
||||
|
||||
func TestTimeSeriesValueRoundTripsHeatmapValues(t *testing.T) {
|
||||
original := &TimeSeriesValue{
|
||||
Timestamp: 1000,
|
||||
Value: 2.5,
|
||||
Values: []float64{1.5, math.NaN(), 3.5},
|
||||
Bucket: &Bucket{Step: 10},
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(original)
|
||||
assert.NoError(t, err)
|
||||
|
||||
var decoded TimeSeriesValue
|
||||
err = json.Unmarshal(encoded, &decoded)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 2.5, decoded.Value)
|
||||
assert.Len(t, decoded.Values, 3)
|
||||
assert.Equal(t, 1.5, decoded.Values[0])
|
||||
assert.True(t, math.IsNaN(decoded.Values[1]))
|
||||
assert.Equal(t, 3.5, decoded.Values[2])
|
||||
assert.Equal(t, float64(10), decoded.Bucket.Step)
|
||||
}
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
"""
|
||||
Regression test for caching PromQL results that contain non-finite values.
|
||||
|
||||
A ratio yields NaN where both sides are zero, and NaN marshals as the string
|
||||
"NaN". Before the fix the cached bucket could not be read back, so a second
|
||||
identical request returned only the window edges.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.metrics import Metrics
|
||||
from fixtures.querier import get_all_series, make_query_request
|
||||
|
||||
SUM_METRIC = "job_duration_sum"
|
||||
COUNT_METRIC = "job_duration_count"
|
||||
HOUR_MS = 3_600_000
|
||||
SAMPLE_INTERVAL_MS = 60_000
|
||||
|
||||
|
||||
def test_cached_promql_result_with_nan_matches_uncached(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
# 12h ending on an hour boundary 15m ago — old enough to be cached.
|
||||
end_ms = (int((datetime.now(tz=UTC) - timedelta(minutes=15)).timestamp() * 1000) // HOUR_MS) * HOUR_MS
|
||||
start_ms = end_ms - 12 * HOUR_MS
|
||||
|
||||
# active_job divides finite; idle_job is 0/0, the NaN the cache must survive.
|
||||
series = {"active_job": (100.0, 4.0), "idle_job": (0.0, 0.0)}
|
||||
metrics: list[Metrics] = []
|
||||
for job_name, (sum_value, count_value) in series.items():
|
||||
for ts_ms in range(start_ms, end_ms + 1, SAMPLE_INTERVAL_MS):
|
||||
timestamp = datetime.fromtimestamp(ts_ms / 1000, tz=UTC)
|
||||
metrics.append(Metrics(metric_name=SUM_METRIC, labels={"job_name": job_name}, timestamp=timestamp, value=sum_value))
|
||||
metrics.append(Metrics(metric_name=COUNT_METRIC, labels={"job_name": job_name}, timestamp=timestamp, value=count_value))
|
||||
insert_metrics(metrics)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
promql = f"sum by (job_name) ({SUM_METRIC}) / sum by (job_name) ({COUNT_METRIC})"
|
||||
|
||||
def run() -> tuple[dict[str, dict[int, object]], int]:
|
||||
query = {"type": "promql", "spec": {"name": "A", "query": promql}}
|
||||
response = make_query_request(signoz, token, start_ms, end_ms, [query], no_cache=False)
|
||||
assert response.status_code == HTTPStatus.OK, response.text[:300]
|
||||
body = response.json()
|
||||
out: dict[str, dict[int, object]] = {}
|
||||
for entry in get_all_series(body, "A") or []:
|
||||
labels = {l["key"]["name"]: str(l["value"]) for l in entry.get("labels") or []}
|
||||
out[labels["job_name"]] = {v["timestamp"]: v["value"] for v in entry.get("values") or []}
|
||||
return out, int(body["data"]["meta"]["stepIntervals"]["A"])
|
||||
|
||||
# First populates the cache, second must be served from it.
|
||||
first, step_seconds = run()
|
||||
second, _ = run()
|
||||
|
||||
expected_points = (end_ms - start_ms) // (step_seconds * 1000) + 1
|
||||
assert set(first) == set(series), sorted(first)
|
||||
assert set(first["idle_job"].values()) == {"NaN"}, sorted(set(first["idle_job"].values()))
|
||||
assert set(first["active_job"].values()) == {25.0}, sorted(set(first["active_job"].values()))
|
||||
assert len(first["idle_job"]) == expected_points, f"expected {expected_points} points, got {len(first['idle_job'])}"
|
||||
|
||||
# The cached read excludes end_ms, the one legitimate difference.
|
||||
assert set(second) == set(first), sorted(second)
|
||||
for job_name, points in first.items():
|
||||
expected = {ts: value for ts, value in points.items() if ts < end_ms}
|
||||
assert second[job_name] == expected, f"{job_name}: got {len(second[job_name])} of {len(expected)} points"
|
||||
Reference in New Issue
Block a user