Compare commits

..

5 Commits

Author SHA1 Message Date
Naman Verma
cd359e676e Merge branch 'main' into nv/4189-3 2026-06-03 17:10:04 +05:30
Naman Verma
a0b14e0835 fix: do not show errors for non-existent cost meter metrics (#10843)
* fix: show warning for non-existent cost meter metrics

* chore: lint fix by removing unused list

* chore: py fmt add new line

* chore: missing newline between tests

* fix: no warnings or errors for internal metrics

* fix: pylint fix by adding new line

* fix: lint fix in test
2026-06-03 10:09:08 +00:00
Naman Verma
3cf7642d2c Merge branch 'main' into nv/4189-3 2026-06-03 11:35:36 +05:30
Naman Verma
7551064aac test: correct errors pkg in test file 2026-05-21 13:14:40 +05:30
Naman Verma
d00be5895d fix: add check for percentile aggregation for non-histogram metrics 2026-05-21 11:57:38 +05:30
12 changed files with 108 additions and 341 deletions

View File

@@ -54,7 +54,7 @@ func (c Config) Validate() error {
if c.MaxConcurrentQueries <= 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "max_concurrent_queries must be positive, got %v", c.MaxConcurrentQueries)
}
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold <= 0 {
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
}
return nil

View File

@@ -217,7 +217,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
}
}
preseededResults := make(map[string]any)
for _, name := range missingMetricQueries { // at this point missing metrics will not have any non existent metrics, only normal ones
for _, name := range missingMetricQueries {
switch req.RequestType {
case qbtypes.RequestTypeTimeSeries:
preseededResults[name] = &qbtypes.TimeSeriesData{QueryName: name}
@@ -375,11 +375,24 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, queries []qbtypes.Q
return missingMetricQueries, "", nil
}
isInternalMetric := func(n string) bool { return strings.HasPrefix(n, "signoz.") || strings.HasPrefix(n, "signoz_") }
externalMissingMetrics := make([]string, 0, len(missingMetrics))
for _, m := range missingMetrics {
if !isInternalMetric(m) {
externalMissingMetrics = append(externalMissingMetrics, m)
}
}
if len(externalMissingMetrics) == 0 {
// this means all missing metrics are internal, and since internal metrics
// aren't user-controlled, skip errors/warnings for them since users can't act on them
return missingMetricQueries, "", nil
}
// Classify each missing metric: never-seen → NotFound error; seen-but-no-
// data-in-window → dormant warning.
lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, missingMetrics...)
lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, externalMissingMetrics...)
nonExistentMetrics := []string{}
for _, name := range missingMetrics {
for _, name := range externalMissingMetrics {
if ts, ok := lastSeenInfo[name]; ok && ts > 0 {
continue
}
@@ -400,11 +413,11 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, queries []qbtypes.Q
}
return name
}
if len(missingMetrics) == 1 {
if len(externalMissingMetrics) == 1 {
dormantWarning = fmt.Sprintf("no data found for the metric %s in the query time range", lastSeenStr(missingMetrics[0]))
} else {
parts := make([]string, len(missingMetrics))
for i, m := range missingMetrics {
parts := make([]string, len(externalMissingMetrics))
for i, m := range externalMissingMetrics {
parts[i] = lastSeenStr(m)
}
dormantWarning = fmt.Sprintf("no data found for the following metrics in the query time range: %s", strings.Join(parts, ", "))

View File

@@ -716,22 +716,6 @@ func aggOrderBy(k qbtypes.OrderBy, q qbtypes.QueryBuilderQuery[qbtypes.LogAggreg
return 0, false
}
// maybeAttachResourceFilter decides whether to pre-filter on the resource table.
//
// The resource table maps resource attributes (e.g. service.name) to fingerprints.
// When it helps, we look up the matching fingerprints there first and feed them to the
// main query as a CTE, so the main table only scans those fingerprints.
//
// There are three outcomes:
//
// 1. Skip it — the filter has nothing we can pre-resolve. This happens when the filter
// is empty,no resource filter, or when its resource conditions sit under an OR (e.g.
// `name='GET' OR service.name='abc'`), because then we can't reduce to a fixed set
// of fingerprints. The main query filters on resource attributes inline instead.
// 2. Skip it — too many fingerprints match (over the configured threshold), so the CTE
// would not be selective enough to be worth it. Again, filter inline on the main table.
// 3. Use it — attach the matching fingerprints as the __resource_filter CTE and join
// the main table on resource_fingerprint.
func (b *logQueryStatementBuilder) maybeAttachResourceFilter(
ctx context.Context,
sb *sqlbuilder.SelectBuilder,

View File

@@ -1230,7 +1230,7 @@ func TestSkipResourceFingerprintLogs(t *testing.T) {
mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
mock := mockStore.Mock()
mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_logs\.distributed_logs_v2_resource`).
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_logs\.distributed_logs_v2_resource`).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
{Name: "count", Type: "UInt64"},
}, []any{uint64(2)}))
@@ -1250,7 +1250,7 @@ func TestSkipResourceFingerprintLogs(t *testing.T) {
mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
mock := mockStore.Mock()
mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_logs\.distributed_logs_v2_resource`).
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_logs\.distributed_logs_v2_resource`).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
{Name: "count", Type: "UInt64"},
}, []any{threshold}))

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"log/slog"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -118,6 +119,16 @@ func (b *MetricQueryStatementBuilder) buildPipelineStatement(
cteArgs [][]any
)
if query.Aggregations[0].SpaceAggregation.IsPercentile() && !query.Aggregations[0].Type.IsPercentileSpaceAggregationAllowed() {
return nil, errors.Newf(
errors.TypeInvalidInput,
errors.CodeInvalidInput,
"invalid space aggregation `%s` for metric type `%s`, percentile space aggregations are only supported for [`histogram`, `exponentialhistogram`, `summary`] metric types",
query.Aggregations[0].SpaceAggregation.StringValue(),
query.Aggregations[0].Type.StringValue(),
)
}
origSpaceAgg := query.Aggregations[0].SpaceAggregation
origTimeAgg := query.Aggregations[0].TimeAggregation
origGroupBy := slices.Clone(query.GroupBy)

View File

@@ -5,6 +5,7 @@ import (
"testing"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
@@ -217,6 +218,25 @@ func TestStatementBuilder(t *testing.T) {
},
expectedErr: nil,
},
{
name: "test_percentile_space_aggregation_on_non_histogram_type",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "signoz_calls_total",
Type: metrictypes.SumType,
Temporality: metrictypes.Cumulative,
TimeAggregation: metrictypes.TimeAggregationRate,
SpaceAggregation: metrictypes.SpaceAggregationPercentile95,
},
},
Limit: 10,
},
expectedErr: errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid space aggregation `p95` for metric type `sum`, percentile space aggregations are only supported for [`histogram`, `exponentialhistogram`, `summary`] metric types"),
},
}
fm := NewFieldMapper()

View File

@@ -98,7 +98,17 @@ func (b *resourceFilterStatementBuilder[T]) Build(
query qbtypes.QueryBuilderQuery[T],
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
q, isNoOp, err := b.buildQuery(ctx, start, end, "fingerprint", query, variables)
q := sqlbuilder.NewSelectBuilder()
q.Select("fingerprint")
q.From(fmt.Sprintf("%s.%s", b.dbName, b.tableName))
keySelectors := b.getKeySelectors(query)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
isNoOp, err := b.addConditions(ctx, q, start, end, query, keys, variables)
if err != nil {
return nil, err
}
@@ -117,37 +127,8 @@ func (b *resourceFilterStatementBuilder[T]) Build(
}, nil
}
// buildQuery selects selectExpr from the resource table and applies the filter and
// time conditions. isNoOp is true when the filter resolves to no resource conditions.
func (b *resourceFilterStatementBuilder[T]) buildQuery(
ctx context.Context,
start, end uint64,
selectExpr string,
query qbtypes.QueryBuilderQuery[T],
variables map[string]qbtypes.VariableItem,
) (*sqlbuilder.SelectBuilder, bool, error) {
q := sqlbuilder.NewSelectBuilder()
q.Select(selectExpr)
q.From(fmt.Sprintf("%s.%s", b.dbName, b.tableName))
keySelectors := b.getKeySelectors(query)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, false, err
}
isNoOp, err := b.addConditions(ctx, q, start, end, query, keys, variables)
if err != nil {
return nil, false, err
}
return q, isNoOp, nil
}
// BuildCount returns a statement that counts the distinct fingerprints matching
// the resource filter. Returns (nil, nil) when the filter is a no-op.
//
// It uses uniq() rather than count() over a GROUP BY: uniq's approximation is well
// within tolerance for a threshold check and is ~2x faster with far less memory.
func (b *resourceFilterStatementBuilder[T]) BuildCount(
ctx context.Context,
start uint64,
@@ -155,18 +136,13 @@ func (b *resourceFilterStatementBuilder[T]) BuildCount(
query qbtypes.QueryBuilderQuery[T],
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
q, isNoOp, err := b.buildQuery(ctx, start, end, "uniq(fingerprint)", query, variables)
if err != nil {
inner, err := b.Build(ctx, start, end, qbtypes.RequestTypeRaw, query, variables)
if err != nil || inner == nil {
return nil, err
}
if isNoOp {
return nil, nil //nolint:nilnil
}
stmt, args := q.BuildWithFlavor(sqlbuilder.ClickHouse)
return &qbtypes.Statement{
Query: stmt,
Args: args,
Query: fmt.Sprintf("SELECT count() FROM (%s)", inner.Query),
Args: inner.Args,
}, nil
}

View File

@@ -818,22 +818,6 @@ func aggOrderBy(k qbtypes.OrderBy, q qbtypes.QueryBuilderQuery[qbtypes.TraceAggr
return 0, false
}
// maybeAttachResourceFilter decides whether to pre-filter on the resource table.
//
// The resource table maps resource attributes (e.g. service.name) to fingerprints.
// When it helps, we look up the matching fingerprints there first and feed them to the
// main query as a CTE, so the main table only scans those fingerprints.
//
// There are three outcomes:
//
// 1. Skip it — the filter has nothing we can pre-resolve. This happens when the filter
// is empty,no resource filter, or when its resource conditions sit under an OR (e.g.
// `name='GET' OR service.name='abc'`), because then we can't reduce to a fixed set
// of fingerprints. The main query filters on resource attributes inline instead.
// 2. Skip it — too many fingerprints match (over the configured threshold), so the CTE
// would not be selective enough to be worth it. Again, filter inline on the main table.
// 3. Use it — attach the matching fingerprints as the __resource_filter CTE and join
// the main table on resource_fingerprint.
func (b *traceQueryStatementBuilder) maybeAttachResourceFilter(
ctx context.Context,
sb *sqlbuilder.SelectBuilder,

View File

@@ -1629,7 +1629,7 @@ func TestSkipResourceFingerprint(t *testing.T) {
// Only the count query runs against the telemetry store; the CTE
// itself is embedded as SQL in the main query (no extra round trip).
mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_traces\.distributed_traces_v3_resource`).
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_traces\.distributed_traces_v3_resource`).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
{Name: "count", Type: "UInt64"},
}, []any{uint64(2)}))
@@ -1649,7 +1649,7 @@ func TestSkipResourceFingerprint(t *testing.T) {
mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
mock := mockStore.Mock()
mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_traces\.distributed_traces_v3_resource`).
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_traces\.distributed_traces_v3_resource`).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
{Name: "count", Type: "UInt64"},
}, []any{threshold}))

View File

@@ -640,6 +640,32 @@ def test_non_existent_metrics_returns_404(
assert get_error_message(response.json()) == "could not find the metric whatevergoennnsgoeshere"
def test_non_existent_internal_metrics_returns_no_warning(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
metric_name = "signoz_calls_total"
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = build_builder_query(
"A",
metric_name,
"doesnotreallymatter",
"sum",
)
end_ms = int(now.timestamp() * 1000)
start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
response = make_query_request(signoz, token, start_2h, end_ms, [query])
assert response.status_code == HTTPStatus.OK
data = response.json()
assert get_all_warnings(data) == []
# Verify /api/v1/fields/values filters label values by metricNamespace prefix.
# Inserts metrics under ns.a and ns.b, then asserts a specific prefix returns
# only matching values while a common prefix returns both.

View File

@@ -1,21 +1,11 @@
"""
Transparency check for the skip_resource_fingerprint optimization (traces and logs).
The optimization changes how a query's resource conditions are resolved depending on
how selective they are, but must change only ClickHouse performance, never the rows.
Each test runs the same query against the primary instance (optimization on,
threshold=3) and `signoz_fingerprint` (optimization off) and asserts the responses
are identical, covering all three resolver outcomes:
- CTE: a filter matching fewer fingerprints than the threshold resolves through the
fingerprint CTE.
- Fallback: a filter matching at or above the threshold pushes resource conditions
onto the main spans/logs table instead.
- No-op: a filter with no resource conditions to pre-resolve (no resource field, or
resource fields only under an OR) filters inline on the main table.
A fingerprint is the hash of the entire resource set, so two rows sharing the filtered
attribute but differing in any other resource attribute are distinct fingerprints.
At or above the configured fingerprint threshold the optimization pushes resource
conditions onto the main spans/logs table instead of the fingerprint CTE. That
rewrite must change ClickHouse performance, never the rows: each test runs the same
query against the primary instance (optimization on, threshold=2) and
`signoz_fingerprint` (optimization off) and asserts the responses are identical.
"""
from collections.abc import Callable
@@ -42,10 +32,10 @@ def test_skip_resource_fingerprint_traces_fallback_matches_fingerprint(
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""A filter matching >= threshold fingerprints drives the fallback path; rows must match the fingerprint baseline."""
"""A >= 2-fingerprint filter drives the fallback path; rows must match the fingerprint baseline."""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# 3 distinct services share one env (3 fingerprints >= threshold 3 -> fallback);
# 3 distinct services share one env (3 fingerprints > threshold 2 -> fallback);
# the 4th has a different env and must be excluded.
env = {"deployment.environment": "skip-fallback"}
insert_traces(
@@ -75,124 +65,6 @@ def test_skip_resource_fingerprint_traces_fallback_matches_fingerprint(
assert_identical_query_response(optimized, fingerprint)
def test_skip_resource_fingerprint_traces_cte_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""A filter matching < threshold fingerprints resolves through the fingerprint CTE; rows must match the baseline."""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# 2 distinct services share one env -> 2 fingerprints, below threshold 3 -> CTE.
# svc-a carries two spans to also exercise CTE dedup. A service in a different env
# is a third fingerprint but excluded by the filter.
env = {"deployment.environment": "skip-cte"}
insert_traces(
[
Traces(timestamp=now - timedelta(seconds=10), resources={"service.name": "skip-cte-svc-a", **env}),
Traces(timestamp=now - timedelta(seconds=9), resources={"service.name": "skip-cte-svc-a", **env}),
Traces(timestamp=now - timedelta(seconds=8), resources={"service.name": "skip-cte-svc-b", **env}),
Traces(timestamp=now - timedelta(seconds=7), resources={"service.name": "skip-cte-other", "deployment.environment": "skip-cte-other-env"}),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = BuilderQuery(
signal="traces",
limit=50,
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
filter_expression="deployment.environment = 'skip-cte'",
select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
).to_dict()
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
assert len(get_rows(optimized)) == 3
assert_identical_query_response(optimized, fingerprint)
def test_skip_resource_fingerprint_traces_or_filter_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""A resource condition under an OR has no fixed fingerprint set, so it filters inline; rows must match the baseline."""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# `name = ... OR service.name = ...` can't reduce to a fingerprint set (no-op path).
# span-1 matches on name, span-2 on service.name, span-3 matches neither.
env = {"deployment.environment": "skip-or"}
insert_traces(
[
Traces(timestamp=now - timedelta(seconds=10), name="tr-or-name", resources={"service.name": "tr-or-svc-x", **env}),
Traces(timestamp=now - timedelta(seconds=9), name="tr-or-other", resources={"service.name": "tr-or-svc-a", **env}),
Traces(timestamp=now - timedelta(seconds=8), name="tr-or-other", resources={"service.name": "tr-or-svc-b", **env}),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = BuilderQuery(
signal="traces",
limit=50,
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
filter_expression="name = 'tr-or-name' OR service.name = 'tr-or-svc-a'",
select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
).to_dict()
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
assert len(get_rows(optimized)) == 2
assert_identical_query_response(optimized, fingerprint)
def test_skip_resource_fingerprint_traces_no_resource_filter_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""A filter with no resource field has nothing to pre-resolve, so it filters inline; rows must match the baseline."""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# Filtering only on the span name (an intrinsic, non-resource field) is a no-op for
# the resolver; span-1 matches, span-2 does not.
env = {"deployment.environment": "skip-nr"}
insert_traces(
[
Traces(timestamp=now - timedelta(seconds=10), name="tr-nr-name", resources={"service.name": "tr-nr-svc-a", **env}),
Traces(timestamp=now - timedelta(seconds=9), name="tr-nr-other", resources={"service.name": "tr-nr-svc-b", **env}),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = BuilderQuery(
signal="traces",
limit=50,
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
filter_expression="name = 'tr-nr-name'",
select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
).to_dict()
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
assert len(get_rows(optimized)) == 1
assert_identical_query_response(optimized, fingerprint)
def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
@@ -200,10 +72,10 @@ def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""A filter matching >= threshold fingerprints drives the fallback path; rows must match the fingerprint baseline."""
"""A >= 2-fingerprint filter drives the fallback path; rows must match the fingerprint baseline."""
now = datetime.now(tz=UTC)
# 3 distinct services share one env (3 fingerprints >= threshold 3 -> fallback);
# 3 distinct services share one env (3 fingerprints > threshold 2 -> fallback);
# the 4th has a different env and must be excluded.
env = {"deployment.environment": "skip-logs-fallback"}
insert_logs(
@@ -231,121 +103,3 @@ def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
assert len(get_rows(optimized)) == 3
assert_identical_query_response(optimized, fingerprint)
def test_skip_resource_fingerprint_logs_cte_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""A filter matching < threshold fingerprints resolves through the fingerprint CTE; rows must match the baseline."""
now = datetime.now(tz=UTC)
# 2 distinct services share one env -> 2 fingerprints, below threshold 3 -> CTE.
# svc-a carries two logs to also exercise CTE dedup. A service in a different env
# is a third fingerprint but excluded by the filter.
env = {"deployment.environment": "skip-logs-cte"}
insert_logs(
[
Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "skip-logs-cte-svc-a", **env}, body="a"),
Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "skip-logs-cte-svc-a", **env}, body="b"),
Logs(timestamp=now - timedelta(seconds=8), resources={"service.name": "skip-logs-cte-svc-b", **env}, body="c"),
Logs(timestamp=now - timedelta(seconds=7), resources={"service.name": "skip-logs-cte-other", "deployment.environment": "skip-logs-cte-other-env"}, body="noise"),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = BuilderQuery(
signal="logs",
limit=50,
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
filter_expression="deployment.environment = 'skip-logs-cte'",
select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
).to_dict()
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
assert len(get_rows(optimized)) == 3
assert_identical_query_response(optimized, fingerprint)
def test_skip_resource_fingerprint_logs_or_filter_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""A resource condition under an OR has no fixed fingerprint set, so it filters inline; rows must match the baseline."""
now = datetime.now(tz=UTC)
# `test.marker = ... OR service.name = ...` can't reduce to a fingerprint set (no-op path).
# log-1 matches on the attribute, log-2 on service.name, log-3 matches neither.
env = {"deployment.environment": "skip-logs-or"}
insert_logs(
[
Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "logs-or-svc-x", **env}, attributes={"test.marker": "logs-or-hit"}, body="a"),
Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "logs-or-svc-a", **env}, body="b"),
Logs(timestamp=now - timedelta(seconds=8), resources={"service.name": "logs-or-svc-b", **env}, body="noise"),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = BuilderQuery(
signal="logs",
limit=50,
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
filter_expression="test.marker = 'logs-or-hit' OR service.name = 'logs-or-svc-a'",
select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
).to_dict()
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
assert len(get_rows(optimized)) == 2
assert_identical_query_response(optimized, fingerprint)
def test_skip_resource_fingerprint_logs_no_resource_filter_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""A filter with no resource field has nothing to pre-resolve, so it filters inline; rows must match the baseline."""
now = datetime.now(tz=UTC)
# Filtering only on an attribute (a non-resource field) is a no-op for the resolver;
# log-1 matches, log-2 does not.
env = {"deployment.environment": "skip-logs-nr"}
insert_logs(
[
Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "logs-nr-svc-a", **env}, attributes={"test.marker": "logs-nr-hit"}, body="a"),
Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "logs-nr-svc-b", **env}, body="b"),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = BuilderQuery(
signal="logs",
limit=50,
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
filter_expression="test.marker = 'logs-nr-hit'",
select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
).to_dict()
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
assert len(get_rows(optimized)) == 1
assert_identical_query_response(optimized, fingerprint)

View File

@@ -18,9 +18,8 @@ def signoz_skip_resource_fingerprint(
) -> types.SigNoz:
"""
Package-scoped SigNoz instance with the skip_resource_fingerprint
optimization enabled and a low threshold (3) so both resolver paths are
exercised: filters matching < 3 fingerprints resolve through the fingerprint
CTE, and filters matching >= 3 fingerprints fall back to the main table.
optimization enabled and a low threshold so the fallback resolver path is
exercised (filters matching >= 2 fingerprints skip the fingerprint CTE).
"""
return create_signoz(
network=network,
@@ -33,7 +32,7 @@ def signoz_skip_resource_fingerprint(
cache_key="signoz-skip-resource-fingerprint",
env_overrides={
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_ENABLED": True,
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_THRESHOLD": 3,
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_THRESHOLD": 2,
},
)