mirror of
https://github.com/SigNoz/signoz.git
synced 2026-06-04 08:00:26 +01:00
Compare commits
5 Commits
issue_5131
...
nv/4189-3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cd359e676e | ||
|
|
a0b14e0835 | ||
|
|
3cf7642d2c | ||
|
|
7551064aac | ||
|
|
d00be5895d |
@@ -54,7 +54,7 @@ func (c Config) Validate() error {
|
||||
if c.MaxConcurrentQueries <= 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "max_concurrent_queries must be positive, got %v", c.MaxConcurrentQueries)
|
||||
}
|
||||
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold <= 0 {
|
||||
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -217,7 +217,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
|
||||
}
|
||||
}
|
||||
preseededResults := make(map[string]any)
|
||||
for _, name := range missingMetricQueries { // at this point missing metrics will not have any non existent metrics, only normal ones
|
||||
for _, name := range missingMetricQueries {
|
||||
switch req.RequestType {
|
||||
case qbtypes.RequestTypeTimeSeries:
|
||||
preseededResults[name] = &qbtypes.TimeSeriesData{QueryName: name}
|
||||
@@ -375,11 +375,24 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, queries []qbtypes.Q
|
||||
return missingMetricQueries, "", nil
|
||||
}
|
||||
|
||||
isInternalMetric := func(n string) bool { return strings.HasPrefix(n, "signoz.") || strings.HasPrefix(n, "signoz_") }
|
||||
externalMissingMetrics := make([]string, 0, len(missingMetrics))
|
||||
for _, m := range missingMetrics {
|
||||
if !isInternalMetric(m) {
|
||||
externalMissingMetrics = append(externalMissingMetrics, m)
|
||||
}
|
||||
}
|
||||
if len(externalMissingMetrics) == 0 {
|
||||
// this means all missing metrics are internal, and since internal metrics
|
||||
// aren't user-controlled, skip errors/warnings for them since users can't act on them
|
||||
return missingMetricQueries, "", nil
|
||||
}
|
||||
|
||||
// Classify each missing metric: never-seen → NotFound error; seen-but-no-
|
||||
// data-in-window → dormant warning.
|
||||
lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, missingMetrics...)
|
||||
lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, externalMissingMetrics...)
|
||||
nonExistentMetrics := []string{}
|
||||
for _, name := range missingMetrics {
|
||||
for _, name := range externalMissingMetrics {
|
||||
if ts, ok := lastSeenInfo[name]; ok && ts > 0 {
|
||||
continue
|
||||
}
|
||||
@@ -400,11 +413,11 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, queries []qbtypes.Q
|
||||
}
|
||||
return name
|
||||
}
|
||||
if len(missingMetrics) == 1 {
|
||||
if len(externalMissingMetrics) == 1 {
|
||||
dormantWarning = fmt.Sprintf("no data found for the metric %s in the query time range", lastSeenStr(missingMetrics[0]))
|
||||
} else {
|
||||
parts := make([]string, len(missingMetrics))
|
||||
for i, m := range missingMetrics {
|
||||
parts := make([]string, len(externalMissingMetrics))
|
||||
for i, m := range externalMissingMetrics {
|
||||
parts[i] = lastSeenStr(m)
|
||||
}
|
||||
dormantWarning = fmt.Sprintf("no data found for the following metrics in the query time range: %s", strings.Join(parts, ", "))
|
||||
|
||||
@@ -716,22 +716,6 @@ func aggOrderBy(k qbtypes.OrderBy, q qbtypes.QueryBuilderQuery[qbtypes.LogAggreg
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// maybeAttachResourceFilter decides whether to pre-filter on the resource table.
|
||||
//
|
||||
// The resource table maps resource attributes (e.g. service.name) to fingerprints.
|
||||
// When it helps, we look up the matching fingerprints there first and feed them to the
|
||||
// main query as a CTE, so the main table only scans those fingerprints.
|
||||
//
|
||||
// There are three outcomes:
|
||||
//
|
||||
// 1. Skip it — the filter has nothing we can pre-resolve. This happens when the filter
|
||||
// is empty,no resource filter, or when its resource conditions sit under an OR (e.g.
|
||||
// `name='GET' OR service.name='abc'`), because then we can't reduce to a fixed set
|
||||
// of fingerprints. The main query filters on resource attributes inline instead.
|
||||
// 2. Skip it — too many fingerprints match (over the configured threshold), so the CTE
|
||||
// would not be selective enough to be worth it. Again, filter inline on the main table.
|
||||
// 3. Use it — attach the matching fingerprints as the __resource_filter CTE and join
|
||||
// the main table on resource_fingerprint.
|
||||
func (b *logQueryStatementBuilder) maybeAttachResourceFilter(
|
||||
ctx context.Context,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
|
||||
@@ -1230,7 +1230,7 @@ func TestSkipResourceFingerprintLogs(t *testing.T) {
|
||||
mockStore := telemetrystoretest.New(telemetrystore.Config{}, ®exQueryMatcher{})
|
||||
mock := mockStore.Mock()
|
||||
|
||||
mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_logs\.distributed_logs_v2_resource`).
|
||||
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_logs\.distributed_logs_v2_resource`).
|
||||
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
|
||||
{Name: "count", Type: "UInt64"},
|
||||
}, []any{uint64(2)}))
|
||||
@@ -1250,7 +1250,7 @@ func TestSkipResourceFingerprintLogs(t *testing.T) {
|
||||
mockStore := telemetrystoretest.New(telemetrystore.Config{}, ®exQueryMatcher{})
|
||||
mock := mockStore.Mock()
|
||||
|
||||
mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_logs\.distributed_logs_v2_resource`).
|
||||
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_logs\.distributed_logs_v2_resource`).
|
||||
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
|
||||
{Name: "count", Type: "UInt64"},
|
||||
}, []any{threshold}))
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
@@ -118,6 +119,16 @@ func (b *MetricQueryStatementBuilder) buildPipelineStatement(
|
||||
cteArgs [][]any
|
||||
)
|
||||
|
||||
if query.Aggregations[0].SpaceAggregation.IsPercentile() && !query.Aggregations[0].Type.IsPercentileSpaceAggregationAllowed() {
|
||||
return nil, errors.Newf(
|
||||
errors.TypeInvalidInput,
|
||||
errors.CodeInvalidInput,
|
||||
"invalid space aggregation `%s` for metric type `%s`, percentile space aggregations are only supported for [`histogram`, `exponentialhistogram`, `summary`] metric types",
|
||||
query.Aggregations[0].SpaceAggregation.StringValue(),
|
||||
query.Aggregations[0].Type.StringValue(),
|
||||
)
|
||||
}
|
||||
|
||||
origSpaceAgg := query.Aggregations[0].SpaceAggregation
|
||||
origTimeAgg := query.Aggregations[0].TimeAggregation
|
||||
origGroupBy := slices.Clone(query.GroupBy)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
@@ -217,6 +218,25 @@ func TestStatementBuilder(t *testing.T) {
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
name: "test_percentile_space_aggregation_on_non_histogram_type",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
||||
Aggregations: []qbtypes.MetricAggregation{
|
||||
{
|
||||
MetricName: "signoz_calls_total",
|
||||
Type: metrictypes.SumType,
|
||||
Temporality: metrictypes.Cumulative,
|
||||
TimeAggregation: metrictypes.TimeAggregationRate,
|
||||
SpaceAggregation: metrictypes.SpaceAggregationPercentile95,
|
||||
},
|
||||
},
|
||||
Limit: 10,
|
||||
},
|
||||
expectedErr: errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid space aggregation `p95` for metric type `sum`, percentile space aggregations are only supported for [`histogram`, `exponentialhistogram`, `summary`] metric types"),
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
|
||||
@@ -98,7 +98,17 @@ func (b *resourceFilterStatementBuilder[T]) Build(
|
||||
query qbtypes.QueryBuilderQuery[T],
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (*qbtypes.Statement, error) {
|
||||
q, isNoOp, err := b.buildQuery(ctx, start, end, "fingerprint", query, variables)
|
||||
q := sqlbuilder.NewSelectBuilder()
|
||||
q.Select("fingerprint")
|
||||
q.From(fmt.Sprintf("%s.%s", b.dbName, b.tableName))
|
||||
|
||||
keySelectors := b.getKeySelectors(query)
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
isNoOp, err := b.addConditions(ctx, q, start, end, query, keys, variables)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -117,37 +127,8 @@ func (b *resourceFilterStatementBuilder[T]) Build(
|
||||
}, nil
|
||||
}
|
||||
|
||||
// buildQuery selects selectExpr from the resource table and applies the filter and
|
||||
// time conditions. isNoOp is true when the filter resolves to no resource conditions.
|
||||
func (b *resourceFilterStatementBuilder[T]) buildQuery(
|
||||
ctx context.Context,
|
||||
start, end uint64,
|
||||
selectExpr string,
|
||||
query qbtypes.QueryBuilderQuery[T],
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (*sqlbuilder.SelectBuilder, bool, error) {
|
||||
q := sqlbuilder.NewSelectBuilder()
|
||||
q.Select(selectExpr)
|
||||
q.From(fmt.Sprintf("%s.%s", b.dbName, b.tableName))
|
||||
|
||||
keySelectors := b.getKeySelectors(query)
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
isNoOp, err := b.addConditions(ctx, q, start, end, query, keys, variables)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return q, isNoOp, nil
|
||||
}
|
||||
|
||||
// BuildCount returns a statement that counts the distinct fingerprints matching
|
||||
// the resource filter. Returns (nil, nil) when the filter is a no-op.
|
||||
//
|
||||
// It uses uniq() rather than count() over a GROUP BY: uniq's approximation is well
|
||||
// within tolerance for a threshold check and is ~2x faster with far less memory.
|
||||
func (b *resourceFilterStatementBuilder[T]) BuildCount(
|
||||
ctx context.Context,
|
||||
start uint64,
|
||||
@@ -155,18 +136,13 @@ func (b *resourceFilterStatementBuilder[T]) BuildCount(
|
||||
query qbtypes.QueryBuilderQuery[T],
|
||||
variables map[string]qbtypes.VariableItem,
|
||||
) (*qbtypes.Statement, error) {
|
||||
q, isNoOp, err := b.buildQuery(ctx, start, end, "uniq(fingerprint)", query, variables)
|
||||
if err != nil {
|
||||
inner, err := b.Build(ctx, start, end, qbtypes.RequestTypeRaw, query, variables)
|
||||
if err != nil || inner == nil {
|
||||
return nil, err
|
||||
}
|
||||
if isNoOp {
|
||||
return nil, nil //nolint:nilnil
|
||||
}
|
||||
|
||||
stmt, args := q.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return &qbtypes.Statement{
|
||||
Query: stmt,
|
||||
Args: args,
|
||||
Query: fmt.Sprintf("SELECT count() FROM (%s)", inner.Query),
|
||||
Args: inner.Args,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -818,22 +818,6 @@ func aggOrderBy(k qbtypes.OrderBy, q qbtypes.QueryBuilderQuery[qbtypes.TraceAggr
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// maybeAttachResourceFilter decides whether to pre-filter on the resource table.
|
||||
//
|
||||
// The resource table maps resource attributes (e.g. service.name) to fingerprints.
|
||||
// When it helps, we look up the matching fingerprints there first and feed them to the
|
||||
// main query as a CTE, so the main table only scans those fingerprints.
|
||||
//
|
||||
// There are three outcomes:
|
||||
//
|
||||
// 1. Skip it — the filter has nothing we can pre-resolve. This happens when the filter
|
||||
// is empty,no resource filter, or when its resource conditions sit under an OR (e.g.
|
||||
// `name='GET' OR service.name='abc'`), because then we can't reduce to a fixed set
|
||||
// of fingerprints. The main query filters on resource attributes inline instead.
|
||||
// 2. Skip it — too many fingerprints match (over the configured threshold), so the CTE
|
||||
// would not be selective enough to be worth it. Again, filter inline on the main table.
|
||||
// 3. Use it — attach the matching fingerprints as the __resource_filter CTE and join
|
||||
// the main table on resource_fingerprint.
|
||||
func (b *traceQueryStatementBuilder) maybeAttachResourceFilter(
|
||||
ctx context.Context,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
|
||||
@@ -1629,7 +1629,7 @@ func TestSkipResourceFingerprint(t *testing.T) {
|
||||
|
||||
// Only the count query runs against the telemetry store; the CTE
|
||||
// itself is embedded as SQL in the main query (no extra round trip).
|
||||
mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_traces\.distributed_traces_v3_resource`).
|
||||
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_traces\.distributed_traces_v3_resource`).
|
||||
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
|
||||
{Name: "count", Type: "UInt64"},
|
||||
}, []any{uint64(2)}))
|
||||
@@ -1649,7 +1649,7 @@ func TestSkipResourceFingerprint(t *testing.T) {
|
||||
mockStore := telemetrystoretest.New(telemetrystore.Config{}, ®exQueryMatcher{})
|
||||
mock := mockStore.Mock()
|
||||
|
||||
mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_traces\.distributed_traces_v3_resource`).
|
||||
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_traces\.distributed_traces_v3_resource`).
|
||||
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
|
||||
{Name: "count", Type: "UInt64"},
|
||||
}, []any{threshold}))
|
||||
|
||||
@@ -640,6 +640,32 @@ def test_non_existent_metrics_returns_404(
|
||||
assert get_error_message(response.json()) == "could not find the metric whatevergoennnsgoeshere"
|
||||
|
||||
|
||||
def test_non_existent_internal_metrics_returns_no_warning(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
metric_name = "signoz_calls_total"
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = build_builder_query(
|
||||
"A",
|
||||
metric_name,
|
||||
"doesnotreallymatter",
|
||||
"sum",
|
||||
)
|
||||
|
||||
end_ms = int(now.timestamp() * 1000)
|
||||
|
||||
start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
|
||||
response = make_query_request(signoz, token, start_2h, end_ms, [query])
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
data = response.json()
|
||||
assert get_all_warnings(data) == []
|
||||
|
||||
|
||||
# Verify /api/v1/fields/values filters label values by metricNamespace prefix.
|
||||
# Inserts metrics under ns.a and ns.b, then asserts a specific prefix returns
|
||||
# only matching values while a common prefix returns both.
|
||||
|
||||
@@ -1,21 +1,11 @@
|
||||
"""
|
||||
Transparency check for the skip_resource_fingerprint optimization (traces and logs).
|
||||
|
||||
The optimization changes how a query's resource conditions are resolved depending on
|
||||
how selective they are, but must change only ClickHouse performance, never the rows.
|
||||
Each test runs the same query against the primary instance (optimization on,
|
||||
threshold=3) and `signoz_fingerprint` (optimization off) and asserts the responses
|
||||
are identical, covering all three resolver outcomes:
|
||||
|
||||
- CTE: a filter matching fewer fingerprints than the threshold resolves through the
|
||||
fingerprint CTE.
|
||||
- Fallback: a filter matching at or above the threshold pushes resource conditions
|
||||
onto the main spans/logs table instead.
|
||||
- No-op: a filter with no resource conditions to pre-resolve (no resource field, or
|
||||
resource fields only under an OR) filters inline on the main table.
|
||||
|
||||
A fingerprint is the hash of the entire resource set, so two rows sharing the filtered
|
||||
attribute but differing in any other resource attribute are distinct fingerprints.
|
||||
At or above the configured fingerprint threshold the optimization pushes resource
|
||||
conditions onto the main spans/logs table instead of the fingerprint CTE. That
|
||||
rewrite must change ClickHouse performance, never the rows: each test runs the same
|
||||
query against the primary instance (optimization on, threshold=2) and
|
||||
`signoz_fingerprint` (optimization off) and asserts the responses are identical.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
@@ -42,10 +32,10 @@ def test_skip_resource_fingerprint_traces_fallback_matches_fingerprint(
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""A filter matching >= threshold fingerprints drives the fallback path; rows must match the fingerprint baseline."""
|
||||
"""A >= 2-fingerprint filter drives the fallback path; rows must match the fingerprint baseline."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
# 3 distinct services share one env (3 fingerprints >= threshold 3 -> fallback);
|
||||
# 3 distinct services share one env (3 fingerprints > threshold 2 -> fallback);
|
||||
# the 4th has a different env and must be excluded.
|
||||
env = {"deployment.environment": "skip-fallback"}
|
||||
insert_traces(
|
||||
@@ -75,124 +65,6 @@ def test_skip_resource_fingerprint_traces_fallback_matches_fingerprint(
|
||||
assert_identical_query_response(optimized, fingerprint)
|
||||
|
||||
|
||||
def test_skip_resource_fingerprint_traces_cte_matches_fingerprint(
|
||||
signoz: types.SigNoz,
|
||||
signoz_fingerprint: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""A filter matching < threshold fingerprints resolves through the fingerprint CTE; rows must match the baseline."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
# 2 distinct services share one env -> 2 fingerprints, below threshold 3 -> CTE.
|
||||
# svc-a carries two spans to also exercise CTE dedup. A service in a different env
|
||||
# is a third fingerprint but excluded by the filter.
|
||||
env = {"deployment.environment": "skip-cte"}
|
||||
insert_traces(
|
||||
[
|
||||
Traces(timestamp=now - timedelta(seconds=10), resources={"service.name": "skip-cte-svc-a", **env}),
|
||||
Traces(timestamp=now - timedelta(seconds=9), resources={"service.name": "skip-cte-svc-a", **env}),
|
||||
Traces(timestamp=now - timedelta(seconds=8), resources={"service.name": "skip-cte-svc-b", **env}),
|
||||
Traces(timestamp=now - timedelta(seconds=7), resources={"service.name": "skip-cte-other", "deployment.environment": "skip-cte-other-env"}),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = BuilderQuery(
|
||||
signal="traces",
|
||||
limit=50,
|
||||
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
|
||||
filter_expression="deployment.environment = 'skip-cte'",
|
||||
select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
|
||||
).to_dict()
|
||||
|
||||
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
|
||||
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
|
||||
assert len(get_rows(optimized)) == 3
|
||||
assert_identical_query_response(optimized, fingerprint)
|
||||
|
||||
|
||||
def test_skip_resource_fingerprint_traces_or_filter_matches_fingerprint(
|
||||
signoz: types.SigNoz,
|
||||
signoz_fingerprint: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""A resource condition under an OR has no fixed fingerprint set, so it filters inline; rows must match the baseline."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
# `name = ... OR service.name = ...` can't reduce to a fingerprint set (no-op path).
|
||||
# span-1 matches on name, span-2 on service.name, span-3 matches neither.
|
||||
env = {"deployment.environment": "skip-or"}
|
||||
insert_traces(
|
||||
[
|
||||
Traces(timestamp=now - timedelta(seconds=10), name="tr-or-name", resources={"service.name": "tr-or-svc-x", **env}),
|
||||
Traces(timestamp=now - timedelta(seconds=9), name="tr-or-other", resources={"service.name": "tr-or-svc-a", **env}),
|
||||
Traces(timestamp=now - timedelta(seconds=8), name="tr-or-other", resources={"service.name": "tr-or-svc-b", **env}),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = BuilderQuery(
|
||||
signal="traces",
|
||||
limit=50,
|
||||
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
|
||||
filter_expression="name = 'tr-or-name' OR service.name = 'tr-or-svc-a'",
|
||||
select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
|
||||
).to_dict()
|
||||
|
||||
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
|
||||
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
|
||||
assert len(get_rows(optimized)) == 2
|
||||
assert_identical_query_response(optimized, fingerprint)
|
||||
|
||||
|
||||
def test_skip_resource_fingerprint_traces_no_resource_filter_matches_fingerprint(
|
||||
signoz: types.SigNoz,
|
||||
signoz_fingerprint: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""A filter with no resource field has nothing to pre-resolve, so it filters inline; rows must match the baseline."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
# Filtering only on the span name (an intrinsic, non-resource field) is a no-op for
|
||||
# the resolver; span-1 matches, span-2 does not.
|
||||
env = {"deployment.environment": "skip-nr"}
|
||||
insert_traces(
|
||||
[
|
||||
Traces(timestamp=now - timedelta(seconds=10), name="tr-nr-name", resources={"service.name": "tr-nr-svc-a", **env}),
|
||||
Traces(timestamp=now - timedelta(seconds=9), name="tr-nr-other", resources={"service.name": "tr-nr-svc-b", **env}),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = BuilderQuery(
|
||||
signal="traces",
|
||||
limit=50,
|
||||
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
|
||||
filter_expression="name = 'tr-nr-name'",
|
||||
select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
|
||||
).to_dict()
|
||||
|
||||
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
|
||||
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
|
||||
assert len(get_rows(optimized)) == 1
|
||||
assert_identical_query_response(optimized, fingerprint)
|
||||
|
||||
|
||||
def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
|
||||
signoz: types.SigNoz,
|
||||
signoz_fingerprint: types.SigNoz,
|
||||
@@ -200,10 +72,10 @@ def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""A filter matching >= threshold fingerprints drives the fallback path; rows must match the fingerprint baseline."""
|
||||
"""A >= 2-fingerprint filter drives the fallback path; rows must match the fingerprint baseline."""
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
# 3 distinct services share one env (3 fingerprints >= threshold 3 -> fallback);
|
||||
# 3 distinct services share one env (3 fingerprints > threshold 2 -> fallback);
|
||||
# the 4th has a different env and must be excluded.
|
||||
env = {"deployment.environment": "skip-logs-fallback"}
|
||||
insert_logs(
|
||||
@@ -231,121 +103,3 @@ def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
|
||||
|
||||
assert len(get_rows(optimized)) == 3
|
||||
assert_identical_query_response(optimized, fingerprint)
|
||||
|
||||
|
||||
def test_skip_resource_fingerprint_logs_cte_matches_fingerprint(
|
||||
signoz: types.SigNoz,
|
||||
signoz_fingerprint: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""A filter matching < threshold fingerprints resolves through the fingerprint CTE; rows must match the baseline."""
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
# 2 distinct services share one env -> 2 fingerprints, below threshold 3 -> CTE.
|
||||
# svc-a carries two logs to also exercise CTE dedup. A service in a different env
|
||||
# is a third fingerprint but excluded by the filter.
|
||||
env = {"deployment.environment": "skip-logs-cte"}
|
||||
insert_logs(
|
||||
[
|
||||
Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "skip-logs-cte-svc-a", **env}, body="a"),
|
||||
Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "skip-logs-cte-svc-a", **env}, body="b"),
|
||||
Logs(timestamp=now - timedelta(seconds=8), resources={"service.name": "skip-logs-cte-svc-b", **env}, body="c"),
|
||||
Logs(timestamp=now - timedelta(seconds=7), resources={"service.name": "skip-logs-cte-other", "deployment.environment": "skip-logs-cte-other-env"}, body="noise"),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = BuilderQuery(
|
||||
signal="logs",
|
||||
limit=50,
|
||||
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
|
||||
filter_expression="deployment.environment = 'skip-logs-cte'",
|
||||
select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
|
||||
).to_dict()
|
||||
|
||||
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
|
||||
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
|
||||
assert len(get_rows(optimized)) == 3
|
||||
assert_identical_query_response(optimized, fingerprint)
|
||||
|
||||
|
||||
def test_skip_resource_fingerprint_logs_or_filter_matches_fingerprint(
|
||||
signoz: types.SigNoz,
|
||||
signoz_fingerprint: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""A resource condition under an OR has no fixed fingerprint set, so it filters inline; rows must match the baseline."""
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
# `test.marker = ... OR service.name = ...` can't reduce to a fingerprint set (no-op path).
|
||||
# log-1 matches on the attribute, log-2 on service.name, log-3 matches neither.
|
||||
env = {"deployment.environment": "skip-logs-or"}
|
||||
insert_logs(
|
||||
[
|
||||
Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "logs-or-svc-x", **env}, attributes={"test.marker": "logs-or-hit"}, body="a"),
|
||||
Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "logs-or-svc-a", **env}, body="b"),
|
||||
Logs(timestamp=now - timedelta(seconds=8), resources={"service.name": "logs-or-svc-b", **env}, body="noise"),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = BuilderQuery(
|
||||
signal="logs",
|
||||
limit=50,
|
||||
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
|
||||
filter_expression="test.marker = 'logs-or-hit' OR service.name = 'logs-or-svc-a'",
|
||||
select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
|
||||
).to_dict()
|
||||
|
||||
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
|
||||
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
|
||||
assert len(get_rows(optimized)) == 2
|
||||
assert_identical_query_response(optimized, fingerprint)
|
||||
|
||||
|
||||
def test_skip_resource_fingerprint_logs_no_resource_filter_matches_fingerprint(
|
||||
signoz: types.SigNoz,
|
||||
signoz_fingerprint: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""A filter with no resource field has nothing to pre-resolve, so it filters inline; rows must match the baseline."""
|
||||
now = datetime.now(tz=UTC)
|
||||
|
||||
# Filtering only on an attribute (a non-resource field) is a no-op for the resolver;
|
||||
# log-1 matches, log-2 does not.
|
||||
env = {"deployment.environment": "skip-logs-nr"}
|
||||
insert_logs(
|
||||
[
|
||||
Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "logs-nr-svc-a", **env}, attributes={"test.marker": "logs-nr-hit"}, body="a"),
|
||||
Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "logs-nr-svc-b", **env}, body="b"),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
query = BuilderQuery(
|
||||
signal="logs",
|
||||
limit=50,
|
||||
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
|
||||
filter_expression="test.marker = 'logs-nr-hit'",
|
||||
select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
|
||||
).to_dict()
|
||||
|
||||
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
|
||||
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
|
||||
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
|
||||
|
||||
assert len(get_rows(optimized)) == 1
|
||||
assert_identical_query_response(optimized, fingerprint)
|
||||
|
||||
@@ -18,9 +18,8 @@ def signoz_skip_resource_fingerprint(
|
||||
) -> types.SigNoz:
|
||||
"""
|
||||
Package-scoped SigNoz instance with the skip_resource_fingerprint
|
||||
optimization enabled and a low threshold (3) so both resolver paths are
|
||||
exercised: filters matching < 3 fingerprints resolve through the fingerprint
|
||||
CTE, and filters matching >= 3 fingerprints fall back to the main table.
|
||||
optimization enabled and a low threshold so the fallback resolver path is
|
||||
exercised (filters matching >= 2 fingerprints skip the fingerprint CTE).
|
||||
"""
|
||||
return create_signoz(
|
||||
network=network,
|
||||
@@ -33,7 +32,7 @@ def signoz_skip_resource_fingerprint(
|
||||
cache_key="signoz-skip-resource-fingerprint",
|
||||
env_overrides={
|
||||
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_ENABLED": True,
|
||||
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_THRESHOLD": 3,
|
||||
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_THRESHOLD": 2,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user