chore: improvements for skip fingerprint

pkg/querier/config.go

@@ -54,7 +54,7 @@ func (c Config) Validate() error {
 	if c.MaxConcurrentQueries <= 0 {
 		return errors.NewInvalidInputf(errors.CodeInvalidInput, "max_concurrent_queries must be positive, got %v", c.MaxConcurrentQueries)
 	}
-	if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
+	if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold <= 0 {
 		return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
 	}
 	return nil

pkg/telemetrylogs/statement_builder.go

@@ -716,6 +716,22 @@ func aggOrderBy(k qbtypes.OrderBy, q qbtypes.QueryBuilderQuery[qbtypes.LogAggreg
 	return 0, false
 }
 
+// maybeAttachResourceFilter decides whether to pre-filter on the resource table.
+//
+// The resource table maps resource attributes (e.g. service.name) to fingerprints.
+// When it helps, we look up the matching fingerprints there first and feed them to the
+// main query as a CTE, so the main table only scans those fingerprints.
+//
+// There are three outcomes:
+//
+//  1. Skip it — the filter has nothing we can pre-resolve. This happens when the filter
+//     is empty,no resource filter, or when its resource conditions sit under an OR (e.g.
+//     `name='GET' OR service.name='abc'`), because then we can't reduce to a fixed set
+//     of fingerprints. The main query filters on resource attributes inline instead.
+//  2. Skip it — too many fingerprints match (over the configured threshold), so the CTE
+//     would not be selective enough to be worth it. Again, filter inline on the main table.
+//  3. Use it — attach the matching fingerprints as the __resource_filter CTE and join
+//     the main table on resource_fingerprint.
 func (b *logQueryStatementBuilder) maybeAttachResourceFilter(
 	ctx context.Context,
 	sb *sqlbuilder.SelectBuilder,

pkg/telemetrylogs/stmt_builder_test.go

@@ -1230,7 +1230,7 @@ func TestSkipResourceFingerprintLogs(t *testing.T) {
 		mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
 		mock := mockStore.Mock()
 
-		mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_logs\.distributed_logs_v2_resource`).
+		mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_logs\.distributed_logs_v2_resource`).
 			WillReturnRow(cmock.NewRow([]cmock.ColumnType{
 				{Name: "count", Type: "UInt64"},
 			}, []any{uint64(2)}))
@@ -1250,7 +1250,7 @@ func TestSkipResourceFingerprintLogs(t *testing.T) {
 		mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
 		mock := mockStore.Mock()
 
-		mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_logs\.distributed_logs_v2_resource`).
+		mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_logs\.distributed_logs_v2_resource`).
 			WillReturnRow(cmock.NewRow([]cmock.ColumnType{
 				{Name: "count", Type: "UInt64"},
 			}, []any{threshold}))

pkg/telemetryresourcefilter/statement_builder.go

@@ -98,17 +98,7 @@ func (b *resourceFilterStatementBuilder[T]) Build(
 	query qbtypes.QueryBuilderQuery[T],
 	variables map[string]qbtypes.VariableItem,
 ) (*qbtypes.Statement, error) {
-	q := sqlbuilder.NewSelectBuilder()
-	q.Select("fingerprint")
-	q.From(fmt.Sprintf("%s.%s", b.dbName, b.tableName))
-
-	keySelectors := b.getKeySelectors(query)
-	keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
-	if err != nil {
-		return nil, err
-	}
-
-	isNoOp, err := b.addConditions(ctx, q, start, end, query, keys, variables)
+	q, isNoOp, err := b.buildQuery(ctx, start, end, "fingerprint", query, variables)
 	if err != nil {
 		return nil, err
 	}
@@ -127,8 +117,37 @@ func (b *resourceFilterStatementBuilder[T]) Build(
 	}, nil
 }
 
+// buildQuery selects selectExpr from the resource table and applies the filter and
+// time conditions. isNoOp is true when the filter resolves to no resource conditions.
+func (b *resourceFilterStatementBuilder[T]) buildQuery(
+	ctx context.Context,
+	start, end uint64,
+	selectExpr string,
+	query qbtypes.QueryBuilderQuery[T],
+	variables map[string]qbtypes.VariableItem,
+) (*sqlbuilder.SelectBuilder, bool, error) {
+	q := sqlbuilder.NewSelectBuilder()
+	q.Select(selectExpr)
+	q.From(fmt.Sprintf("%s.%s", b.dbName, b.tableName))
+
+	keySelectors := b.getKeySelectors(query)
+	keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
+	if err != nil {
+		return nil, false, err
+	}
+
+	isNoOp, err := b.addConditions(ctx, q, start, end, query, keys, variables)
+	if err != nil {
+		return nil, false, err
+	}
+	return q, isNoOp, nil
+}
+
 // BuildCount returns a statement that counts the distinct fingerprints matching
 // the resource filter. Returns (nil, nil) when the filter is a no-op.
+//
+// It uses uniq() rather than count() over a GROUP BY: uniq's approximation is well
+// within tolerance for a threshold check and is ~2x faster with far less memory.
 func (b *resourceFilterStatementBuilder[T]) BuildCount(
 	ctx context.Context,
 	start uint64,
@@ -136,13 +155,18 @@ func (b *resourceFilterStatementBuilder[T]) BuildCount(
 	query qbtypes.QueryBuilderQuery[T],
 	variables map[string]qbtypes.VariableItem,
 ) (*qbtypes.Statement, error) {
-	inner, err := b.Build(ctx, start, end, qbtypes.RequestTypeRaw, query, variables)
-	if err != nil || inner == nil {
+	q, isNoOp, err := b.buildQuery(ctx, start, end, "uniq(fingerprint)", query, variables)
+	if err != nil {
 		return nil, err
 	}
+	if isNoOp {
+		return nil, nil //nolint:nilnil
+	}
+
+	stmt, args := q.BuildWithFlavor(sqlbuilder.ClickHouse)
 	return &qbtypes.Statement{
-		Query: fmt.Sprintf("SELECT count() FROM (%s)", inner.Query),
-		Args:  inner.Args,
+		Query: stmt,
+		Args:  args,
 	}, nil
 }
 

pkg/telemetrytraces/statement_builder.go

@@ -818,6 +818,22 @@ func aggOrderBy(k qbtypes.OrderBy, q qbtypes.QueryBuilderQuery[qbtypes.TraceAggr
 	return 0, false
 }
 
+// maybeAttachResourceFilter decides whether to pre-filter on the resource table.
+//
+// The resource table maps resource attributes (e.g. service.name) to fingerprints.
+// When it helps, we look up the matching fingerprints there first and feed them to the
+// main query as a CTE, so the main table only scans those fingerprints.
+//
+// There are three outcomes:
+//
+//  1. Skip it — the filter has nothing we can pre-resolve. This happens when the filter
+//     is empty,no resource filter, or when its resource conditions sit under an OR (e.g.
+//     `name='GET' OR service.name='abc'`), because then we can't reduce to a fixed set
+//     of fingerprints. The main query filters on resource attributes inline instead.
+//  2. Skip it — too many fingerprints match (over the configured threshold), so the CTE
+//     would not be selective enough to be worth it. Again, filter inline on the main table.
+//  3. Use it — attach the matching fingerprints as the __resource_filter CTE and join
+//     the main table on resource_fingerprint.
 func (b *traceQueryStatementBuilder) maybeAttachResourceFilter(
 	ctx context.Context,
 	sb *sqlbuilder.SelectBuilder,

pkg/telemetrytraces/stmt_builder_test.go

@@ -1629,7 +1629,7 @@ func TestSkipResourceFingerprint(t *testing.T) {
 
 		// Only the count query runs against the telemetry store; the CTE
 		// itself is embedded as SQL in the main query (no extra round trip).
-		mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_traces\.distributed_traces_v3_resource`).
+		mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_traces\.distributed_traces_v3_resource`).
 			WillReturnRow(cmock.NewRow([]cmock.ColumnType{
 				{Name: "count", Type: "UInt64"},
 			}, []any{uint64(2)}))
@@ -1649,7 +1649,7 @@ func TestSkipResourceFingerprint(t *testing.T) {
 		mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
 		mock := mockStore.Mock()
 
-		mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_traces\.distributed_traces_v3_resource`).
+		mock.ExpectQueryRow(`SELECT uniq\(fingerprint\) FROM signoz_traces\.distributed_traces_v3_resource`).
 			WillReturnRow(cmock.NewRow([]cmock.ColumnType{
 				{Name: "count", Type: "UInt64"},
 			}, []any{threshold}))

tests/integration/tests/querier_skip_resource_fingerprint/01_skip_resource_fingerprint.py

@@ -1,11 +1,18 @@
 """
 Transparency check for the skip_resource_fingerprint optimization (traces and logs).
 
-At or above the configured fingerprint threshold the optimization pushes resource
-conditions onto the main spans/logs table instead of the fingerprint CTE. That
-rewrite must change ClickHouse performance, never the rows: each test runs the same
-query against the primary instance (optimization on, threshold=2) and
-`signoz_fingerprint` (optimization off) and asserts the responses are identical.
+The optimization changes how a query's resource conditions are resolved depending on
+how selective they are, but must change only ClickHouse performance, never the rows.
+Each test runs the same query against the primary instance (optimization on,
+threshold=2) and `signoz_fingerprint` (optimization off) and asserts the responses
+are identical, covering all three resolver outcomes:
+
+  - CTE: a filter matching fewer fingerprints than the threshold resolves through the
+    fingerprint CTE.
+  - Fallback: a filter matching at or above the threshold pushes resource conditions
+    onto the main spans/logs table instead.
+  - No-op: a filter with no resource conditions to pre-resolve (no resource field, or
+    resource fields only under an OR) filters inline on the main table.
 """
 
 from collections.abc import Callable
@@ -65,6 +72,122 @@ def test_skip_resource_fingerprint_traces_fallback_matches_fingerprint(
     assert_identical_query_response(optimized, fingerprint)
 
 
+def test_skip_resource_fingerprint_traces_cte_matches_fingerprint(
+    signoz: types.SigNoz,
+    signoz_fingerprint: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """A < 2-fingerprint filter resolves through the fingerprint CTE; rows must match the baseline."""
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+
+    # One service shares the env (1 fingerprint < threshold 2 -> CTE); two spans on it
+    # exercise dedup. A second service in a different env must be excluded.
+    env = {"deployment.environment": "skip-cte"}
+    insert_traces(
+        [
+            Traces(timestamp=now - timedelta(seconds=10), resources={"service.name": "skip-cte-svc-a", **env}),
+            Traces(timestamp=now - timedelta(seconds=9), resources={"service.name": "skip-cte-svc-a", **env}),
+            Traces(timestamp=now - timedelta(seconds=8), resources={"service.name": "skip-cte-other", "deployment.environment": "skip-cte-other-env"}),
+        ]
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    query = BuilderQuery(
+        signal="traces",
+        limit=50,
+        order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
+        filter_expression="deployment.environment = 'skip-cte'",
+        select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
+    ).to_dict()
+
+    start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
+    end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
+    optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+    fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+
+    assert len(get_rows(optimized)) == 2
+    assert_identical_query_response(optimized, fingerprint)
+
+
+def test_skip_resource_fingerprint_traces_or_filter_matches_fingerprint(
+    signoz: types.SigNoz,
+    signoz_fingerprint: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """A resource condition under an OR has no fixed fingerprint set, so it filters inline; rows must match the baseline."""
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+
+    # `name = ... OR service.name = ...` can't reduce to a fingerprint set (no-op path).
+    # span-1 matches on name, span-2 on service.name, span-3 matches neither.
+    env = {"deployment.environment": "skip-or"}
+    insert_traces(
+        [
+            Traces(timestamp=now - timedelta(seconds=10), name="tr-or-name", resources={"service.name": "tr-or-svc-x", **env}),
+            Traces(timestamp=now - timedelta(seconds=9), name="tr-or-other", resources={"service.name": "tr-or-svc-a", **env}),
+            Traces(timestamp=now - timedelta(seconds=8), name="tr-or-other", resources={"service.name": "tr-or-svc-b", **env}),
+        ]
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    query = BuilderQuery(
+        signal="traces",
+        limit=50,
+        order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
+        filter_expression="name = 'tr-or-name' OR service.name = 'tr-or-svc-a'",
+        select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
+    ).to_dict()
+
+    start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
+    end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
+    optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+    fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+
+    assert len(get_rows(optimized)) == 2
+    assert_identical_query_response(optimized, fingerprint)
+
+
+def test_skip_resource_fingerprint_traces_no_resource_filter_matches_fingerprint(
+    signoz: types.SigNoz,
+    signoz_fingerprint: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """A filter with no resource field has nothing to pre-resolve, so it filters inline; rows must match the baseline."""
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+
+    # Filtering only on the span name (an intrinsic, non-resource field) is a no-op for
+    # the resolver; span-1 matches, span-2 does not.
+    env = {"deployment.environment": "skip-nr"}
+    insert_traces(
+        [
+            Traces(timestamp=now - timedelta(seconds=10), name="tr-nr-name", resources={"service.name": "tr-nr-svc-a", **env}),
+            Traces(timestamp=now - timedelta(seconds=9), name="tr-nr-other", resources={"service.name": "tr-nr-svc-b", **env}),
+        ]
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    query = BuilderQuery(
+        signal="traces",
+        limit=50,
+        order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
+        filter_expression="name = 'tr-nr-name'",
+        select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
+    ).to_dict()
+
+    start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
+    end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
+    optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+    fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+
+    assert len(get_rows(optimized)) == 1
+    assert_identical_query_response(optimized, fingerprint)
+
+
 def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
     signoz: types.SigNoz,
     signoz_fingerprint: types.SigNoz,
@@ -103,3 +226,119 @@ def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
 
     assert len(get_rows(optimized)) == 3
     assert_identical_query_response(optimized, fingerprint)
+
+
+def test_skip_resource_fingerprint_logs_cte_matches_fingerprint(
+    signoz: types.SigNoz,
+    signoz_fingerprint: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_logs: Callable[[list[Logs]], None],
+) -> None:
+    """A < 2-fingerprint filter resolves through the fingerprint CTE; rows must match the baseline."""
+    now = datetime.now(tz=UTC)
+
+    # One service shares the env (1 fingerprint < threshold 2 -> CTE); two logs on it
+    # exercise dedup. A second service in a different env must be excluded.
+    env = {"deployment.environment": "skip-logs-cte"}
+    insert_logs(
+        [
+            Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "skip-logs-cte-svc-a", **env}, body="a"),
+            Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "skip-logs-cte-svc-a", **env}, body="b"),
+            Logs(timestamp=now - timedelta(seconds=8), resources={"service.name": "skip-logs-cte-other", "deployment.environment": "skip-logs-cte-other-env"}, body="noise"),
+        ]
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    query = BuilderQuery(
+        signal="logs",
+        limit=50,
+        order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
+        filter_expression="deployment.environment = 'skip-logs-cte'",
+        select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
+    ).to_dict()
+
+    start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
+    end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
+    optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+    fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+
+    assert len(get_rows(optimized)) == 2
+    assert_identical_query_response(optimized, fingerprint)
+
+
+def test_skip_resource_fingerprint_logs_or_filter_matches_fingerprint(
+    signoz: types.SigNoz,
+    signoz_fingerprint: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_logs: Callable[[list[Logs]], None],
+) -> None:
+    """A resource condition under an OR has no fixed fingerprint set, so it filters inline; rows must match the baseline."""
+    now = datetime.now(tz=UTC)
+
+    # `test.marker = ... OR service.name = ...` can't reduce to a fingerprint set (no-op path).
+    # log-1 matches on the attribute, log-2 on service.name, log-3 matches neither.
+    env = {"deployment.environment": "skip-logs-or"}
+    insert_logs(
+        [
+            Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "logs-or-svc-x", **env}, attributes={"test.marker": "logs-or-hit"}, body="a"),
+            Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "logs-or-svc-a", **env}, body="b"),
+            Logs(timestamp=now - timedelta(seconds=8), resources={"service.name": "logs-or-svc-b", **env}, body="noise"),
+        ]
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    query = BuilderQuery(
+        signal="logs",
+        limit=50,
+        order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
+        filter_expression="test.marker = 'logs-or-hit' OR service.name = 'logs-or-svc-a'",
+        select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
+    ).to_dict()
+
+    start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
+    end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
+    optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+    fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+
+    assert len(get_rows(optimized)) == 2
+    assert_identical_query_response(optimized, fingerprint)
+
+
+def test_skip_resource_fingerprint_logs_no_resource_filter_matches_fingerprint(
+    signoz: types.SigNoz,
+    signoz_fingerprint: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_logs: Callable[[list[Logs]], None],
+) -> None:
+    """A filter with no resource field has nothing to pre-resolve, so it filters inline; rows must match the baseline."""
+    now = datetime.now(tz=UTC)
+
+    # Filtering only on an attribute (a non-resource field) is a no-op for the resolver;
+    # log-1 matches, log-2 does not.
+    env = {"deployment.environment": "skip-logs-nr"}
+    insert_logs(
+        [
+            Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "logs-nr-svc-a", **env}, attributes={"test.marker": "logs-nr-hit"}, body="a"),
+            Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "logs-nr-svc-b", **env}, body="b"),
+        ]
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    query = BuilderQuery(
+        signal="logs",
+        limit=50,
+        order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
+        filter_expression="test.marker = 'logs-nr-hit'",
+        select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
+    ).to_dict()
+
+    start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
+    end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
+    optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+    fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
+
+    assert len(get_rows(optimized)) == 1
+    assert_identical_query_response(optimized, fingerprint)