chore: remove bucket cache change

* fix(alertmanager): resolve email SMTP settings from env

* refactor(alertmanager): remove dead legacy config and tidy integration test helpers

* test(alertmanager): scope smtp rotation test to env rotation only

* chore: drop redundant test

* chore: log and warn; do not persist global config

.github/workflows/integrationci.yaml

@@ -39,6 +39,8 @@ jobs:
       matrix:
         suite:
           - alerts
+          - alertmanager
+          - alertmanagerrotation
           - basepath
           - callbackauthn
           - cloudintegrations
@@ -53,7 +55,6 @@ jobs:
           - queriermetrics
           - querierscalar
           - queriercommon
-          - querierai
           - rawexportdata
           - promqlconformance
           - querierauthz

docs/api/openapi.yml

@@ -6902,7 +6902,6 @@ components:
     Querybuildertypesv5QueryEnvelope:
       discriminator:
         mapping:
-          builder_ai_query: '#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilderAI'
           builder_formula: '#/components/schemas/Querybuildertypesv5QueryEnvelopeFormula'
           builder_query: '#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilder'
           builder_trace_operator: '#/components/schemas/Querybuildertypesv5QueryEnvelopeTraceOperator'
@@ -6911,7 +6910,6 @@ components:
         propertyName: type
       oneOf:
       - $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilder'
-      - $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilderAI'
       - $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopeFormula'
       - $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopeTraceOperator'
       - $ref: '#/components/schemas/Querybuildertypesv5QueryEnvelopePromQL'
@@ -6926,15 +6924,6 @@ components:
       required:
       - type
       type: object
-    Querybuildertypesv5QueryEnvelopeBuilderAI:
-      properties:
-        spec:
-          $ref: '#/components/schemas/Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQuerybuildertypesQuerybuildertypesv5TraceAggregation'
-        type:
-          $ref: '#/components/schemas/Querybuildertypesv5QueryType'
-      required:
-      - type
-      type: object
     Querybuildertypesv5QueryEnvelopeClickHouseSQL:
       properties:
         spec:
@@ -7048,7 +7037,6 @@ components:
     Querybuildertypesv5QueryType:
       enum:
       - builder_query
-      - builder_ai_query
       - builder_formula
       - builder_trace_operator
       - clickhouse_sql

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -4301,18 +4301,6 @@ export interface Querybuildertypesv5QueryEnvelopeBuilderDTO {
 	type: Querybuildertypesv5QueryEnvelopeBuilderDTOType;
 }
 
-export enum Querybuildertypesv5QueryEnvelopeBuilderAIDTOType {
-	builder_ai_query = 'builder_ai_query',
-}
-export interface Querybuildertypesv5QueryEnvelopeBuilderAIDTO {
-	spec?: Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQuerybuildertypesQuerybuildertypesv5TraceAggregationDTO;
-	/**
-	 * @type string
-	 * @enum builder_ai_query
-	 */
-	type: Querybuildertypesv5QueryEnvelopeBuilderAIDTOType;
-}
-
 export interface Querybuildertypesv5QueryBuilderFormulaDTO {
 	/**
 	 * @type boolean
@@ -4496,7 +4484,6 @@ export interface Querybuildertypesv5QueryEnvelopeClickHouseSQLDTO {
 
 export type Querybuildertypesv5QueryEnvelopeDTO =
 	| Querybuildertypesv5QueryEnvelopeBuilderDTO
-	| Querybuildertypesv5QueryEnvelopeBuilderAIDTO
 	| Querybuildertypesv5QueryEnvelopeFormulaDTO
 	| Querybuildertypesv5QueryEnvelopeTraceOperatorDTO
 	| Querybuildertypesv5QueryEnvelopePromQLDTO
@@ -8300,7 +8287,6 @@ export interface Querybuildertypesv5QueryRangeResponseDTO {
 
 export enum Querybuildertypesv5QueryTypeDTO {
 	builder_query = 'builder_query',
-	builder_ai_query = 'builder_ai_query',
 	builder_formula = 'builder_formula',
 	builder_trace_operator = 'builder_trace_operator',
 	clickhouse_sql = 'clickhouse_sql',

pkg/alertmanager/alertmanagerserver/server.go

@@ -241,9 +241,12 @@ func (server *Server) PutAlerts(ctx context.Context, postableAlerts alertmanager
 }
 
 func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertmanagertypes.Config) error {
-	config := alertmanagerConfig.AlertmanagerConfig()
+	resolved, err := alertmanagerConfig.Resolved()
+	if err != nil {
+		return err
+	}
+	config := resolved.AlertmanagerConfig()
 
-	var err error
 	// Load SigNoz's alertmanager notification templates from the configured
 	// globs. The upstream default templates (default.tmpl, email.tmpl) are
 	// always loaded from the embedded alertmanager assets inside FromGlobs, so
@@ -275,7 +278,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 			server.logger.InfoContext(ctx, "skipping creation of receiver not referenced by any route", slog.String("receiver", rcv.Name))
 			continue
 		}
-		extendedRcv, err := alertmanagerConfig.GetReceiver(rcv.Name)
+		extendedRcv, err := resolved.GetReceiver(rcv.Name)
 		if err != nil {
 			return err
 		}
@@ -350,7 +353,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 	go server.dispatcher.Run()
 	go server.inhibitor.Run()
 
-	server.alertmanagerConfig = alertmanagerConfig
+	server.alertmanagerConfig = resolved
 	return nil
 }
 

pkg/alertmanager/config.go

@@ -1,7 +1,6 @@
 package alertmanager
 
 import (
-	"net/url"
 	"strings"
 	"time"
 
@@ -26,11 +25,6 @@ type Signoz struct {
 	alertmanagerserver.Config `mapstructure:",squash" yaml:",squash"`
 }
 
-type Legacy struct {
-	// ApiURL is the URL of the legacy signoz alertmanager.
-	ApiURL *url.URL `mapstructure:"api_url"`
-}
-
 func NewConfigFactory() factory.ConfigFactory {
 	return factory.NewConfigFactory(factory.MustNewName("alertmanager"), newConfig)
 }

pkg/alertmanager/signozalertmanager/provider.go

@@ -167,6 +167,10 @@ func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgI
 		return err
 	}
 
+	if err := config.SetGlobalConfig(provider.config.Signoz.Global); err != nil {
+		return err
+	}
+
 	if err := config.UpdateReceiver(receiver); err != nil {
 		return err
 	}
@@ -217,6 +221,10 @@ func (provider *provider) CreateChannel(ctx context.Context, orgID string, recei
 		return nil, err
 	}
 
+	if err := config.SetGlobalConfig(provider.config.Signoz.Global); err != nil {
+		return nil, err
+	}
+
 	if err := config.CreateReceiver(receiver); err != nil {
 		return nil, err
 	}

pkg/modules/llmpricingrule/impllmpricingrule/module.go

@@ -213,18 +213,18 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
 					Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
 						Name:   "A",
 						Signal: telemetrytypes.SignalTraces,
-						Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", telemetrytypes.GenAIRequestModel)},
+						Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", llmpricingruletypes.GenAIRequestModel)},
 						Aggregations: []qbtypes.TraceAggregation{
 							{Expression: "count()", Alias: "spanCount"},
 						},
 						GroupBy: []qbtypes.GroupByKey{
 							{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-								Name:          telemetrytypes.GenAIRequestModel,
+								Name:          llmpricingruletypes.GenAIRequestModel,
 								FieldContext:  telemetrytypes.FieldContextSpan,
 								FieldDataType: telemetrytypes.FieldDataTypeString,
 							}},
 							{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-								Name:          telemetrytypes.GenAIProviderName,
+								Name:          llmpricingruletypes.GenAIProviderName,
 								FieldContext:  telemetrytypes.FieldContextSpan,
 								FieldDataType: telemetrytypes.FieldDataTypeString,
 							}},
@@ -254,9 +254,9 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
 		switch c.Type {
 		case qbtypes.ColumnTypeGroup:
 			switch c.Name {
-			case telemetrytypes.GenAIRequestModel:
+			case llmpricingruletypes.GenAIRequestModel:
 				modelIdx = i
-			case telemetrytypes.GenAIProviderName:
+			case llmpricingruletypes.GenAIProviderName:
 				providerIdx = i
 			}
 		case qbtypes.ColumnTypeAggregation:

pkg/querier/api.go

@@ -250,7 +250,7 @@ func (handler *handler) ReplaceVariables(rw http.ResponseWriter, req *http.Reque
 	errs := []error{}
 
 	for idx, item := range queryRangeRequest.CompositeQuery.Queries {
-		if item.Type == qbtypes.QueryTypeBuilder || item.Type == qbtypes.QueryTypeBuilderAI {
+		if item.Type == qbtypes.QueryTypeBuilder {
 			switch spec := item.Spec.(type) {
 			case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
 				if spec.Filter != nil && spec.Filter.Expression != "" {

pkg/querier/bucket_cache_test.go

@@ -3,6 +3,7 @@ package querier
 import (
 	"context"
 	"fmt"
+	"math"
 	"testing"
 	"time"
 
@@ -1401,3 +1402,61 @@ func TestBucketCache_NoCache(t *testing.T) {
 	// The actual NoCache logic is implemented in querier.run(), not in bucket cache
 	// This test verifies that the cache works normally and NoCache bypasses it at a higher level
 }
+
+// A promql ratio yields NaN wherever the denominator is zero. If those do not
+// survive the cache, every good point in the same bucket is lost with them.
+func TestBucketCacheServesBucketsHoldingNonFiniteValues(t *testing.T) {
+	ctx := context.Background()
+	orgID := valuer.GenerateUUID()
+	bc := NewBucketCache(instrumentationtest.New().ToProviderSettings(), createTestCache(t), cacheTTL, defaultFluxInterval)
+
+	step := qbtypes.Step{Duration: 300 * time.Second}
+	stepMs := uint64(step.Milliseconds())
+	end := (uint64(time.Now().UnixMilli()) - uint64(20*time.Minute.Milliseconds())) / stepMs * stepMs
+	start := end - uint64(36*time.Hour.Milliseconds())
+
+	series := &qbtypes.TimeSeries{
+		Labels: []*qbtypes.Label{{
+			Key:   telemetrytypes.TelemetryFieldKey{Name: "job_name"},
+			Value: "dbBloatMonitorJob",
+		}},
+	}
+	finitePoints := 0
+	for ts := start; ts < end; ts += stepMs {
+		value := 11.524
+		if (ts/stepMs)%7 == 0 {
+			value = math.NaN()
+		} else {
+			finitePoints++
+		}
+		series.Values = append(series.Values, &qbtypes.TimeSeriesValue{Timestamp: int64(ts), Value: value})
+	}
+
+	q := &mockQuery{fingerprint: "promql&ratio&5m0s", startMs: start, endMs: end}
+	bc.Put(ctx, orgID, q, step, &qbtypes.Result{
+		Type: qbtypes.RequestTypeTimeSeries,
+		Value: &qbtypes.TimeSeriesData{
+			QueryName:    "A",
+			Aggregations: []*qbtypes.AggregationBucket{{Series: []*qbtypes.TimeSeries{series}}},
+		},
+	})
+
+	cached, missing := bc.GetMissRanges(ctx, orgID, q, step)
+	require.NotNil(t, cached)
+
+	servedFinite := 0
+	tsData, ok := cached.Value.(*qbtypes.TimeSeriesData)
+	require.True(t, ok)
+	for _, agg := range tsData.Aggregations {
+		for _, s := range agg.Series {
+			for _, v := range s.Values {
+				if !math.IsNaN(v.Value) {
+					servedFinite++
+				}
+			}
+		}
+	}
+
+	assert.Equal(t, finitePoints, servedFinite, "every finite point in the bucket is still served")
+	assert.Empty(t, missing, "and the covered span needs no re-query")
+}

pkg/querier/preview.go

@@ -249,7 +249,6 @@ func (q *querier) buildPreviewProviders(
 func rendersStandaloneStatement(t qbtypes.QueryType) bool {
 	switch t {
 	case qbtypes.QueryTypeBuilder,
-		qbtypes.QueryTypeBuilderAI,
 		qbtypes.QueryTypePromQL,
 		qbtypes.QueryTypeClickHouseSQL,
 		qbtypes.QueryTypeTraceOperator:

pkg/querier/querier.go

@@ -61,7 +61,6 @@ type querier struct {
 	// stay clean.
 	promV2                   prometheus.Prometheus
 	traceStmtBuilder         qbtypes.StatementBuilder[qbtypes.TraceAggregation]
-	aiTraceStmtBuilder       qbtypes.StatementBuilder[qbtypes.TraceAggregation]
 	logStmtBuilder           qbtypes.StatementBuilder[qbtypes.LogAggregation]
 	auditStmtBuilder         qbtypes.StatementBuilder[qbtypes.LogAggregation]
 	metricStmtBuilder        qbtypes.StatementBuilder[qbtypes.MetricAggregation]
@@ -90,7 +89,6 @@ func New(
 	promEngine prometheus.Prometheus,
 	promV2 prometheus.Prometheus,
 	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
-	aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
@@ -113,7 +111,6 @@ func New(
 		promEngine:               promEngine,
 		promV2:                   promV2,
 		traceStmtBuilder:         traceStmtBuilder,
-		aiTraceStmtBuilder:       aiTraceStmtBuilder,
 		logStmtBuilder:           logStmtBuilder,
 		auditStmtBuilder:         auditStmtBuilder,
 		metricStmtBuilder:        metricStmtBuilder,
@@ -298,16 +295,6 @@ func (q *querier) buildQueries(
 			}
 			queries[traceOpQuery.Name] = toq
 			steps[traceOpQuery.Name] = traceOpQuery.StepInterval
-		case qbtypes.QueryTypeBuilderAI:
-			spec, ok := query.Spec.(qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation])
-			if !ok {
-				return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid AI builder query spec %T", query.Spec)
-			}
-			spec.ShiftBy = extractShiftFromBuilderQuery(spec)
-			timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
-			bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
-			queries[spec.Name] = bq
-			steps[spec.Name] = spec.StepInterval
 		case qbtypes.QueryTypeBuilder:
 			switch spec := query.Spec.(type) {
 			case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
@@ -374,11 +361,6 @@ func (q *querier) populateQBEvent(event *qbtypes.QBEvent, queries []qbtypes.Quer
 			case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
 				event.MetricsUsed = true
 			}
-		case qbtypes.QueryTypeBuilderAI:
-			filter := query.GetFilter()
-			event.FilterApplied = event.FilterApplied || (filter != nil && filter.Expression != "")
-			event.GroupByApplied = event.GroupByApplied || len(query.GetGroupBy()) > 0
-			event.TracesUsed = true
 		case qbtypes.QueryTypePromQL:
 			event.MetricsUsed = true
 		case qbtypes.QueryTypeTraceOperator:
@@ -941,9 +923,7 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
 		specCopy := qt.spec.Copy()
 		specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
 		adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
-		// Reuse the statement builder the original query was created with, so an AI
-		// query keeps its AI builder without re-deriving it from the spec.
-		return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, qt.stmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
+		return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.traceStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
 
 	case *builderQuery[qbtypes.LogAggregation]:
 		specCopy := qt.spec.Copy()
@@ -1300,8 +1280,6 @@ func (q *querier) adjustStepInterval(queries []qbtypes.QueryEnvelope, start, end
 			if qe.GetStepInterval().Seconds() == 0 {
 				qe.SetStepInterval(secondsStep(metricRecommended))
 			}
-		case qbtypes.QueryTypeBuilderAI:
-			clampStep(qe, traceLogRecommended, traceLogMin, &warnings)
 		case qbtypes.QueryTypeTraceOperator:
 			clampStep(qe, traceLogRecommended, traceLogMin, &warnings)
 		}

pkg/querier/querier_test.go

@@ -50,7 +50,6 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
 		nil,                // prometheus
 		nil,                // promV2
 		nil,                // traceStmtBuilder
-		nil,                // aiTraceStmtBuilder
 		nil,                // logStmtBuilder
 		nil,                // auditStmtBuilder
 		nil,                // metricStmtBuilder
@@ -124,7 +123,6 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
 		nil, // prometheus
 		nil, // promV2
 		nil, // traceStmtBuilder
-		nil, // aiTraceStmtBuilder
 		nil, // logStmtBuilder
 		nil, // auditStmtBuilder
 		&mockMetricStmtBuilder{},

pkg/querier/signozquerier/provider.go

@@ -21,7 +21,6 @@ func NewFactory(
 	promV2 prometheus.Prometheus,
 	metadataStore telemetrytypes.MetadataStore,
 	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
-	aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
@@ -44,7 +43,6 @@ func NewFactory(
 				prometheus,
 				promV2,
 				traceStmtBuilder,
-				aiTraceStmtBuilder,
 				logStmtBuilder,
 				auditStmtBuilder,
 				metricStmtBuilder,

pkg/query-service/rules/managertestfactory.go

@@ -19,7 +19,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/sqlstore"
 	"github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest"
 	"github.com/SigNoz/signoz/pkg/statementbuilder"
-	"github.com/SigNoz/signoz/pkg/statementbuilder/aistatementbuilder"
 	"github.com/SigNoz/signoz/pkg/statementbuilder/auditstatementbuilder"
 	"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
 	"github.com/SigNoz/signoz/pkg/statementbuilder/meterstatementbuilder"
@@ -118,8 +117,6 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
 	ctx := context.Background()
 	traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
 	require.NoError(t, err)
-	aiTraceStmtBuilder, err := aistatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
-	require.NoError(t, err)
 	traceOperatorStmtBuilder, err := tracesstatementbuilder.NewOperatorFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
 	require.NoError(t, err)
 	logStmtBuilder, err := logsstatementbuilder.NewFactory(telemetryStore, metadataStore, flagger).New(ctx, providerSettings, cfg)
@@ -131,7 +128,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
 	meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
 	require.NoError(t, err)
 	bucketCache := querier.NewBucketCache(providerSettings, cache, 0, 0)
-	providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, nil, metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger)
+	providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, nil, metadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger)
 	mockQuerier, err := providerFactory.New(context.Background(), providerSettings, querier.Config{})
 	require.NoError(t, err)
 

pkg/query-service/rules/setups_test.go

@@ -42,7 +42,6 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
 		nil, // prometheus
 		nil, // promV2
 		nil, // traceStmtBuilder
-		nil, // aiTraceStmtBuilder
 		nil, // logStmtBuilder
 		nil, // auditStmtBuilder
 		metricStmtBuilder,
@@ -78,7 +77,6 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
 		nil, // prometheus
 		nil, // promV2
 		nil, // traceStmtBuilder
-		nil, // aiTraceStmtBuilder
 		logStmtBuilder,
 		nil, // auditStmtBuilder
 		nil, // metricStmtBuilder
@@ -115,7 +113,6 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
 		nil, // prometheus
 		nil, // promV2
 		traceStmtBuilder,
-		nil, // aiTraceStmtBuilder
 		nil, // logStmtBuilder
 		nil, // auditStmtBuilder
 		nil, // metricStmtBuilder

pkg/querybuilder/filter_split.go

@@ -1,176 +0,0 @@
-package querybuilder
-
-import (
-	"strings"
-
-	"github.com/SigNoz/signoz/pkg/errors"
-	grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-	"github.com/antlr4-go/antlr/v4"
-)
-
-// SplitFilterForAggregates partitions a single filter expression into a span-level
-// part (a WHERE over spans) and a trace-level part (a HAVING over per-trace
-// aggregates), splitting on the top-level AND.
-//
-// A key is trace-level when it carries the trace field context (`trace.completion_tokens`)
-// or, with no context, its bare name is in aggregateNames. Any other explicit context
-// (`span.`, `resource.`, …) is span-level. Trace-level and span-level keys may be
-// AND-combined (they run at different query stages) but not OR-combined; an OR that
-// mixes the two is an error.
-func SplitFilterForAggregates(query string, aggregateNames map[string]struct{}) (spanExpr string, havingExpr string, err error) {
-	if strings.TrimSpace(query) == "" {
-		return "", "", nil
-	}
-
-	tree, syntaxErrors := parseFilterQuery(query)
-	if len(syntaxErrors) > 0 {
-		combinedErrors := errors.Newf(
-			errors.TypeInvalidInput,
-			errors.CodeInvalidInput,
-			"Found %d syntax errors while parsing the filter expression.",
-			len(syntaxErrors),
-		)
-		additionals := make([]string, 0, len(syntaxErrors))
-		for _, syntaxError := range syntaxErrors {
-			if syntaxError.Error() != "" {
-				additionals = append(additionals, syntaxError.Error())
-			}
-		}
-		// TODO: add troubleshooting link to the filter query syntax guide once it's published.
-		return "", "", combinedErrors.WithAdditional(additionals...)
-	}
-
-	s := filterSplitter{query: []rune(query), aggregateNames: aggregateNames}
-	s.visit(tree)
-
-	if s.mixed {
-		return "", "", errors.NewInvalidInputf(errors.CodeInvalidInput,
-			"trace-level and span-level filters cannot be combined within an OR/NOT group; separate them with a top-level AND")
-	}
-	return strings.Join(s.span, " AND "), strings.Join(s.having, " AND "), nil
-}
-
-func parseFilterQuery(query string) (antlr.Tree, []*SyntaxErr) {
-	lexerErrorListener := NewErrorListener()
-	lexer := grammar.NewFilterQueryLexer(antlr.NewInputStream(query))
-	lexer.RemoveErrorListeners()
-	lexer.AddErrorListener(lexerErrorListener)
-
-	parserErrorListener := NewErrorListener()
-	parser := grammar.NewFilterQueryParser(antlr.NewCommonTokenStream(lexer, 0))
-	parser.RemoveErrorListeners()
-	parser.AddErrorListener(parserErrorListener)
-
-	tree := parser.Query()
-	return tree, append(lexerErrorListener.SyntaxErrors, parserErrorListener.SyntaxErrors...)
-}
-
-// filterSplitter walks the parse tree once, flattening the top-level AND chain and
-// routing each atom (a comparison, a NOT expression, or a whole multi-branch OR group)
-// to the span or having bucket by the class of the keys it references.
-type filterSplitter struct {
-	query          []rune
-	aggregateNames map[string]struct{}
-	span           []string
-	having         []string
-	mixed          bool
-}
-
-func (s *filterSplitter) visit(node antlr.Tree) {
-	switch n := node.(type) {
-	case *grammar.QueryContext:
-		if n.Expression() != nil {
-			s.visit(n.Expression())
-		}
-	case *grammar.ExpressionContext:
-		if n.OrExpression() != nil {
-			s.visit(n.OrExpression())
-		}
-	case *grammar.OrExpressionContext:
-		// a single branch is just an AND chain; multiple branches are a real OR, kept
-		// whole so a class-mixing OR can be rejected.
-		if ands := n.AllAndExpression(); len(ands) == 1 {
-			s.visit(ands[0])
-		} else {
-			s.route(n)
-		}
-	case *grammar.AndExpressionContext:
-		for _, u := range n.AllUnaryExpression() {
-			s.visit(u)
-		}
-	case *grammar.UnaryExpressionContext:
-		if n.NOT() != nil {
-			s.route(n)
-		} else if n.Primary() != nil {
-			s.visit(n.Primary())
-		}
-	case *grammar.PrimaryContext:
-		if n.OrExpression() != nil { // parenthesized sub-expression
-			s.visit(n.OrExpression())
-		} else {
-			s.route(n)
-		}
-	}
-}
-
-// route classifies an atom and appends its original source text to the right bucket.
-func (s *filterSplitter) route(atom antlr.ParserRuleContext) {
-	isTrace, isSpan := classifyKeys(atom, s.aggregateNames)
-	if isTrace && isSpan {
-		s.mixed = true
-		return
-	}
-	text := atomSourceText(s.query, atom)
-	// A multi-branch OR group's source slice excludes its enclosing parens (they belong
-	// to the parent Primary). Re-wrap it so rejoining a bucket with " AND " cannot invert
-	// OR/AND precedence, e.g. `a AND (b OR c)` must not flatten to `a AND b OR c`.
-	if or, ok := atom.(*grammar.OrExpressionContext); ok && len(or.AllAndExpression()) > 1 {
-		text = "(" + text + ")"
-	}
-	if isTrace {
-		s.having = append(s.having, text)
-	} else {
-		s.span = append(s.span, text)
-	}
-}
-
-// classifyKeys reports whether a subtree references trace-level and/or span-level keys.
-// A key is trace-level when it carries the trace field context or, with no context,
-// its name is a known aggregate; an unknown name under the trace context stays
-// trace-level so the aggregate validation rejects it with a targeted error. Any other
-// explicit context (`span.`, `resource.`, …) is span-level.
-func classifyKeys(node antlr.Tree, aggregateNames map[string]struct{}) (isTrace, isSpan bool) {
-	kc, ok := node.(*grammar.KeyContext)
-	if ok {
-		key := telemetrytypes.GetFieldKeyFromKeyText(kc.GetText())
-		switch key.FieldContext {
-		case telemetrytypes.FieldContextTrace:
-			isTrace = true
-		case telemetrytypes.FieldContextUnspecified:
-			_, isTrace = aggregateNames[key.Name]
-			isSpan = !isTrace
-		default:
-			isSpan = true
-		}
-		return
-	}
-	for i := 0; i < node.GetChildCount(); i++ {
-		t, s := classifyKeys(node.GetChild(i), aggregateNames)
-		isTrace = isTrace || t
-		isSpan = isSpan || s
-	}
-	return
-}
-
-// atomSourceText returns the original source substring for an atom, preserving
-// whitespace. The token stream drops skipped whitespace, which would glue word
-// operators (OR/AND/NOT) to their operands, so slice the input by token offsets.
-// ANTLR offsets are rune indices (InputStream holds []rune), hence the rune slice.
-func atomSourceText(query []rune, atom antlr.ParserRuleContext) string {
-	start, stop := atom.GetStart(), atom.GetStop()
-	if start == nil || stop == nil || start.GetStart() < 0 || stop.GetStop() >= len(query) || stop.GetStop() < start.GetStart() {
-		return atom.GetText()
-	}
-	return string(query[start.GetStart() : stop.GetStop()+1])
-}

pkg/querybuilder/filter_split_test.go

@@ -1,223 +0,0 @@
-package querybuilder
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSplitFilterForAggregates(t *testing.T) {
-	agg := map[string]struct{}{"completion_tokens": {}, "span_count": {}, "prompt_tokens": {}}
-
-	type tc struct {
-		name    string
-		query   string
-		span    string // expected span-level (WHERE) part; "" => empty
-		having  string // expected trace-level (HAVING) part; "" => empty
-		wantErr bool
-	}
-
-	cases := []tc{
-		// --- empty input ---------------------------------------------------------
-		{
-			name: "empty",
-		},
-		{
-			name:  "whitespace only",
-			query: "   ",
-		},
-
-		// --- single class --------------------------------------------------------
-		{
-			name:  "span only",
-			query: "service.name = 'x'",
-			span:  "service.name = 'x'",
-		},
-		{
-			name:   "agg only bare",
-			query:  "completion_tokens > 1000",
-			having: "completion_tokens > 1000",
-		},
-		{
-			// the user-facing `trace.` prefix marks a trace-level aggregate.
-			name:   "agg only trace prefix",
-			query:  "trace.completion_tokens > 1000",
-			having: "trace.completion_tokens > 1000",
-		},
-		{
-			// an unknown name under the trace context still routes trace-level, so the
-			// aggregate validation rejects it with a targeted error instead of the span
-			// path failing on an unknown field.
-			name:   "unknown aggregate under trace context stays trace-level",
-			query:  "trace.not_an_aggregate > 1000",
-			having: "trace.not_an_aggregate > 1000",
-		},
-
-		{
-			// ANTLR token offsets are rune indices; slicing must not shift after a
-			// multi-byte char (this used to truncate 1000 → 100).
-			name:   "unicode value before the split",
-			query:  "service.name = 'héllo' AND completion_tokens > 1000",
-			span:   "service.name = 'héllo'",
-			having: "completion_tokens > 1000",
-		},
-
-		// --- top-level AND splits across the two buckets -------------------------
-		{
-			name:   "span AND agg",
-			query:  "service.name = 'x' AND completion_tokens > 1000",
-			span:   "service.name = 'x'",
-			having: "completion_tokens > 1000",
-		},
-		{
-			// order within a bucket is preserved; the two span atoms join with AND.
-			name:   "span AND span AND agg",
-			query:  "service.name = 'x' AND kind_string = 'Internal' AND completion_tokens > 1000",
-			span:   "service.name = 'x' AND kind_string = 'Internal'",
-			having: "completion_tokens > 1000",
-		},
-		{
-			// a parenthesized top-level AND still splits across the two buckets.
-			name:   "parenthesized span AND agg",
-			query:  "(service.name = 'x' AND completion_tokens > 1000)",
-			span:   "service.name = 'x'",
-			having: "completion_tokens > 1000",
-		},
-
-		// --- OR groups are re-wrapped in parens so a later AND-join can't invert
-		//     precedence (`a AND (b OR c)` must not flatten to `a AND b OR c`) ------
-		{
-			name:   "agg OR agg",
-			query:  "completion_tokens > 1000 OR span_count > 3",
-			having: "(completion_tokens > 1000 OR span_count > 3)",
-		},
-		{
-			name:  "span OR span",
-			query: "service.name = 'x' OR kind_string = 'Internal'",
-			span:  "(service.name = 'x' OR kind_string = 'Internal')",
-		},
-		{
-			name:  "span AND (span OR span)",
-			query: "service.name = 'x' AND (kind_string = 'Internal' OR kind_string = 'Client')",
-			span:  "service.name = 'x' AND (kind_string = 'Internal' OR kind_string = 'Client')",
-		},
-		{
-			name:   "agg AND (agg OR agg)",
-			query:  "prompt_tokens > 5 AND (completion_tokens > 1000 OR span_count > 3)",
-			having: "prompt_tokens > 5 AND (completion_tokens > 1000 OR span_count > 3)",
-		},
-		{
-			// the OR group routes to span, the trailing aggregate to having.
-			name:   "span AND (span OR span) AND agg",
-			query:  "a.b = 'x' AND (c.d = 'y' OR e.f = 'z') AND completion_tokens > 1000",
-			span:   "a.b = 'x' AND (c.d = 'y' OR e.f = 'z')",
-			having: "completion_tokens > 1000",
-		},
-
-		// --- a nested AND group flattens across the buckets (no spurious parens) --
-		{
-			name:   "(span AND agg) AND agg",
-			query:  "(service.name = 'x' AND completion_tokens > 1000) AND prompt_tokens > 5",
-			span:   "service.name = 'x'",
-			having: "completion_tokens > 1000 AND prompt_tokens > 5",
-		},
-
-		// --- NOT wrapping a single-class group is routed whole to that class ------
-		{
-			name:   "not agg",
-			query:  "NOT (completion_tokens > 1000)",
-			having: "NOT (completion_tokens > 1000)",
-		},
-		{
-			name:  "not span",
-			query: "NOT (service.name = 'x')",
-			span:  "NOT (service.name = 'x')",
-		},
-
-		// --- an explicit non-trace context escapes the aggregate-alias shadow -----
-		{
-			// a span attribute named like an aggregate stays reachable via `attribute.`.
-			name:  "attribute prefix on aggregate name routes span-level",
-			query: "attribute.completion_tokens > 5",
-			span:  "attribute.completion_tokens > 5",
-		},
-		{
-			name:  "span prefix on aggregate name routes span-level",
-			query: "span.completion_tokens > 5",
-			span:  "span.completion_tokens > 5",
-		},
-		{
-			name:   "prefixed attribute AND bare aggregate split across buckets",
-			query:  "attribute.completion_tokens > 5 AND completion_tokens > 1000",
-			span:   "attribute.completion_tokens > 5",
-			having: "completion_tokens > 1000",
-		},
-
-		// --- class-mixing is rejected in an OR group, a NOT group, or a nested OR -
-		{
-			name:    "agg OR span rejected",
-			query:   "completion_tokens > 1000 OR service.name = 'x'",
-			wantErr: true,
-		},
-		{
-			name:    "not mixed rejected",
-			query:   "NOT (completion_tokens > 1000 AND service.name = 'x')",
-			wantErr: true,
-		},
-		{
-			name:    "span AND (agg OR span) rejected",
-			query:   "service.name = 'x' AND (completion_tokens > 1000 OR kind_string = 'Client')",
-			wantErr: true,
-		},
-
-		// --- syntax errors are rejected, not silently dropped by error recovery ---
-		{
-			// recovery would yield an empty tree → both buckets empty → filter ignored.
-			name:    "lone paren rejected",
-			query:   ")",
-			wantErr: true,
-		},
-		{
-			name:    "unbalanced parens rejected",
-			query:   "((",
-			wantErr: true,
-		},
-		{
-			name:    "bare operator rejected",
-			query:   "AND",
-			wantErr: true,
-		},
-		{
-			// lexer-level error: recovery drops the whole expression.
-			name:    "unterminated quote rejected",
-			query:   "'unterminated",
-			wantErr: true,
-		},
-		{
-			// recovery would drop only the malformed atom and keep the rest — a
-			// partially applied filter with no error.
-			name:    "garbage atom alongside valid agg rejected",
-			query:   ") AND completion_tokens > 5",
-			wantErr: true,
-		},
-		{
-			name:    "missing value rejected",
-			query:   "completion_tokens >",
-			wantErr: true,
-		},
-	}
-
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			span, having, err := SplitFilterForAggregates(c.query, agg)
-			if c.wantErr {
-				require.Error(t, err)
-				return
-			}
-			require.NoError(t, err)
-			assert.Equal(t, c.span, span, "span part")
-			assert.Equal(t, c.having, having, "having part")
-		})
-	}
-}

pkg/querybuilder/having_rewriter.go

@@ -18,19 +18,6 @@ func NewHavingExpressionRewriter() *HavingExpressionRewriter {
 	}
 }
 
-// Rewrite rewrites and validates a HAVING expression against a caller-supplied
-// column map (user-facing name -> SQL identifier/expression). Values are inlined, so
-// the result is a bare SQL boolean expression with no bound args. Used by callers
-// that project their own aggregate columns (e.g. the AI trace list) rather than the
-// query's Aggregations.
-func (r *HavingExpressionRewriter) Rewrite(expression string, columnMap map[string]string) (string, error) {
-	if len(strings.TrimSpace(expression)) == 0 {
-		return "", nil
-	}
-	r.columnMap = columnMap
-	return r.rewriteAndValidate(expression)
-}
-
 // RewriteForTraces rewrites and validates the HAVING expression for a traces query.
 func (r *HavingExpressionRewriter) RewriteForTraces(expression string, aggregations []qbtypes.TraceAggregation) (string, error) {
 	if len(strings.TrimSpace(expression)) == 0 {

pkg/querybuilder/query_range_resources.go

@@ -82,10 +82,6 @@ func resourcesForQuery(query gjson.Result, variables map[string]qbtypes.Variable
 	switch queryType {
 	case qbtypes.QueryTypeBuilder.StringValue(), qbtypes.QueryTypeSubQuery.StringValue():
 		return resourcesForBuilderQuery(queryType, query.Get("spec"), variables)
-	case qbtypes.QueryTypeBuilderAI.StringValue():
-		// An AI builder query is always a traces query; the signal is implied by the
-		// type (and may be absent from the payload), so pin the resource directly.
-		return builderQueryResourceRefs(queryType, coretypes.ResourceTelemetryResourceTraces, query.Get("spec"), variables)
 	case qbtypes.QueryTypePromQL.StringValue():
 		return []coretypes.ResourceWithID{{Resource: coretypes.ResourceTelemetryResourceMetrics, ID: typeWildcard}}, nil
 	case qbtypes.QueryTypeClickHouseSQL.StringValue():
@@ -107,10 +103,7 @@ func resourcesForBuilderQuery(queryType string, spec gjson.Result, variables map
 	if err != nil {
 		return nil, err
 	}
-	return builderQueryResourceRefs(queryType, resource, spec, variables)
-}
 
-func builderQueryResourceRefs(queryType string, resource coretypes.Resource, spec gjson.Result, variables map[string]qbtypes.VariableItem) ([]coretypes.ResourceWithID, error) {
 	ids, err := builderQuerySelectors(queryType, spec.Get("filter.expression").String(), variables)
 	if err != nil {
 		return nil, err

pkg/querybuilder/query_range_resources_test.go

@@ -92,20 +92,6 @@ func TestQueryRangeResources(t *testing.T) {
 				{Resource: coretypes.ResourceTelemetryResourceAuditLogs, ID: "builder_query/signoz.workspace.key.id/a"},
 			},
 		},
-		{
-			name: "ai builder query maps to traces resource without a signal",
-			body: `{"compositeQuery":{"queries":[{"type":"builder_ai_query","spec":{"filter":{"expression":"signoz.workspace.key.id = 'checkout'"}}}]}}`,
-			expected: []coretypes.ResourceWithID{
-				{Resource: coretypes.ResourceTelemetryResourceTraces, ID: "builder_ai_query/signoz.workspace.key.id/checkout"},
-			},
-		},
-		{
-			name: "ai builder query without filter is wildcard",
-			body: `{"compositeQuery":{"queries":[{"type":"builder_ai_query","spec":{}}]}}`,
-			expected: []coretypes.ResourceWithID{
-				{Resource: coretypes.ResourceTelemetryResourceTraces, ID: "builder_ai_query/*"},
-			},
-		},
 		{
 			name: "promql is wildcard only",
 			body: `{"compositeQuery":{"queries":[{"type":"promql","spec":{"query":"up"}}]}}`,

pkg/querybuilder/where_clause_visitor.go

@@ -83,10 +83,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
 }
 
 type PreparedWhereClause struct {
-	WhereClause *sqlbuilder.WhereClause
-	// Expr is the bare predicate in builder-internal format ($n markers bound to
-	// opts.Builder), embeddable outside a WHERE clause (e.g. inside countIf).
-	Expr              string
+	WhereClause       *sqlbuilder.WhereClause
 	Warnings          []string
 	WarningsDocURL    string
 	RequiresCostGuard bool
@@ -176,7 +173,7 @@ func PrepareWhereClause(query string, opts FilterExprVisitorOpts) (PreparedWhere
 
 	whereClause := sqlbuilder.NewWhereClause().AddWhereExpr(visitor.builder.Args, cond)
 
-	return PreparedWhereClause{WhereClause: whereClause, Expr: cond, Warnings: visitor.warnings, WarningsDocURL: visitor.mainWarnURL, RequiresCostGuard: visitor.requiresCostGuard}, nil
+	return PreparedWhereClause{WhereClause: whereClause, Warnings: visitor.warnings, WarningsDocURL: visitor.mainWarnURL, RequiresCostGuard: visitor.requiresCostGuard}, nil
 }
 
 // Visit dispatches to the specific visit method based on node type.

pkg/signoz/provider.go

@@ -233,6 +233,7 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewFillDashboardMeterSourceFactory(sqlstore, dashboardStore),
 		sqlmigration.NewUpdateRoleTransactionGroupsFactory(),
 		sqlmigration.NewFillDashboardSpecCollectionsFactory(sqlstore, dashboardStore),
+		sqlmigration.NewScrubEmailChannelTransportFactory(sqlstore),
 	)
 }
 
@@ -292,9 +293,9 @@ func NewStatsReporterProviderFactories(aggregator statsreporter.Aggregator, orgG
 	)
 }
 
-func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, promV2 prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
+func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, promV2 prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
 	return factory.MustNewNamedMap(
-		signozquerier.NewFactory(telemetryStore, prometheus, promV2, metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
+		signozquerier.NewFactory(telemetryStore, prometheus, promV2, metadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
 	)
 }
 

pkg/signoz/signoz.go

@@ -49,7 +49,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/sqlmigrator"
 	"github.com/SigNoz/signoz/pkg/sqlschema"
 	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/statementbuilder/aistatementbuilder"
 	"github.com/SigNoz/signoz/pkg/statementbuilder/auditstatementbuilder"
 	"github.com/SigNoz/signoz/pkg/statementbuilder/logsstatementbuilder"
 	"github.com/SigNoz/signoz/pkg/statementbuilder/meterstatementbuilder"
@@ -101,9 +100,9 @@ type SigNoz struct {
 
 // newQueryStack assembles the query stack once and returns, in order: the shared
 // telemetry metadata store (reused elsewhere in signoz.New), the per-signal
-// statement builders (trace, ai-trace, log, audit, metric, meter, trace-operator),
-// and the bucket cache. It is the only place that imports the concrete
-// statement-builder sub-packages.
+// statement builders (trace, log, audit, metric, meter, trace-operator), and the
+// bucket cache. It is the only place that imports the concrete statement-builder
+// sub-packages.
 func newQueryStack(
 	ctx context.Context,
 	settings factory.ProviderSettings,
@@ -114,7 +113,6 @@ func newQueryStack(
 ) (
 	telemetrytypes.MetadataStore,
 	qbtypes.StatementBuilder[qbtypes.TraceAggregation],
-	qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	qbtypes.StatementBuilder[qbtypes.MetricAggregation],
@@ -128,36 +126,32 @@ func newQueryStack(
 	cfg := config.Querier.Config
 	traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
 	if err != nil {
-		return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
-	}
-	aiTraceStmtBuilder, err := aistatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
-	if err != nil {
-		return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, nil, nil, err
 	}
 	traceOperatorStmtBuilder, err := tracesstatementbuilder.NewOperatorFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
 	if err != nil {
-		return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, nil, nil, err
 	}
 	logStmtBuilder, err := logsstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
 	if err != nil {
-		return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, nil, nil, err
 	}
 	auditStmtBuilder, err := auditstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
 	if err != nil {
-		return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, nil, nil, err
 	}
 	metricStmtBuilder, err := metricsstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
 	if err != nil {
-		return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, nil, nil, err
 	}
 	meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, fl).New(ctx, settings, cfg)
 	if err != nil {
-		return nil, nil, nil, nil, nil, nil, nil, nil, nil, err
+		return nil, nil, nil, nil, nil, nil, nil, nil, err
 	}
 
 	bucketCache := querier.NewBucketCache(settings, cache, config.Querier.CacheTTL, config.Querier.FluxInterval)
 
-	return metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, nil
+	return metadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, nil
 }
 
 func New(
@@ -342,7 +336,7 @@ func New(
 
 	// Assemble the query stack (metadata store, statement builders, bucket cache) once,
 	// and reuse the single metadata store everywhere downstream.
-	telemetryMetadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, err := newQueryStack(ctx, providerSettings, config, telemetrystore, cache, flagger)
+	telemetryMetadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, err := newQueryStack(ctx, providerSettings, config, telemetrystore, cache, flagger)
 	if err != nil {
 		return nil, err
 	}
@@ -352,7 +346,7 @@ func New(
 		ctx,
 		providerSettings,
 		config.Querier,
-		NewQuerierProviderFactories(telemetrystore, prometheus, promV2, telemetryMetadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
+		NewQuerierProviderFactories(telemetrystore, prometheus, promV2, telemetryMetadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
 		config.Querier.Provider(),
 	)
 	if err != nil {

pkg/sqlmigration/107_scrub_email_channel_transport.go

@@ -0,0 +1,274 @@
+package sqlmigration
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/sqlstore"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/migrate"
+)
+
+type scrubEmailChannelTransport struct {
+	sqlstore sqlstore.SQLStore
+	logger   *slog.Logger
+}
+
+type alertmanagerConfigScrubRow struct {
+	bun.BaseModel `bun:"table:alertmanager_config"`
+
+	ID     string `bun:"id"`
+	Config string `bun:"config"`
+}
+
+type notificationChannelScrubRow struct {
+	bun.BaseModel `bun:"table:notification_channel"`
+
+	ID   string `bun:"id"`
+	Data string `bun:"data"`
+}
+
+var emailTransportKeys = []string{
+	"from",
+	"hello",
+	"smarthost",
+	"auth_username",
+	"auth_password",
+	"auth_password_file",
+	"auth_secret",
+	"auth_secret_file",
+	"auth_identity",
+	"require_tls",
+	"tls_config",
+	"force_implicit_tls",
+}
+
+var globalSMTPKeys = []string{
+	"smtp_from",
+	"smtp_hello",
+	"smtp_smarthost",
+	"smtp_auth_username",
+	"smtp_auth_password",
+	"smtp_auth_password_file",
+	"smtp_auth_secret",
+	"smtp_auth_secret_file",
+	"smtp_auth_identity",
+	"smtp_require_tls",
+	"smtp_tls_config",
+	"smtp_force_implicit_tls",
+}
+
+func NewScrubEmailChannelTransportFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
+	return factory.NewProviderFactory(
+		factory.MustNewName("scrub_email_channel_transport"),
+		func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
+			return &scrubEmailChannelTransport{sqlstore: sqlstore, logger: ps.Logger}, nil
+		},
+	)
+}
+
+func (migration *scrubEmailChannelTransport) Register(migrations *migrate.Migrations) error {
+	if err := migrations.Register(migration.Up, migration.Down); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (migration *scrubEmailChannelTransport) Up(ctx context.Context, db *bun.DB) error {
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = tx.Rollback()
+	}()
+
+	if err := migration.scrubConfigs(ctx, tx); err != nil {
+		return err
+	}
+
+	if err := migration.scrubChannels(ctx, tx); err != nil {
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func (migration *scrubEmailChannelTransport) scrubConfigs(ctx context.Context, tx bun.Tx) error {
+	rows := make([]*alertmanagerConfigScrubRow, 0)
+	if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		cfg := make(map[string]json.RawMessage)
+		if err := json.Unmarshal([]byte(row.Config), &cfg); err != nil {
+			migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable config", slog.String("config_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		changed := false
+
+		if globalRaw, ok := cfg["global"]; ok && string(globalRaw) != "null" {
+			global := make(map[string]json.RawMessage)
+			if err := json.Unmarshal(globalRaw, &global); err != nil {
+				migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable global", slog.String("config_id", row.ID), errors.Attr(err))
+				continue
+			}
+			if deleteKeys(global, globalSMTPKeys) {
+				newGlobal, err := json.Marshal(global)
+				if err != nil {
+					migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal global", slog.String("config_id", row.ID), errors.Attr(err))
+					continue
+				}
+				cfg["global"] = newGlobal
+				changed = true
+			}
+		}
+
+		if receiversRaw, ok := cfg["receivers"]; ok && string(receiversRaw) != "null" {
+			receivers := make([]map[string]json.RawMessage, 0)
+			if err := json.Unmarshal(receiversRaw, &receivers); err != nil {
+				migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable receivers", slog.String("config_id", row.ID), errors.Attr(err))
+				continue
+			}
+
+			receiversChanged := false
+			unreadable := false
+			for _, receiver := range receivers {
+				scrubbed, err := scrubEmailConfigs(receiver)
+				if err != nil {
+					migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable email configs", slog.String("config_id", row.ID), errors.Attr(err))
+					unreadable = true
+					break
+				}
+				receiversChanged = receiversChanged || scrubbed
+			}
+			if unreadable {
+				continue
+			}
+
+			if receiversChanged {
+				newReceivers, err := json.Marshal(receivers)
+				if err != nil {
+					migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal receivers", slog.String("config_id", row.ID), errors.Attr(err))
+					continue
+				}
+				cfg["receivers"] = newReceivers
+				changed = true
+			}
+		}
+
+		if !changed {
+			continue
+		}
+
+		newConfig, err := json.Marshal(cfg)
+		if err != nil {
+			migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal config", slog.String("config_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		if _, err := tx.NewUpdate().
+			Model((*alertmanagerConfigScrubRow)(nil)).
+			Set("config = ?", string(newConfig)).
+			Set("hash = ?", fmt.Sprintf("%x", md5.Sum(newConfig))).
+			Where("id = ?", row.ID).
+			Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (migration *scrubEmailChannelTransport) scrubChannels(ctx context.Context, tx bun.Tx) error {
+	rows := make([]*notificationChannelScrubRow, 0)
+	if err := tx.NewSelect().Model(&rows).Where("type = ?", "email").Scan(ctx); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		receiver := make(map[string]json.RawMessage)
+		if err := json.Unmarshal([]byte(row.Data), &receiver); err != nil {
+			migration.logger.WarnContext(ctx, "skipping notification channel with unreadable data", slog.String("channel_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		scrubbed, err := scrubEmailConfigs(receiver)
+		if err != nil {
+			migration.logger.WarnContext(ctx, "skipping notification channel with unreadable email configs", slog.String("channel_id", row.ID), errors.Attr(err))
+			continue
+		}
+		if !scrubbed {
+			continue
+		}
+
+		newData, err := json.Marshal(receiver)
+		if err != nil {
+			migration.logger.WarnContext(ctx, "skipping notification channel, cannot marshal data", slog.String("channel_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		if _, err := tx.NewUpdate().
+			Model((*notificationChannelScrubRow)(nil)).
+			Set("data = ?", string(newData)).
+			Where("id = ?", row.ID).
+			Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func scrubEmailConfigs(receiver map[string]json.RawMessage) (bool, error) {
+	emailConfigsRaw, ok := receiver["email_configs"]
+	if !ok || string(emailConfigsRaw) == "null" {
+		return false, nil
+	}
+
+	emailConfigs := make([]map[string]json.RawMessage, 0)
+	if err := json.Unmarshal(emailConfigsRaw, &emailConfigs); err != nil {
+		return false, err
+	}
+
+	changed := false
+	for _, emailConfig := range emailConfigs {
+		changed = deleteKeys(emailConfig, emailTransportKeys) || changed
+	}
+
+	if !changed {
+		return false, nil
+	}
+
+	newEmailConfigs, err := json.Marshal(emailConfigs)
+	if err != nil {
+		return false, err
+	}
+	receiver["email_configs"] = newEmailConfigs
+
+	return true, nil
+}
+
+func deleteKeys(m map[string]json.RawMessage, keys []string) bool {
+	deleted := false
+	for _, key := range keys {
+		if _, ok := m[key]; ok {
+			delete(m, key)
+			deleted = true
+		}
+	}
+
+	return deleted
+}
+
+func (migration *scrubEmailChannelTransport) Down(context.Context, *bun.DB) error {
+	return nil
+}

pkg/statementbuilder/aistatementbuilder/statement_builder.go

@@ -1,84 +0,0 @@
-package aistatementbuilder
-
-import (
-	"strings"
-
-	"github.com/SigNoz/signoz/pkg/factory"
-	"github.com/SigNoz/signoz/pkg/flagger"
-	"github.com/SigNoz/signoz/pkg/statementbuilder"
-	scopedtraces "github.com/SigNoz/signoz/pkg/statementbuilder/scopedtracesstatementbuilder"
-	"github.com/SigNoz/signoz/pkg/telemetrystore"
-	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-)
-
-// NewFactory returns a provider factory for the AI trace statement builder
-// (builder_ai_query): the gen_ai Scope paired with the domain-neutral scoped-trace
-// topology, which owns the query construction.
-//
-// The gen_ai gate/column keys are surfaced by the metadata store itself
-// (enrichWithGenAIKeys), so queries work before any gen_ai metadata is ingested.
-func NewFactory(
-	telemetryStore telemetrystore.TelemetryStore,
-	metadataStore telemetrytypes.MetadataStore,
-	fl flagger.Flagger,
-) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.TraceAggregation], statementbuilder.Config] {
-	return scopedtraces.NewFactory(factory.MustNewName("ai"), Scope(), telemetryStore, metadataStore, fl)
-}
-
-// Scope describes gen_ai for the scoped trace builder: an AI trace has >=1 gen_ai
-// LLM, tool, or agent span, and its list adds AI/LLM per-trace metrics on top of the
-// common columns. This package holds only gen_ai domain knowledge; the query
-// topology lives in scopedtracesstatementbuilder.
-func Scope() scopedtraces.TraceScope {
-	gateKeyNames := []string{telemetrytypes.GenAIRequestModel, telemetrytypes.GenAIToolName, telemetrytypes.GenAIAgentName}
-	gateExprs := make([]string, 0, len(gateKeyNames))
-	gateKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(gateKeyNames))
-	for _, name := range gateKeyNames {
-		gateExprs = append(gateExprs, name+" EXISTS")
-		gateKeys = append(gateKeys, &telemetrytypes.TelemetryFieldKey{
-			Name:         name,
-			Signal:       telemetrytypes.SignalTraces,
-			FieldContext: telemetrytypes.FieldContextAttribute,
-		})
-	}
-
-	defs := telemetrytypes.GenAIFieldDefinitions
-	reqModel := defs[telemetrytypes.GenAIRequestModel]
-	toolName := defs[telemetrytypes.GenAIToolName]
-	inTok := defs[telemetrytypes.GenAIUsageInputTokens]
-	outTok := defs[telemetrytypes.GenAIUsageOutputTokens]
-	cost := defs[telemetrytypes.SignozGenAITotalCost]
-	inMsg := defs[telemetrytypes.GenAIInputMessages]
-	outMsg := defs[telemetrytypes.GenAIOutputMessages]
-
-	str := telemetrytypes.FieldDataTypeString
-	columns := append(scopedtraces.CommonTraceColumns(),
-		// LLM calls only (request model present), not the full gate.
-		scopedtraces.TraceColumn{Alias: "llm_call_count", Orderable: true, Expr: scopedtraces.CountExists(&reqModel)},
-		scopedtraces.TraceColumn{Alias: "tool_call_count", Orderable: true, Expr: scopedtraces.CountExists(&toolName)},
-		scopedtraces.TraceColumn{Alias: "distinct_tool_count", Orderable: true, Expr: scopedtraces.UniqCount(&toolName, str)},
-		// tokens live only on LLM spans, so a plain sum needs no gate scoping.
-		scopedtraces.TraceColumn{Alias: "input_tokens", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &inTok)},
-		scopedtraces.TraceColumn{Alias: "output_tokens", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &outTok)},
-		scopedtraces.TraceColumn{Alias: "total_tokens", Orderable: true, Expr: scopedtraces.SumOfKeys(telemetrytypes.FieldDataTypeFloat64, &inTok, &outTok)},
-		// per-span cost attached by the SigNoz LLM pricing processor.
-		scopedtraces.TraceColumn{Alias: "estimated_total_cost", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &cost)},
-		// slowest single LLM call in the trace.
-		scopedtraces.TraceColumn{Alias: "max_llm_duration_nano", Orderable: true, Expr: scopedtraces.ScopedToKeyColumn(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("duration_nano"), &reqModel)},
-		// errors across the whole trace (any span), so display-only.
-		scopedtraces.TraceColumn{Alias: "error_count", Expr: scopedtraces.CondCount(scopedtraces.IntrinsicSpanKey("has_error"), qbtypes.FilterOperatorEqual, true)},
-		// timestamp of the last gen_ai span (LLM/tool/agent), hence gate-scoped.
-		scopedtraces.TraceColumn{Alias: "last_activity_time", Orderable: true, Expr: scopedtraces.ScopedReduce(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("timestamp"))},
-		// previews: first call's input (the prompt), last call's output (the answer).
-		scopedtraces.TraceColumn{Alias: "input", SpanLevel: true, Expr: scopedtraces.PickBy(&inMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickEarliest)},
-		scopedtraces.TraceColumn{Alias: "output", SpanLevel: true, Expr: scopedtraces.PickBy(&outMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickLatest)},
-	)
-
-	return scopedtraces.TraceScope{
-		FilterExpression:  strings.Join(gateExprs, " OR "),
-		FieldKeys:         gateKeys,
-		Columns:           columns,
-		DefaultOrderAlias: "last_activity_time",
-	}
-}

pkg/statementbuilder/scopedtracesstatementbuilder/aggregate.go

@@ -1,205 +0,0 @@
-package scopedtracesstatementbuilder
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
-	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
-)
-
-// This file holds the Aggregate constructors a TraceScope's columns are declared
-// with. All SQL rendering goes through the resolvers: columns/values via the
-// columnResolver, predicates via the predicateResolver.
-
-// Aggregate renders one column's SQL through the resolvers and lists the attribute
-// keys it references so the builder can pre-fetch their metadata. Build one with the
-// constructors below; the zero value is not usable.
-type Aggregate struct {
-	keys   []*telemetrytypes.TelemetryFieldKey
-	render func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, preds *predicateResolver) (expr string, err error)
-}
-
-// IntrinsicSpanKey references an intrinsic span-index field (timestamp, name, …) by
-// its canonical name; the field mapper resolves it to the physical column.
-func IntrinsicSpanKey(name string) *telemetrytypes.TelemetryFieldKey {
-	return &telemetrytypes.TelemetryFieldKey{
-		Name:         name,
-		Signal:       telemetrytypes.SignalTraces,
-		FieldContext: telemetrytypes.FieldContextSpan,
-	}
-}
-
-// AggFunc is a ClickHouse aggregate function name.
-type AggFunc string
-
-const (
-	AggSum AggFunc = "sum"
-	AggMax AggFunc = "max"
-	AggMin AggFunc = "min"
-)
-
-// PickDirection selects the earliest (argMin) or latest (argMax) span by ordering.
-type PickDirection int
-
-const (
-	PickLatest PickDirection = iota
-	PickEarliest
-)
-
-// CountAll renders count().
-func CountAll() Aggregate {
-	return Aggregate{render: func(context.Context, valuer.UUID, uint64, uint64, *columnResolver, *predicateResolver) (string, error) {
-		return "count()", nil
-	}}
-}
-
-// FieldReduce renders <fn>(<field>) over a field-mapper-resolved column.
-func FieldReduce(fn AggFunc, key *telemetrytypes.TelemetryFieldKey) Aggregate {
-	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, _ *predicateResolver) (string, error) {
-		f, err := cols.FieldFor(ctx, orgID, startNs, endNs, key)
-		if err != nil {
-			return "", err
-		}
-		return fmt.Sprintf("%s(%s)", fn, f), nil
-	}}
-}
-
-// TraceDuration renders the full-trace wall duration: last span end minus first
-// span start.
-func TraceDuration(tsKey, durationKey *telemetrytypes.TelemetryFieldKey) Aggregate {
-	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, _ *predicateResolver) (string, error) {
-		ts, err := cols.FieldFor(ctx, orgID, startNs, endNs, tsKey)
-		if err != nil {
-			return "", err
-		}
-		dur, err := cols.FieldFor(ctx, orgID, startNs, endNs, durationKey)
-		if err != nil {
-			return "", err
-		}
-		tsNano := tracestelemetryschema.UnixNanoExpr(ts)
-		return fmt.Sprintf("(max(%s + %s) - min(%s))", tsNano, dur, tsNano), nil
-	}}
-}
-
-// FieldAnyWhere renders anyIf(<field>, <cond>) — the field value from any span
-// matching the condition.
-func FieldAnyWhere(valueKey, condKey *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, condValue any) Aggregate {
-	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, preds *predicateResolver) (string, error) {
-		v, err := cols.FieldFor(ctx, orgID, startNs, endNs, valueKey)
-		if err != nil {
-			return "", err
-		}
-		cond, err := preds.ConditionFor(ctx, orgID, startNs, endNs, condKey, op, condValue)
-		return fmt.Sprintf("anyIf(%s, %s)", v, cond), err
-	}}
-}
-
-// AnyValue renders any(<value>) over a metadata-resolved attribute value.
-func AnyValue(key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) Aggregate {
-	return Aggregate{keys: []*telemetrytypes.TelemetryFieldKey{key}, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, _ *predicateResolver) (string, error) {
-		v, err := cols.ValueFor(ctx, orgID, startNs, endNs, key, dt)
-		return fmt.Sprintf("any(%s)", v), err
-	}}
-}
-
-// CountExists renders countIf(<key> EXISTS) — counts spans carrying key.
-func CountExists(key *telemetrytypes.TelemetryFieldKey) Aggregate {
-	return Aggregate{keys: []*telemetrytypes.TelemetryFieldKey{key}, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, _ *columnResolver, preds *predicateResolver) (string, error) {
-		cond, err := preds.ExistsFor(ctx, orgID, startNs, endNs, key)
-		return fmt.Sprintf("countIf(%s)", cond), err
-	}}
-}
-
-// CondCount renders countIf(<cond>) over a condition-builder-resolved predicate.
-func CondCount(key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) Aggregate {
-	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, _ *columnResolver, preds *predicateResolver) (string, error) {
-		cond, err := preds.ConditionFor(ctx, orgID, startNs, endNs, key, op, value)
-		return fmt.Sprintf("countIf(%s)", cond), err
-	}}
-}
-
-// Reduce renders <fn>(<value>) over a resolved numeric attribute value.
-func Reduce(fn AggFunc, valueKey *telemetrytypes.TelemetryFieldKey) Aggregate {
-	return Aggregate{keys: []*telemetrytypes.TelemetryFieldKey{valueKey}, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, _ *predicateResolver) (string, error) {
-		v, err := cols.ValueFor(ctx, orgID, startNs, endNs, valueKey, telemetrytypes.FieldDataTypeFloat64)
-		return fmt.Sprintf("%s(%s)", fn, v), err
-	}}
-}
-
-// ScopedReduce renders <fn>If(<field>, <gate mask>) over a field-mapper-resolved column.
-func ScopedReduce(fn AggFunc, key *telemetrytypes.TelemetryFieldKey) Aggregate {
-	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, preds *predicateResolver) (string, error) {
-		f, err := cols.FieldFor(ctx, orgID, startNs, endNs, key)
-		if err != nil {
-			return "", err
-		}
-		return fmt.Sprintf("%sIf(%s, %s)", fn, f, preds.maskExpr), nil
-	}}
-}
-
-// ScopedToKeyColumn renders <fn>If(<field>, <scopeKey> EXISTS) — a span-index field
-// aggregated over spans carrying scopeKey (e.g. max LLM latency).
-func ScopedToKeyColumn(fn AggFunc, columnKey, scopeKey *telemetrytypes.TelemetryFieldKey) Aggregate {
-	return Aggregate{keys: []*telemetrytypes.TelemetryFieldKey{scopeKey}, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, preds *predicateResolver) (string, error) {
-		col, err := cols.FieldFor(ctx, orgID, startNs, endNs, columnKey)
-		if err != nil {
-			return "", err
-		}
-		cond, err := preds.ExistsFor(ctx, orgID, startNs, endNs, scopeKey)
-		return fmt.Sprintf("%sIf(%s, %s)", fn, col, cond), err
-	}}
-}
-
-// PickBy renders argMinIf/argMaxIf(<value>, <orderField>, <value> EXISTS) — the value
-// from the earliest/latest span that carries it.
-func PickBy(valueKey *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType, orderKey *telemetrytypes.TelemetryFieldKey, dir PickDirection) Aggregate {
-	fn := "argMaxIf"
-	if dir == PickEarliest {
-		fn = "argMinIf"
-	}
-	return Aggregate{keys: []*telemetrytypes.TelemetryFieldKey{valueKey}, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, preds *predicateResolver) (string, error) {
-		v, err := cols.ValueFor(ctx, orgID, startNs, endNs, valueKey, dt)
-		if err != nil {
-			return "", err
-		}
-		order, err := cols.FieldFor(ctx, orgID, startNs, endNs, orderKey)
-		if err != nil {
-			return "", err
-		}
-		cond, err := preds.ExistsFor(ctx, orgID, startNs, endNs, valueKey)
-		return fmt.Sprintf("%s(%s, %s, %s)", fn, v, order, cond), err
-	}}
-}
-
-// UniqCount renders uniqIf(<value>, <value> EXISTS) — distinct count of an attribute.
-func UniqCount(valueKey *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) Aggregate {
-	return Aggregate{keys: []*telemetrytypes.TelemetryFieldKey{valueKey}, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, preds *predicateResolver) (string, error) {
-		v, err := cols.ValueFor(ctx, orgID, startNs, endNs, valueKey, dt)
-		if err != nil {
-			return "", err
-		}
-		cond, err := preds.ExistsFor(ctx, orgID, startNs, endNs, valueKey)
-		return fmt.Sprintf("uniqIf(%s, %s)", v, cond), err
-	}}
-}
-
-// SumOfKeys renders coalesce(sum(<v1>), 0) + coalesce(sum(<v2>), 0) + … over several
-// numeric attributes. Coalesced because a key absent from every span sums to NULL and
-// NULL + n = NULL — a trace with only output tokens would otherwise total NULL.
-func SumOfKeys(dt telemetrytypes.FieldDataType, valueKeys ...*telemetrytypes.TelemetryFieldKey) Aggregate {
-	return Aggregate{keys: valueKeys, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, cols *columnResolver, _ *predicateResolver) (string, error) {
-		parts := make([]string, 0, len(valueKeys))
-		for _, k := range valueKeys {
-			v, err := cols.ValueFor(ctx, orgID, startNs, endNs, k, dt)
-			if err != nil {
-				return "", err
-			}
-			parts = append(parts, fmt.Sprintf("coalesce(sum(%s), 0)", v))
-		}
-		return strings.Join(parts, " + "), nil
-	}}
-}

pkg/statementbuilder/scopedtracesstatementbuilder/column_resolver.go

@@ -1,45 +0,0 @@
-package scopedtracesstatementbuilder
-
-import (
-	"context"
-
-	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-// columnResolver resolves keys to bare column/value expressions through the shared
-// field mapper, following its method shapes (FieldFor / …) so column resolution reads
-// like the other statement builders. keys is the fetched metadata for the keys the
-// scope's columns reference. It binds no args, so its expressions embed in any
-// builder; predicates (which do bind args) are the predicateResolver's job.
-type columnResolver struct {
-	fm   qbtypes.FieldMapper
-	keys map[string][]*telemetrytypes.TelemetryFieldKey
-}
-
-func newColumnResolver(fm qbtypes.FieldMapper, keys map[string][]*telemetrytypes.TelemetryFieldKey) *columnResolver {
-	return &columnResolver{fm: fm, keys: keys}
-}
-
-// FieldFor returns the column expression for key via the field mapper.
-func (r *columnResolver) FieldFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
-	return r.fm.FieldFor(ctx, orgID, startNs, endNs, key)
-}
-
-// ValueFor returns the value expression for an attribute key.
-func (r *columnResolver) ValueFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) (string, error) {
-	// TODO(nitya): Fix this as this is not correct way
-	if cands := r.keys[key.Name]; len(cands) > 0 {
-		key = cands[0]
-	}
-	expr, err := r.fm.ColumnExpressionFor(ctx, orgID, startNs, endNs, key, dt, r.keys)
-	if err != nil {
-		return "", err
-	}
-	// Escape before embedding in the outer builder: a materialized column name carries
-	// `$$` (from the dotted attribute name), which go-sqlbuilder's Build would otherwise
-	// unescape to a single `$` and reference the wrong column.
-	return sqlbuilder.Escape(expr), nil
-}

pkg/statementbuilder/scopedtracesstatementbuilder/predicate_resolver.go

@@ -1,52 +0,0 @@
-package scopedtracesstatementbuilder
-
-import (
-	"context"
-
-	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-// predicateResolver resolves key + operator + value to boolean predicates through the
-// shared condition builder, following its method shapes (ConditionFor / ExistsFor).
-// keys is the fetched metadata for the keys the scope's columns reference; the gate
-// mask is set by the builder after resolveMask (Scoped* aggregates embed it). Args
-// bind into sb as $n markers, so returned predicates can be embedded anywhere in sb
-// (SELECT, WHERE, HAVING) and every occurrence resolves to the same arg.
-type predicateResolver struct {
-	cb       qbtypes.ConditionBuilder
-	keys     map[string][]*telemetrytypes.TelemetryFieldKey
-	sb       *sqlbuilder.SelectBuilder
-	maskExpr string
-}
-
-func newPredicateResolver(cb qbtypes.ConditionBuilder, keys map[string][]*telemetrytypes.TelemetryFieldKey, sb *sqlbuilder.SelectBuilder) *predicateResolver {
-	return &predicateResolver{cb: cb, keys: keys, sb: sb}
-}
-
-// ConditionFor returns a boolean predicate for key via the condition builder
-// (materialized column when present, else map access), args bound into sb.
-func (r *predicateResolver) ConditionFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) (string, error) {
-	// The condition builder owns key resolution: hand it the raw key plus the full
-	// metadata map and it matches/synthesizes the candidates itself.
-	conds, _, err := r.cb.ConditionFor(ctx, orgID, startNs, endNs, key, r.keys, qbtypes.ConditionBuilderOptions{}, op, value, r.sb)
-	if err != nil {
-		return "", err
-	}
-	if len(conds) == 0 {
-		return "", nil
-	}
-	// One condition per candidate variant (a key can be ingested under several data
-	// types); OR them all, like the visitor does for EXISTS.
-	if len(conds) == 1 {
-		return conds[0], nil
-	}
-	return r.sb.Or(conds...), nil
-}
-
-// ExistsFor returns the EXISTS predicate for key.
-func (r *predicateResolver) ExistsFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
-	return r.ConditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorExists, nil)
-}

pkg/statementbuilder/scopedtracesstatementbuilder/scope.go

@@ -1,66 +0,0 @@
-package scopedtracesstatementbuilder
-
-import (
-	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-)
-
-// TraceScope is the configuration the scoped trace builder accepts: which spans are
-// in scope and which per-trace columns the list computes. It only declares the gate
-// and the columns; the builder resolves everything through the field mapper, so
-// attribute access stays materialization-aware. A new span category only needs a
-// new TraceScope.
-type TraceScope struct {
-	// FilterExpression is the grammar-level (EXISTS) gate, used on the delegated
-	// span-list path.
-	FilterExpression string
-	// FieldKeys are the gate's keys, used to build the per-span mask
-	// (OR of resolved EXISTS conditions).
-	FieldKeys []*telemetrytypes.TelemetryFieldKey
-	// Columns are the per-trace output columns.
-	Columns []TraceColumn
-	// DefaultOrderAlias is sorted by (desc) when the query gives no order.
-	DefaultOrderAlias string
-}
-
-// TraceColumn is one per-trace output column.
-type TraceColumn struct {
-	// Alias must not reuse a physical span-index column name (e.g. duration_nano):
-	// ClickHouse resolves bare identifiers to same-SELECT aliases first, so any
-	// expression referencing that column would silently bind to the alias.
-	Alias string
-	// Orderable columns can be used in ORDER BY and the aggregate filter. All-span
-	// aggregates (span_count, trace_duration_nano, …) are display-only and set false.
-	Orderable bool
-	// SpanLevel columns surface a real span/resource attribute (service.name,
-	// input/output messages); a filter on them is applied span-level, so they are
-	// excluded from the trace-level aliases.
-	SpanLevel bool
-	Expr      Aggregate
-}
-
-// CommonTraceColumns are domain-neutral columns any trace list can reuse. All
-// aggregate over every span, so none is Orderable.
-func CommonTraceColumns() []TraceColumn {
-	ts := IntrinsicSpanKey("timestamp")
-	duration := IntrinsicSpanKey("duration_nano")
-	name := IntrinsicSpanKey("name")
-	parentSpanID := IntrinsicSpanKey("parent_span_id")
-	serviceName := &telemetrytypes.TelemetryFieldKey{
-		Name:          "service.name",
-		Signal:        telemetrytypes.SignalTraces,
-		FieldContext:  telemetrytypes.FieldContextResource,
-		FieldDataType: telemetrytypes.FieldDataTypeString,
-	}
-	return []TraceColumn{
-		{Alias: "start_time", Expr: FieldReduce(AggMin, ts)},
-		{Alias: "end_time", Expr: FieldReduce(AggMax, ts)},
-		// Not plain "duration_nano": that name is the intrinsic span field, and an
-		// alias would shadow it — both in ClickHouse identifier resolution and in
-		// bare-name filter classification.
-		{Alias: "trace_duration_nano", Expr: TraceDuration(ts, duration)},
-		{Alias: "span_count", Expr: CountAll()},
-		{Alias: "root_span_name", Expr: FieldAnyWhere(name, parentSpanID, qbtypes.FilterOperatorEqual, "")},
-		{Alias: "service.name", SpanLevel: true, Expr: AnyValue(serviceName, telemetrytypes.FieldDataTypeString)},
-	}
-}

pkg/statementbuilder/scopedtracesstatementbuilder/statement_builder.go

@@ -1,735 +0,0 @@
-package scopedtracesstatementbuilder
-
-import (
-	"context"
-	"fmt"
-	"log/slog"
-	"sort"
-	"strings"
-
-	"github.com/SigNoz/signoz/pkg/errors"
-	"github.com/SigNoz/signoz/pkg/factory"
-	"github.com/SigNoz/signoz/pkg/flagger"
-	"github.com/SigNoz/signoz/pkg/querybuilder"
-	"github.com/SigNoz/signoz/pkg/statementbuilder"
-	"github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter"
-	"github.com/SigNoz/signoz/pkg/statementbuilder/tracesstatementbuilder"
-	"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
-	"github.com/SigNoz/signoz/pkg/telemetrystore"
-	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
-	qbvariables "github.com/SigNoz/signoz/pkg/variables"
-	"github.com/huandu/go-sqlbuilder"
-)
-
-var (
-	ErrUnsupportedRequestType = errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type for the scoped trace builder")
-)
-
-// scopedTraceStatementBuilder builds a trace list scoped to one span category
-// (e.g. gen_ai spans). The query shape is fixed; the TraceScope decides which spans
-// are in scope and which per-trace columns to compute, so a new category only needs
-// a new scope.
-type scopedTraceStatementBuilder struct {
-	logger                    *slog.Logger
-	metadataStore             telemetrytypes.MetadataStore
-	fm                        qbtypes.FieldMapper
-	cb                        qbtypes.ConditionBuilder
-	scope                     TraceScope
-	traceStmtBuilder          qbtypes.StatementBuilder[qbtypes.TraceAggregation]
-	resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
-}
-
-var _ qbtypes.StatementBuilder[qbtypes.TraceAggregation] = (*scopedTraceStatementBuilder)(nil)
-
-// NewFactory returns a provider factory for a scoped trace statement builder. Unlike
-// the per-signal factories this package is domain-neutral, so the caller supplies the
-// factory name and the TraceScope (see aistatementbuilder for the gen_ai scope). Its
-// New delegates the span-list path to a trace statement builder built via the traces
-// factory — mirroring how the meter factory builds its own metrics builder.
-func NewFactory(
-	name factory.Name,
-	scope TraceScope,
-	telemetryStore telemetrystore.TelemetryStore,
-	metadataStore telemetrytypes.MetadataStore,
-	fl flagger.Flagger,
-) factory.ProviderFactory[qbtypes.StatementBuilder[qbtypes.TraceAggregation], statementbuilder.Config] {
-	return factory.NewProviderFactory(
-		name,
-		func(ctx context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.TraceAggregation], error) {
-			traceStmtBuilder, err := tracesstatementbuilder.NewFactory(telemetryStore, metadataStore, fl).New(ctx, settings, cfg)
-			if err != nil {
-				return nil, err
-			}
-			fm := tracestelemetryschema.NewFieldMapper()
-			cb := tracestelemetryschema.NewConditionBuilder(fm)
-			return NewScopedTraceStatementBuilder(settings, metadataStore, fm, cb, scope, traceStmtBuilder, fl), nil
-		},
-	)
-}
-
-// NewScopedTraceStatementBuilder wires the generic trace-list builder.
-// traceStmtBuilder (the delegate for the span-list path) is injected because
-// NewFactory already builds the canonical instance.
-func NewScopedTraceStatementBuilder(
-	settings factory.ProviderSettings,
-	metadataStore telemetrytypes.MetadataStore,
-	fieldMapper qbtypes.FieldMapper,
-	conditionBuilder qbtypes.ConditionBuilder,
-	scope TraceScope,
-	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
-	fl flagger.Flagger,
-) qbtypes.StatementBuilder[qbtypes.TraceAggregation] {
-	scopedSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/statementbuilder/scopedtracesstatementbuilder")
-
-	// Same resource-fingerprint prune as the standard trace builder — the list scans
-	// the same span index.
-	resourceFilterStmtBuilder := resourcefilter.New[qbtypes.TraceAggregation](
-		settings,
-		tracestelemetryschema.DBName,
-		tracestelemetryschema.TracesResourceV3TableName,
-		telemetrytypes.SignalTraces,
-		telemetrytypes.SourceUnspecified,
-		metadataStore,
-		nil,
-		fl,
-	)
-
-	return &scopedTraceStatementBuilder{
-		logger:                    scopedSettings.Logger(),
-		metadataStore:             metadataStore,
-		fm:                        fieldMapper,
-		cb:                        conditionBuilder,
-		scope:                     scope,
-		traceStmtBuilder:          traceStmtBuilder,
-		resourceFilterStmtBuilder: resourceFilterStmtBuilder,
-	}
-}
-
-func (b *scopedTraceStatementBuilder) Build(
-	ctx context.Context,
-	orgID valuer.UUID,
-	start uint64,
-	end uint64,
-	requestType qbtypes.RequestType,
-	query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
-	variables map[string]qbtypes.VariableItem,
-) (*qbtypes.Statement, error) {
-	switch requestType {
-	case qbtypes.RequestTypeTrace:
-		return b.buildTraceListQuery(ctx, orgID, querybuilder.ToNanoSecs(start), querybuilder.ToNanoSecs(end), query, variables)
-	case qbtypes.RequestTypeRaw:
-		return b.buildDelegated(ctx, orgID, start, end, requestType, query, variables)
-	default:
-		return nil, ErrUnsupportedRequestType
-	}
-}
-
-// buildDelegated ANDs the base gate into the user filter and delegates to the
-// standard trace builder (the span-list / raw path).
-func (b *scopedTraceStatementBuilder) buildDelegated(
-	ctx context.Context,
-	orgID valuer.UUID,
-	start, end uint64,
-	requestType qbtypes.RequestType,
-	query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
-	variables map[string]qbtypes.VariableItem,
-) (*qbtypes.Statement, error) {
-	gate := b.scope.FilterExpression
-	expr := gate
-	if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
-		expr = fmt.Sprintf("(%s) AND (%s)", gate, query.Filter.Expression)
-	}
-
-	// shallow copy; only Filter is replaced, caller's query untouched
-	gated := query
-	gated.Filter = &qbtypes.Filter{Expression: expr}
-
-	return b.traceStmtBuilder.Build(ctx, orgID, start, end, requestType, gated, variables)
-}
-
-// buildTraceListQuery wires the CTE pipeline: one windowed pass picks the top-N
-// traces, then a bucket-pruned pass enriches only those.
-// Helpers appear in this file in the order they run. start/end are nanoseconds.
-//
-//	RESOLVE (keys/columns → SQL via the field mapper)
-//	  fetchKeys         metadata for every key we reference
-//	  resolveMask       the "span is in scope" predicate (OR of EXISTS)
-//	  resolveColumns    per-trace column SQL
-//	  resolveListOrders which columns to ORDER BY
-//	  splitFilter       span-level predicate + trace-level HAVING
-//
-//	BUILD
-//	  matched    one windowed, mask-pruned GROUP BY trace_id scan fusing gate + span
-//	     │       filter + HAVING + ORDER BY + LIMIT/OFFSET → the top-N trace_ids
-//	     ▼
-//	  ranked     [start,end] bounds of those traces, from the small summary table
-//	     ▼
-//	  buckets    the ts_bucket_start values they touch, to prune the next scan
-//	     ▼
-//	  enrichment every per-trace column for those traces over their full extent
-//	             (not window-clipped), scanning only their buckets
-//
-// Only Orderable columns are computable in the mask-pruned matched pass, so only they
-// can be ordered or filtered on; all-span columns (span_count, …) are output-only.
-func (b *scopedTraceStatementBuilder) buildTraceListQuery(
-	ctx context.Context,
-	orgID valuer.UUID,
-	start, end uint64,
-	query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
-	variables map[string]qbtypes.VariableItem,
-) (*qbtypes.Statement, error) {
-
-	startBucket := start/querybuilder.NsToSeconds - querybuilder.BucketAdjustment
-	endBucket := end / querybuilder.NsToSeconds
-
-	limit := query.Limit
-	if limit <= 0 {
-		limit = 100
-	}
-
-	// Resolve keys once; all attribute access goes through the field mapper. Condition
-	// args bind into the builder an expression is embedded in, so the matched and
-	// enrichment passes each resolve against their own builder.
-	keys, err := b.fetchKeys(ctx, orgID)
-	if err != nil {
-		return nil, err
-	}
-	matchedSB := sqlbuilder.NewSelectBuilder()
-	maskExpr, resolved, err := b.resolveFor(ctx, orgID, start, end, keys, matchedSB)
-	if err != nil {
-		return nil, err
-	}
-	enrichSB := sqlbuilder.NewSelectBuilder()
-	_, enrichResolved, err := b.resolveFor(ctx, orgID, start, end, keys, enrichSB)
-	if err != nil {
-		return nil, err
-	}
-	orders, err := b.resolveListOrders(query.Order, resolved)
-	if err != nil {
-		return nil, err
-	}
-	orderableSet := orderableAliasSet(resolved)
-
-	// If the filter references resource attributes, add a __resource_filter CTE and
-	// narrow the matched scan by resource_fingerprint; the span predicate drops those
-	// keys so they aren't applied twice.
-	resourceFrag, resourceArgs, resourcePred, err := b.maybeAttachResourceFilter(ctx, orgID, query, start, end, variables)
-	if err != nil {
-		return nil, err
-	}
-
-	// Split the user filter: span-level predicate + trace-level HAVING expression.
-	fp, err := b.splitFilter(ctx, orgID, query, b.aggregateAliasSet(), orderableSet, start, end, variables, matchedSB)
-	if err != nil {
-		return nil, err
-	}
-
-	// matched → ranked → buckets → enrichment
-	matchedFrag, matchedArgs, err := b.buildMatchedCTE(matchedSB, start, end, startBucket, endBucket, resolved, orders, orderableSet, maskExpr, fp, resourcePred, limit, query.Offset)
-	if err != nil {
-		return nil, err
-	}
-	rankedFrag, rankedArgs := b.buildRankedCTE(start, end)
-
-	// buckets: the ts_bucket_start values the matched traces span, so the enrichment
-	// scan is primary-key pruned. No args.
-	adj := querybuilder.BucketAdjustment // 30-min bucket width in seconds
-	bucketsFrag := fmt.Sprintf("buckets AS (SELECT DISTINCT b AS ts_bucket FROM ranked "+
-		"ARRAY JOIN range("+
-		"toUInt64(intDiv(toUnixTimestamp(t_start), %d) * %d - %d), "+
-		"toUInt64(intDiv(toUnixTimestamp(t_end), %d) * %d + %d), "+
-		"%d) AS b)", adj, adj, adj, adj, adj, adj, adj)
-
-	mainSQL, mainArgs := b.buildEnrichmentSelect(enrichSB, enrichResolved, orders)
-
-	cteFragments := []string{matchedFrag, rankedFrag, bucketsFrag}
-	cteArgs := [][]any{matchedArgs, rankedArgs, nil}
-
-	// __resource_filter must precede `matched`, which references it.
-	if resourceFrag != "" {
-		cteFragments = append([]string{resourceFrag}, cteFragments...)
-		cteArgs = append([][]any{resourceArgs}, cteArgs...)
-	}
-
-	finalSQL := querybuilder.CombineCTEs(cteFragments) + mainSQL + " SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000"
-	finalArgs := querybuilder.PrependArgs(cteArgs, mainArgs)
-
-	return &qbtypes.Statement{
-		Query:          finalSQL,
-		Args:           finalArgs,
-		Warnings:       fp.warnings,
-		WarningsDocURL: fp.warningsURL,
-	}, nil
-}
-
-// maybeAttachResourceFilter builds the __resource_filter CTE (fingerprints matching
-// the filter's resource conditions) and the predicate narrowing the span scan by
-// resource_fingerprint; with no resource conditions it returns empty fragments.
-//
-// Unlike the standard trace builder there is deliberately no skip-fingerprint
-// fallback: falling back would leave the resource conditions inside the OR'd
-// span-filter bucket, which changes trace membership (any span from the resource +
-// any gen_ai span, instead of a gen_ai span from the resource). Resource conditions
-// always scope the whole matched scan.
-func (b *scopedTraceStatementBuilder) maybeAttachResourceFilter(
-	ctx context.Context,
-	orgID valuer.UUID,
-	query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
-	start, end uint64,
-	variables map[string]qbtypes.VariableItem,
-) (cteFrag string, cteArgs []any, fingerprintPred string, err error) {
-	stmt, err := b.resourceFilterStmtBuilder.Build(
-		ctx, orgID, start, end, qbtypes.RequestTypeRaw, query, variables,
-	)
-	if err != nil {
-		return "", nil, "", err
-	}
-	if stmt == nil {
-		return "", nil, "", nil
-	}
-	return fmt.Sprintf("__resource_filter AS (%s)", stmt.Query), stmt.Args,
-		"resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)", nil
-}
-
-// ---------------------------------------------------------------------------
-// RESOLVE — turn keys/columns into field-mapper-aware SQL
-// ---------------------------------------------------------------------------
-
-func (b *scopedTraceStatementBuilder) fetchKeys(ctx context.Context, orgID valuer.UUID) (map[string][]*telemetrytypes.TelemetryFieldKey, error) {
-	fields := b.resolverFieldKeys()
-	selectors := make([]*telemetrytypes.FieldKeySelector, 0, len(fields))
-	for _, k := range fields {
-		selectors = append(selectors, &telemetrytypes.FieldKeySelector{
-			Name:              k.Name,
-			Signal:            k.Signal,
-			FieldContext:      k.FieldContext,
-			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
-		})
-	}
-	keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
-	return keys, err
-}
-
-func (b *scopedTraceStatementBuilder) resolverFieldKeys() []*telemetrytypes.TelemetryFieldKey {
-	seen := make(map[string]struct{})
-	var out []*telemetrytypes.TelemetryFieldKey
-	add := func(k *telemetrytypes.TelemetryFieldKey) {
-		if k == nil {
-			return
-		}
-		if _, dup := seen[k.Name]; dup {
-			return
-		}
-		seen[k.Name] = struct{}{}
-		out = append(out, k)
-	}
-	for _, k := range b.scope.FieldKeys {
-		add(k)
-	}
-	for _, c := range b.scope.Columns {
-		for _, k := range c.Expr.keys {
-			add(k)
-		}
-	}
-	return out
-}
-
-// resolveFor renders the gate mask and every scope column with condition args bound
-// into sb, so the returned expressions embed anywhere in that builder.
-func (b *scopedTraceStatementBuilder) resolveFor(ctx context.Context, orgID valuer.UUID, start, end uint64, keys map[string][]*telemetrytypes.TelemetryFieldKey, sb *sqlbuilder.SelectBuilder) (string, []resolvedColumn, error) {
-	cols := newColumnResolver(b.fm, keys)
-	preds := newPredicateResolver(b.cb, keys, sb)
-	maskExpr, err := b.resolveMask(ctx, orgID, start, end, preds)
-	if err != nil {
-		return "", nil, err
-	}
-	preds.maskExpr = maskExpr
-	resolved, err := b.resolveColumns(ctx, orgID, start, end, cols, preds)
-	if err != nil {
-		return "", nil, err
-	}
-	return maskExpr, resolved, nil
-}
-
-// resolveMask builds the per-span in-scope mask: OR of resolved EXISTS predicates
-// over the base condition's field keys.
-func (b *scopedTraceStatementBuilder) resolveMask(ctx context.Context, orgID valuer.UUID, start, end uint64, preds *predicateResolver) (string, error) {
-	fieldKeys := b.scope.FieldKeys
-	parts := make([]string, 0, len(fieldKeys))
-	for _, key := range fieldKeys {
-		e, err := preds.ExistsFor(ctx, orgID, start, end, key)
-		if err != nil {
-			return "", err
-		}
-		parts = append(parts, e)
-	}
-	return "(" + strings.Join(parts, " OR ") + ")", nil
-}
-
-// resolvedColumn is a column resolved to SQL via the field mapper, ready to embed in
-// the builder it was resolved against.
-type resolvedColumn struct {
-	alias     string
-	expr      string
-	orderable bool
-}
-
-// resolveColumns turns the declarative columns into SQL through the resolvers, so all
-// attribute access goes through the field mapper / condition builder.
-func (b *scopedTraceStatementBuilder) resolveColumns(ctx context.Context, orgID valuer.UUID, start, end uint64, cols *columnResolver, preds *predicateResolver) ([]resolvedColumn, error) {
-	out := make([]resolvedColumn, 0, len(b.scope.Columns))
-	for _, c := range b.scope.Columns {
-		expr, err := c.Expr.render(ctx, orgID, start, end, cols, preds)
-		if err != nil {
-			return nil, err
-		}
-		out = append(out, resolvedColumn{alias: c.Alias, expr: expr, orderable: c.Orderable})
-	}
-	return out, nil
-}
-
-// listOrder is a sort key resolved to a column alias + direction; both the matched
-// CTE and the enrichment ORDER BY it.
-type listOrder struct {
-	alias     string
-	direction string
-}
-
-// resolveListOrders maps order keys to the resolved orderable columns; non-orderable
-// columns are rejected. Defaults to the column provider's default order.
-func (b *scopedTraceStatementBuilder) resolveListOrders(order []qbtypes.OrderBy, resolved []resolvedColumn) ([]listOrder, error) {
-	byAlias := make(map[string]resolvedColumn, len(resolved))
-	orderable := make([]string, 0, len(resolved))
-	for _, rc := range resolved {
-		byAlias[rc.alias] = rc
-		if rc.orderable {
-			orderable = append(orderable, rc.alias)
-		}
-	}
-
-	if len(order) == 0 {
-		return []listOrder{{alias: b.scope.DefaultOrderAlias, direction: "DESC"}}, nil
-	}
-
-	orders := make([]listOrder, 0, len(order))
-	for _, o := range order {
-		direction := "DESC"
-		if o.Direction == qbtypes.OrderDirectionAsc {
-			direction = "ASC"
-		}
-		rc, ok := byAlias[o.Key.Name]
-		if !ok || !rc.orderable {
-			return nil, errors.NewInvalidInputf(errors.CodeInvalidInput,
-				"unsupported order key %q for the trace list; orderable keys: %s", o.Key.Name, strings.Join(orderable, ", "))
-		}
-		orders = append(orders, listOrder{alias: rc.alias, direction: direction})
-	}
-	return orders, nil
-}
-
-// filterParts is the user filter split into a span-level predicate (widens the
-// matched WHERE prune and becomes a countIf existence check in HAVING) and a
-// trace-level HAVING expression.
-type filterParts struct {
-	spanPred      string
-	hasSpanFilter bool
-	havingExpr    string
-	warnings      []string
-	warningsURL   string
-}
-
-// splitFilter splits query.Filter into a span-level predicate (args bound into sb)
-// and a trace-level HAVING expression (an explicit query.Having is ANDed onto the
-// latter), then validates the trace-level part against the matched-pass aggregates.
-func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID valuer.UUID, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], classifySet, orderableSet map[string]struct{}, start, end uint64, variables map[string]qbtypes.VariableItem, sb *sqlbuilder.SelectBuilder) (filterParts, error) {
-	var fp filterParts
-	if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
-		spanExpr, traceExpr, err := querybuilder.SplitFilterForAggregates(query.Filter.Expression, classifySet)
-		if err != nil {
-			return fp, err
-		}
-		fp.havingExpr = traceExpr
-		if strings.TrimSpace(spanExpr) != "" {
-			pred, warnings, url, err := b.resolveSpanPredicate(ctx, orgID, start, end, spanExpr, variables, sb)
-			if err != nil {
-				return fp, err
-			}
-			// pred is empty when the span-level keys were all resource attributes
-			// already handled by the __resource_filter CTE.
-			if strings.TrimSpace(pred) != "" {
-				fp.spanPred, fp.hasSpanFilter = pred, true
-			}
-			fp.warnings, fp.warningsURL = warnings, url
-		}
-	}
-	if query.Having != nil && strings.TrimSpace(query.Having.Expression) != "" {
-		if fp.havingExpr != "" {
-			fp.havingExpr = fmt.Sprintf("(%s) AND (%s)", fp.havingExpr, query.Having.Expression)
-		} else {
-			fp.havingExpr = query.Having.Expression
-		}
-	}
-	// The span predicate binds variables via PrepareWhereClause; the HAVING is a plain
-	// text rewrite, so substitute variables here (list/IN quoting, __all__ drops the
-	// condition) before validating.
-	if strings.TrimSpace(fp.havingExpr) != "" && len(variables) > 0 {
-		replaced, err := qbvariables.ReplaceVariablesInExpression(fp.havingExpr, variables)
-		if err != nil {
-			return fp, err
-		}
-		fp.havingExpr = replaced
-	}
-	if err := validateAggregateFilter(fp.havingExpr, orderableSet); err != nil {
-		return fp, err
-	}
-	return fp, nil
-}
-
-// resolveSpanPredicate resolves a span-level filter expression to a bare boolean
-// SQL predicate via the field mapper, args bound into sb.
-func (b *scopedTraceStatementBuilder) resolveSpanPredicate(ctx context.Context, orgID valuer.UUID, start, end uint64, expr string, variables map[string]qbtypes.VariableItem, sb *sqlbuilder.SelectBuilder) (string, []string, string, error) {
-	selectors := querybuilder.QueryStringToKeysSelectors(expr)
-	for i := range selectors {
-		selectors[i].Signal = telemetrytypes.SignalTraces
-	}
-	keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
-	if err != nil {
-		return "", nil, "", err
-	}
-	prepared, err := querybuilder.PrepareWhereClause(expr, querybuilder.FilterExprVisitorOpts{
-		Context:          ctx,
-		OrgID:            orgID,
-		Logger:           b.logger,
-		FieldMapper:      b.fm,
-		ConditionBuilder: b.cb,
-		FieldKeys:        keys,
-		Builder:          sb,
-		// resource conditions are always handled by the __resource_filter CTE
-		SkipResourceFilter: true,
-		Variables:          variables,
-		StartNs:            start,
-		EndNs:              end,
-	})
-	if err != nil {
-		return "", nil, "", err
-	}
-	if prepared.IsEmpty() {
-		return "", nil, "", nil
-	}
-	return prepared.Expr, prepared.Warnings, prepared.WarningsDocURL, nil
-}
-
-// buildMatchedCTE builds `matched`: the single windowed GROUP BY trace_id scan that
-// fuses gate + span filter + HAVING + ORDER BY + LIMIT/OFFSET, selecting only the
-// aliases the ORDER BY / HAVING reference. resolved, maskExpr and fp.spanPred carry
-// $n markers bound to sb, so each can appear several times (SELECT, WHERE, HAVING)
-// and every occurrence resolves to the same arg.
-func (b *scopedTraceStatementBuilder) buildMatchedCTE(sb *sqlbuilder.SelectBuilder, start, end, startBucket, endBucket uint64, resolved []resolvedColumn, orders []listOrder, orderableSet map[string]struct{}, maskExpr string, fp filterParts, resourcePred string, limit, offset int) (string, []any, error) {
-	// SELECT trace_id + only the aggregates ORDER BY / HAVING reference (as aliases).
-	needed := neededMatchedAliases(orders, fp.havingExpr, orderableSet)
-	selects := []string{"trace_id"}
-	for _, rc := range resolved {
-		if _, ok := needed[rc.alias]; !ok {
-			continue
-		}
-		selects = append(selects, rc.expr+" AS "+quoteAlias(rc.alias))
-	}
-	sb.Select(selects...)
-	sb.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
-
-	// WHERE: window + prune to in-scope spans, widened by the span filter so its
-	// spans survive for the countIf existence check below.
-	prune := "(" + maskExpr
-	if fp.hasSpanFilter {
-		prune += " OR " + fp.spanPred
-	}
-	prune += ")"
-	where := []string{
-		sb.GE("timestamp", fmt.Sprintf("%d", start)),
-		sb.L("timestamp", fmt.Sprintf("%d", end)),
-		sb.GE("ts_bucket_start", startBucket),
-		sb.LE("ts_bucket_start", endBucket),
-		prune,
-	}
-	if resourcePred != "" {
-		where = append(where, resourcePred)
-	}
-	sb.Where(where...)
-	sb.GroupBy("trace_id")
-
-	// HAVING: the gate/span existence checks are only needed when the WHERE was
-	// widened by a span filter; otherwise the mask alone already enforces the gate.
-	var having []string
-	if fp.hasSpanFilter {
-		having = append(having, "countIf("+maskExpr+") > 0")
-		having = append(having, "countIf("+fp.spanPred+") > 0")
-	}
-	if strings.TrimSpace(fp.havingExpr) != "" {
-		// Rewrite the trace-level HAVING to the matched-pass column aliases. The
-		// rewriter matches raw key text, so the trace. form is mapped alongside the
-		// bare name.
-		columnMap := make(map[string]string, len(orderableSet)*2)
-		for a := range orderableSet {
-			columnMap[a] = quoteAlias(a)
-			columnMap[telemetrytypes.FieldContextTrace.StringValue()+"."+a] = quoteAlias(a)
-		}
-		hv, err := querybuilder.NewHavingExpressionRewriter().Rewrite(fp.havingExpr, columnMap)
-		if err != nil {
-			return "", nil, err
-		}
-		if hv != "" {
-			// hv carries user text with values inlined by the rewriter; escape it so a
-			// literal $ can't be read as an arg marker at Build time. The countIf
-			// entries above hold live $n markers and must stay unescaped.
-			having = append(having, sqlbuilder.Escape(hv))
-		}
-	}
-	if len(having) > 0 {
-		sb.Having(strings.Join(having, " AND "))
-	}
-
-	sb.OrderBy(orderClause(orders)...)
-	sb.Limit(limit)
-	if offset > 0 {
-		sb.Offset(offset)
-	}
-
-	sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
-	return fmt.Sprintf("matched AS (%s)", sql), args, nil
-}
-
-// buildRankedCTE builds `ranked`: [start,end] bounds per matched trace, read from the
-// small trace-summary table.
-func (b *scopedTraceStatementBuilder) buildRankedCTE(start, end uint64) (string, []any) {
-	sb := sqlbuilder.NewSelectBuilder()
-	sb.Select("trace_id", "min(start) AS t_start", "max(end) AS t_end")
-	sb.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.TraceSummaryTableName))
-	sb.Where(
-		"trace_id GLOBAL IN (SELECT trace_id FROM matched)",
-		"end >= fromUnixTimestamp64Nano("+sb.Var(start)+")",
-		"start < fromUnixTimestamp64Nano("+sb.Var(end)+")",
-	)
-	sb.GroupBy("trace_id")
-	sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
-	return fmt.Sprintf("ranked AS (%s)", sql), args
-}
-
-// buildEnrichmentSelect builds the final SELECT: every per-trace column for the
-// matched traces over their full extent, scanning only their buckets.
-//
-// Accepted discrepancy: matched ranks/paginates on window-clipped values (and, with a
-// resource filter, only over fingerprint-matching spans), while this pass recomputes
-// and ORDER BYs full-trace values — so a trace with activity outside the window or
-// resource can sort differently than it ranked. Page membership is unaffected
-// (LIMIT/OFFSET runs only in matched); rows still sort by the values the user sees.
-// Ordering by matched's values instead would re-run the matched scan (ClickHouse
-// re-executes a CTE per reference) without fixing the visible cross-page artifact.
-func (b *scopedTraceStatementBuilder) buildEnrichmentSelect(sb *sqlbuilder.SelectBuilder, resolved []resolvedColumn, orders []listOrder) (string, []any) {
-	selects := []string{"trace_id"}
-	for _, rc := range resolved {
-		selects = append(selects, rc.expr+" AS "+quoteAlias(rc.alias))
-	}
-	sb.Select(selects...)
-	sb.From(fmt.Sprintf("%s.%s", tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName))
-	sb.Where(
-		"ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)",
-		"trace_id GLOBAL IN (SELECT trace_id FROM ranked)",
-	)
-	sb.GroupBy("trace_id")
-	sb.OrderBy(orderClause(orders)...)
-	return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
-}
-
-// aggregateAliasSet is every trace-level column alias, used to classify filter keys
-// as trace-level vs span-level. Derived from the scope's columns so a new column
-// can't be forgotten; SpanLevel columns are filtered span-level, so skip them.
-func (b *scopedTraceStatementBuilder) aggregateAliasSet() map[string]struct{} {
-	set := make(map[string]struct{}, len(b.scope.Columns))
-	for _, c := range b.scope.Columns {
-		if !c.SpanLevel {
-			set[c.Alias] = struct{}{}
-		}
-	}
-	return set
-}
-
-// orderableAliasSet is the subset of aliases computable in the matched pass — the
-// only ones usable in ORDER BY and the aggregate filter.
-func orderableAliasSet(resolved []resolvedColumn) map[string]struct{} {
-	set := make(map[string]struct{})
-	for _, rc := range resolved {
-		if rc.orderable {
-			set[rc.alias] = struct{}{}
-		}
-	}
-	return set
-}
-
-// neededMatchedAliases is the minimal alias set the matched pass must select: those
-// in ORDER BY plus those in the aggregate HAVING. Everything else is left to the
-// enrichment scan.
-func neededMatchedAliases(orders []listOrder, havingExpr string, orderableSet map[string]struct{}) map[string]struct{} {
-	needed := make(map[string]struct{})
-	for _, o := range orders {
-		needed[o.alias] = struct{}{}
-	}
-	for _, name := range traceAggregateNames(havingExpr) {
-		if _, ok := orderableSet[name]; ok {
-			needed[name] = struct{}{}
-		}
-	}
-	return needed
-}
-
-// traceAggregateNames extracts the aggregate names a trace-level HAVING expression
-// references. QueryStringToKeysSelectors emits an extra attribute-context fallback
-// selector for context-prefixed keys (`trace.x` → attribute "trace.x"); only the
-// unspecified- and trace-context selectors name aggregates.
-func traceAggregateNames(havingExpr string) []string {
-	var names []string
-	for _, sel := range querybuilder.QueryStringToKeysSelectors(havingExpr) {
-		if sel.FieldContext == telemetrytypes.FieldContextUnspecified || sel.FieldContext == telemetrytypes.FieldContextTrace {
-			names = append(names, sel.Name)
-		}
-	}
-	return names
-}
-
-// validateAggregateFilter rejects a trace-level filter referencing an aggregate not
-// computable in the matched pass (e.g. span_count, trace_duration_nano).
-func validateAggregateFilter(havingExpr string, orderableSet map[string]struct{}) error {
-	if strings.TrimSpace(havingExpr) == "" {
-		return nil
-	}
-	allowed := make([]string, 0, len(orderableSet))
-	for a := range orderableSet {
-		allowed = append(allowed, a)
-	}
-	sort.Strings(allowed)
-	for _, name := range traceAggregateNames(havingExpr) {
-		if _, ok := orderableSet[name]; !ok {
-			return errors.NewInvalidInputf(errors.CodeInvalidInput,
-				"aggregate %q cannot be used in the trace-list filter; filterable aggregates: %s", name, strings.Join(allowed, ", "))
-		}
-	}
-	return nil
-}
-
-// orderClause renders the ORDER BY terms plus the trace_id tiebreak.
-func orderClause(orders []listOrder) []string {
-	out := make([]string, 0, len(orders)+1)
-	for _, o := range orders {
-		out = append(out, fmt.Sprintf("%s %s", quoteAlias(o.alias), o.direction))
-	}
-	return append(out, "trace_id DESC")
-}
-
-// quoteAlias backticks an alias containing characters special to the SQL builder.
-func quoteAlias(alias string) string {
-	if strings.ContainsAny(alias, ".$`") {
-		return "`" + alias + "`"
-	}
-	return alias
-}

pkg/telemetrymetadata/metadata.go

@@ -1168,27 +1168,6 @@ func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFi
 	return keys
 }
 
-// enrichWithGenAIKeys adds keys that can be queried for GenAI signals, even though they have not been ingested yet.
-func enrichWithGenAIKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
-	for _, selector := range selectors {
-		if selector.Signal != telemetrytypes.SignalTraces && selector.Signal != telemetrytypes.SignalUnspecified {
-			continue
-		}
-		for name, def := range telemetrytypes.GenAIFieldDefinitions {
-			if len(keys[name]) > 0 {
-				continue // already resolved from ingested data
-			}
-			if !selectorMatchesIntrinsicField(selector, def) {
-				continue
-			}
-			keyCopy := def
-			keys[name] = []*telemetrytypes.TelemetryFieldKey{&keyCopy}
-		}
-	}
-
-	return keys
-}
-
 func selectorMatchesIntrinsicField(selector *telemetrytypes.FieldKeySelector, definition telemetrytypes.TelemetryFieldKey) bool {
 	if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != definition.FieldContext {
 		return false
@@ -1274,9 +1253,6 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, orgID valuer.UUID, fie
 
 	applyBackwardCompatibleKeys(mapOfKeys)
 	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, selectors)
-	if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
-		mapOfKeys = enrichWithGenAIKeys(mapOfKeys, selectors)
-	}
 
 	return mapOfKeys, complete, nil
 }
@@ -1355,9 +1331,6 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, orgID valuer.UUID
 
 	applyBackwardCompatibleKeys(mapOfKeys)
 	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, fieldKeySelectors)
-	if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
-		mapOfKeys = enrichWithGenAIKeys(mapOfKeys, fieldKeySelectors)
-	}
 
 	return mapOfKeys, complete, nil
 }

pkg/telemetryschema/tracestelemetryschema/trace_time_range.go

@@ -51,11 +51,11 @@ func (f *TraceTimeRangeFinder) GetTraceTimeRangeMulti(ctx context.Context, trace
 	query := fmt.Sprintf(`
 		SELECT
 			count(),
-			%s,
-			%s
+			toUnixTimestamp64Nano(min(start)),
+			toUnixTimestamp64Nano(max(end))
 		FROM %s.%s
 		WHERE trace_id IN (%s)
-	`, UnixNanoExpr("min(start)"), UnixNanoExpr("max(end)"), DBName, TraceSummaryTableName, strings.Join(placeholders, ", "))
+	`, DBName, TraceSummaryTableName, strings.Join(placeholders, ", "))
 
 	row := f.telemetryStore.ClickhouseDB().QueryRow(ctx, query, args...)
 
@@ -76,9 +76,3 @@ func (f *TraceTimeRangeFinder) GetTraceTimeRangeMulti(ctx context.Context, trace
 
 	return startNano, endNano, true, nil
 }
-
-// UnixNanoExpr renders the conversion of a timestamp-typed column expression
-// (DateTime64(9)) to Unix epoch nanoseconds.
-func UnixNanoExpr(expr string) string {
-	return fmt.Sprintf("toUnixTimestamp64Nano(%s)", expr)
-}

pkg/types/alertmanagertypes/channel_test.go

@@ -35,12 +35,8 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"email_configs": []any{map[string]any{
 						"send_resolved": false,
 						"to":            "test@example.com",
-						"from":          "alerts@example.com",
-						"hello":         "localhost",
-						"smarthost":     "smtp.example.com:587",
-						"require_tls":   true,
+						"smarthost":     "",
 						"html":          "{{ template \"email.default.html\" . }}",
-						"tls_config":    map[string]any{"insecure_skip_verify": false},
 						"threading":     map[string]any{},
 					}},
 				},
@@ -63,7 +59,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"slack_configs": []any{map[string]any{
 						"send_resolved": true,
 						"api_url":       "https://slack.com/api/test",
-						"app_url":       "https://slack.com/api/chat.postMessage",
 						"channel":       "#alerts",
 						"callback_id":   "{{ template \"slack.default.callbackid\" . }}",
 						"color":         "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",
@@ -77,12 +72,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"title":         "{{ template \"slack.default.title\" . }}",
 						"title_link":    "{{ template \"slack.default.titlelink\" . }}",
 						"username":      "{{ template \"slack.default.username\" . }}",
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},
@@ -104,7 +93,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"pagerduty_configs": []any{map[string]any{
 						"send_resolved": false,
 						"service_key":   "test",
-						"url":           "https://events.pagerduty.com/v2/enqueue",
 						"client":        "{{ template \"pagerduty.default.client\" . }}",
 						"client_url":    "{{ template \"pagerduty.default.clientURL\" . }}",
 						"description":   "{{ template \"pagerduty.default.description\" .}}",
@@ -116,12 +104,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 							"num_resolved": "{{ .Alerts.Resolved | len }}",
 							"resolved":     "{{ .Alerts.Resolved | toJson }}",
 						},
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},
@@ -148,7 +130,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"pagerduty_configs": []any{map[string]any{
 						"send_resolved": false,
 						"service_key":   "test",
-						"url":           "https://events.pagerduty.com/v2/enqueue",
 						"client":        "{{ template \"pagerduty.default.client\" . }}",
 						"client_url":    "{{ template \"pagerduty.default.clientURL\" . }}",
 						"description":   "{{ template \"pagerduty.default.description\" .}}",
@@ -160,12 +141,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 							"num_resolved": "{{ .Alerts.Resolved | len }}",
 							"resolved":     "{{ .Alerts.Resolved | toJson }}",
 						},
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 				{
@@ -173,7 +148,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"slack_configs": []any{map[string]any{
 						"send_resolved": true,
 						"api_url":       "https://slack.com/api/test",
-						"app_url":       "https://slack.com/api/chat.postMessage",
 						"channel":       "#alerts",
 						"callback_id":   "{{ template \"slack.default.callbackid\" . }}",
 						"color":         "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",
@@ -187,12 +161,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"title":         "{{ template \"slack.default.title\" . }}",
 						"title_link":    "{{ template \"slack.default.titlelink\" . }}",
 						"username":      "{{ template \"slack.default.username\" . }}",
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},

pkg/types/alertmanagertypes/config.go

@@ -117,6 +117,11 @@ func NewConfigFromStoreableConfig(sc *StoreableConfig) (*Config, error) {
 		return nil, err
 	}
 
+	// It must be replaced with an empty, non-nil global, upstream swaps nil for
+	// DefaultGlobalConfig, which would let a path that skips SetGlobalConfig pass
+	// validation and fail silently at delivery instead of failing fast here.
+	alertmanagerConfig.Global = &config.GlobalConfig{}
+
 	return &Config{
 		alertmanagerConfig: alertmanagerConfig,
 		customConfigs:      customConfigs,
@@ -174,7 +179,7 @@ func newConfigFromString(s string) (*config.Config, map[string]customReceiverCon
 	return amConfig, customConfigs, nil
 }
 
-func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverConfigs) []byte {
+func extendedReceivers(c *config.Config, customConfigs map[string]customReceiverConfigs) []*Receiver {
 	receivers := make([]*Receiver, len(c.Receivers))
 	for i := range c.Receivers {
 		base := c.Receivers[i]
@@ -185,7 +190,14 @@ func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverC
 		}
 	}
 
-	b, err := json.Marshal(storedConfig{Config: c, Receivers: receivers})
+	return receivers
+}
+
+func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverConfigs) []byte {
+	persistable := *c
+	persistable.Global = nil
+
+	b, err := json.Marshal(storedConfig{Config: &persistable, Receivers: extendedReceivers(c, customConfigs)})
 	if err != nil {
 		// Taking inspiration from the upstream. This is never expected to happen.
 		return []byte(fmt.Sprintf("<error creating config string: %s>", err))
@@ -206,6 +218,37 @@ func (c *Config) flush() {
 	c.storeableConfig.UpdatedAt = time.Now()
 }
 
+func (c *Config) Resolved() (*Config, error) {
+	raw, err := json.Marshal(storedConfig{Config: c.alertmanagerConfig, Receivers: extendedReceivers(c.alertmanagerConfig, c.customConfigs)})
+	if err != nil {
+		return nil, err
+	}
+
+	alertmanagerConfig, customConfigs, err := newConfigFromString(string(raw))
+	if err != nil {
+		return nil, err
+	}
+
+	if err := alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+		return nil, err
+	}
+
+	storeableConfig := *c.storeableConfig
+	resolved := &Config{
+		alertmanagerConfig: alertmanagerConfig,
+		customConfigs:      customConfigs,
+		storeableConfig:    &storeableConfig,
+	}
+	resolved.applyNativeDefaults()
+
+	return resolved, nil
+}
+
+func (c *Config) validate() error {
+	_, err := c.Resolved()
+	return err
+}
+
 func (c *Config) CopyWithReset() (*Config, error) {
 	newConfig, err := NewDefaultConfig(
 		*c.alertmanagerConfig.Global,
@@ -271,6 +314,15 @@ func (c *Config) StoreableConfig() *StoreableConfig {
 	return c.storeableConfig
 }
 
+func cloneReceiver(receiver *Receiver) (*Receiver, error) {
+	raw, err := json.Marshal(receiver)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewReceiver(string(raw))
+}
+
 func (c *Config) CreateReceiver(receiver *Receiver) error {
 	// check that receiver name is not already used
 	for _, existingReceiver := range c.alertmanagerConfig.Receivers {
@@ -279,16 +331,21 @@ func (c *Config) CreateReceiver(receiver *Receiver) error {
 		}
 	}
 
-	route, err := NewRouteFromReceiver(receiver)
+	owned, err := cloneReceiver(receiver)
+	if err != nil {
+		return err
+	}
+
+	route, err := NewRouteFromReceiver(owned)
 	if err != nil {
 		return err
 	}
 
 	c.alertmanagerConfig.Route.Routes = append(c.alertmanagerConfig.Route.Routes, route)
-	c.alertmanagerConfig.Receivers = append(c.alertmanagerConfig.Receivers, *receiver.Receiver)
-	c.setCustomConfigs(receiver)
+	c.alertmanagerConfig.Receivers = append(c.alertmanagerConfig.Receivers, *owned.Receiver)
+	c.setCustomConfigs(owned)
 
-	if err := c.alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+	if err := c.validate(); err != nil {
 		return err
 	}
 	c.applyNativeDefaults()
@@ -313,16 +370,21 @@ func (c *Config) GetReceiver(name string) (*Receiver, error) {
 }
 
 func (c *Config) UpdateReceiver(receiver *Receiver) error {
+	owned, err := cloneReceiver(receiver)
+	if err != nil {
+		return err
+	}
+
 	// find and update receiver
 	for i, existingReceiver := range c.alertmanagerConfig.Receivers {
-		if existingReceiver.Name == receiver.Name {
-			c.alertmanagerConfig.Receivers[i] = *receiver.Receiver
-			c.setCustomConfigs(receiver)
+		if existingReceiver.Name == owned.Name {
+			c.alertmanagerConfig.Receivers[i] = *owned.Receiver
+			c.setCustomConfigs(owned)
 			break
 		}
 	}
 
-	if err := c.alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+	if err := c.validate(); err != nil {
 		return err
 	}
 	c.applyNativeDefaults()

pkg/types/alertmanagertypes/config_test.go

@@ -330,6 +330,150 @@ func TestSetGlobalConfigPreservesSMTPRequireTLS(t *testing.T) {
 	}
 }
 
+func newSMTPGlobalConfig() GlobalConfig {
+	return GlobalConfig{
+		SMTPFrom:         "alerts@example.com",
+		SMTPHello:        "example.com",
+		SMTPSmarthost:    config.HostPort{Host: "smtp.sendgrid.net", Port: "587"},
+		SMTPAuthUsername: "apikey",
+		SMTPAuthPassword: "operator-secret",
+		SMTPRequireTLS:   true,
+	}
+}
+
+func newEmailTestConfig(t *testing.T) *Config {
+	t.Helper()
+
+	cfg, err := NewDefaultConfig(
+		newSMTPGlobalConfig(),
+		RouteConfig{GroupInterval: time.Minute, GroupWait: time.Minute, RepeatInterval: time.Minute},
+		"1",
+	)
+	require.NoError(t, err)
+
+	receiver, err := NewReceiver(`{"name":"email-receiver","email_configs":[{"to":"team@example.com"}]}`)
+	require.NoError(t, err)
+	require.NoError(t, cfg.CreateReceiver(receiver))
+
+	return cfg
+}
+
+func TestStoreableConfigCarriesNoSMTPSettings(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	raw := cfg.StoreableConfig().Config
+	assert.NotContains(t, raw, "operator-secret")
+	assert.NotContains(t, raw, "smtp.sendgrid.net")
+	assert.NotContains(t, raw, "apikey")
+	assert.NotContains(t, raw, "alerts@example.com")
+
+	assert.Equal(t, "operator-secret", string(cfg.alertmanagerConfig.Global.SMTPAuthPassword))
+}
+
+func TestStoreableConfigCarriesNoGlobal(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	stored := map[string]json.RawMessage{}
+	require.NoError(t, json.Unmarshal([]byte(cfg.StoreableConfig().Config), &stored))
+	assert.NotContains(t, stored, "global")
+}
+
+func TestSetGlobalConfigDoesNotChangeStoreableHash(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+	hash := cfg.StoreableConfig().Hash
+
+	require.NoError(t, cfg.SetGlobalConfig(GlobalConfig{SMTPSmarthost: config.HostPort{Host: "smtp.other.net", Port: "2525"}, SMTPAuthPassword: "rotated-secret"}))
+
+	assert.Equal(t, hash, cfg.StoreableConfig().Hash)
+	assert.NotContains(t, cfg.StoreableConfig().Config, "rotated-secret")
+}
+
+func TestNewConfigFromStoreableConfigDiscardsStoredGlobal(t *testing.T) {
+	stored := &StoreableConfig{
+		Config: `{"global":{"resolve_timeout":"5m","smtp_smarthost":"email-smtp.us-east-1.amazonaws.com:587","smtp_auth_password":"old-secret","slack_api_url":"https://hooks.slack.com/services/T/B/X"},"route":{"receiver":"default-receiver"},"receivers":[{"name":"default-receiver"}]}`,
+		OrgID:  "1",
+	}
+
+	cfg, err := NewConfigFromStoreableConfig(stored)
+	require.NoError(t, err)
+
+	assert.Equal(t, &config.GlobalConfig{}, cfg.alertmanagerConfig.Global)
+}
+
+func TestResolvedFillsEmailTransportFromGlobal(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Len(t, receiver.EmailConfigs, 1)
+
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "team@example.com", got.To)
+	assert.Equal(t, "smtp.sendgrid.net:587", got.Smarthost.String())
+	assert.Equal(t, "alerts@example.com", got.From)
+	assert.Equal(t, "apikey", got.AuthUsername)
+	assert.Equal(t, "operator-secret", string(got.AuthPassword))
+	require.NotNil(t, got.RequireTLS)
+	assert.True(t, *got.RequireTLS)
+
+	stored, err := cfg.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	assert.Empty(t, stored.EmailConfigs[0].Smarthost.String())
+	assert.NotContains(t, cfg.StoreableConfig().Config, "operator-secret")
+}
+
+func TestStaleStoredSMTPSettingsAreReplacedOnLoad(t *testing.T) {
+	stored := &StoreableConfig{
+		Config: `{"global":{"resolve_timeout":"5m","smtp_from":"old@example.com","smtp_hello":"localhost","smtp_smarthost":"email-smtp.us-east-1.amazonaws.com:587","smtp_auth_username":"old-user","smtp_auth_password":"old-secret","smtp_require_tls":true},"route":{"receiver":"default-receiver","group_by":["ruleId"],"routes":[{"receiver":"email-receiver","continue":true,"matchers":["ruleId=~\"-1\""]}],"group_wait":"30s","group_interval":"5m","repeat_interval":"4h"},"receivers":[{"name":"default-receiver"},{"name":"email-receiver","email_configs":[{"send_resolved":false,"to":"team@example.com","from":"old@example.com","hello":"localhost","smarthost":"email-smtp.us-east-1.amazonaws.com:587","auth_username":"old-user","auth_password":"old-secret","require_tls":true}]}]}`,
+		OrgID:  "1",
+	}
+
+	cfg, err := NewConfigFromStoreableConfig(stored)
+	require.NoError(t, err)
+
+	loaded, err := cfg.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Len(t, loaded.EmailConfigs, 1)
+	assert.Empty(t, loaded.EmailConfigs[0].Smarthost.String())
+	assert.Empty(t, string(loaded.EmailConfigs[0].AuthPassword))
+
+	require.NoError(t, cfg.SetGlobalConfig(newSMTPGlobalConfig()))
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "smtp.sendgrid.net:587", got.Smarthost.String())
+	assert.Equal(t, "operator-secret", string(got.AuthPassword))
+	assert.Equal(t, "alerts@example.com", got.From)
+
+	assert.NotContains(t, cfg.StoreableConfig().Config, "old-secret")
+	assert.NotContains(t, cfg.StoreableConfig().Config, "amazonaws.com")
+	assert.NotContains(t, cfg.StoreableConfig().Config, "operator-secret")
+}
+
+func TestCreateReceiverDoesNotMutateCaller(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Equal(t, "smtp.sendgrid.net:587", receiver.EmailConfigs[0].Smarthost.String())
+
+	throwaway, err := cfg.CopyWithReset()
+	require.NoError(t, err)
+	require.NoError(t, throwaway.CreateReceiver(receiver))
+
+	assert.Equal(t, "smtp.sendgrid.net:587", receiver.EmailConfigs[0].Smarthost.String())
+	assert.Equal(t, "operator-secret", string(receiver.EmailConfigs[0].AuthPassword))
+}
+
 // Round-trip: create → serialize → reload → GetReceiver still has the configs.
 func TestConfigPreservesGoogleChatConfigs(t *testing.T) {
 	webhookURL, err := url.Parse("https://chat.googleapis.com/v1/spaces/test/messages")

pkg/types/alertmanagertypes/receiver.go

@@ -37,6 +37,7 @@ func NewReceiver(input string) (*Receiver, error) {
 	if err != nil {
 		return nil, err
 	}
+	stripEmailTransport(withDefaults)
 	receiver.Receiver = withDefaults
 
 	// Extend this block when adding another native notifier type.
@@ -53,6 +54,23 @@ func NewReceiver(input string) (*Receiver, error) {
 	return receiver, nil
 }
 
+func stripEmailTransport(base *config.Receiver) {
+	for _, ec := range base.EmailConfigs {
+		ec.From = ""
+		ec.Hello = ""
+		ec.Smarthost = config.HostPort{}
+		ec.AuthUsername = ""
+		ec.AuthPassword = ""
+		ec.AuthPasswordFile = ""
+		ec.AuthSecret = ""
+		ec.AuthSecretFile = ""
+		ec.AuthIdentity = ""
+		ec.RequireTLS = nil
+		ec.TLSConfig = nil
+		ec.ForceImplicitTLS = nil
+	}
+}
+
 func defaultedBaseReceiver(base *config.Receiver) (*config.Receiver, error) {
 	bytes, err := yaml.Marshal(base)
 	if err != nil {
@@ -102,7 +120,12 @@ func TestReceiver(ctx context.Context, receiver *Receiver, receiverIntegrationsF
 		return err
 	}
 
-	defaultedReceiver, err := testConfig.GetReceiver(receiver.Name)
+	resolvedConfig, err := testConfig.Resolved()
+	if err != nil {
+		return err
+	}
+
+	defaultedReceiver, err := resolvedConfig.GetReceiver(receiver.Name)
 	if err != nil {
 		return err
 	}

pkg/types/alertmanagertypes/receiver_test.go

@@ -46,6 +46,31 @@ func TestNewReceiver(t *testing.T) {
 	}
 }
 
+func TestNewReceiverStripsEmailTransport(t *testing.T) {
+	receiver, err := NewReceiver(`{"name":"email","email_configs":[{"to":"team@example.com","from":"attacker@example.com","hello":"example.com","smarthost":"smtp.example.com:587","auth_username":"user","auth_password":"supersecret","auth_secret":"alsosecret","auth_identity":"id","require_tls":false,"headers":{"Subject":"custom"}}]}`)
+	require.NoError(t, err)
+	require.Len(t, receiver.EmailConfigs, 1)
+
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "team@example.com", got.To)
+	assert.Equal(t, map[string]string{"Subject": "custom"}, got.Headers)
+
+	assert.Empty(t, got.From)
+	assert.Empty(t, got.Hello)
+	assert.Empty(t, got.Smarthost.String())
+	assert.Empty(t, got.AuthUsername)
+	assert.Empty(t, string(got.AuthPassword))
+	assert.Empty(t, string(got.AuthSecret))
+	assert.Empty(t, got.AuthIdentity)
+	assert.Nil(t, got.RequireTLS)
+	assert.Nil(t, got.TLSConfig)
+
+	bytes, err := json.Marshal(receiver)
+	require.NoError(t, err)
+	assert.NotContains(t, string(bytes), "supersecret")
+	assert.NotContains(t, string(bytes), "smtp.example.com")
+}
+
 // Omitted fields fall back to DefaultGoogleChatReceiverConfig.
 func TestNewReceiverGoogleChatAppliesDefaults(t *testing.T) {
 	receiver, err := NewReceiver(`{"name":"googlechat","googlechat_configs":[{"webhook_url":"https://chat.googleapis.com/v1/spaces/test/messages"}]}`)

pkg/types/llmpricingruletypes/pricing.go

@@ -16,10 +16,18 @@ import (
 const (
 	LLMCostFeatureType agentConf.AgentFeatureType = "llm_pricing"
 
+	GenAIRequestModel                  = "gen_ai.request.model"
+	GenAIProviderName                  = "gen_ai.provider.name"
+	GenAIUsageInputTokens              = "gen_ai.usage.input_tokens"
+	GenAIUsageOutputTokens             = "gen_ai.usage.output_tokens"
+	GenAIUsageCacheReadInputTokens     = "gen_ai.usage.cache_read.input_tokens"
+	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
+
 	SignozGenAICostInput      = "_signoz.gen_ai.cost_input"
 	SignozGenAICostOutput     = "_signoz.gen_ai.cost_output"
 	SignozGenAICostCacheRead  = "_signoz.gen_ai.cost_cache_read"
 	SignozGenAICostCacheWrite = "_signoz.gen_ai.cost_cache_write"
+	SignozGenAITotalCost      = "_signoz.gen_ai.total_cost"
 )
 
 var (

pkg/types/llmpricingruletypes/processorconfig.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 
 	"github.com/SigNoz/signoz/pkg/errors"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"gopkg.in/yaml.v3"
 )
 
@@ -84,11 +83,11 @@ func buildProcessorConfig(rules []*LLMPricingRule) *LLMPricingRuleProcessorConfi
 
 	return &LLMPricingRuleProcessorConfig{
 		Attrs: LLMPricingRuleProcessorAttrs{
-			Model:      telemetrytypes.GenAIRequestModel,
-			In:         telemetrytypes.GenAIUsageInputTokens,
-			Out:        telemetrytypes.GenAIUsageOutputTokens,
-			CacheRead:  telemetrytypes.GenAIUsageCacheReadInputTokens,
-			CacheWrite: telemetrytypes.GenAIUsageCacheCreationInputTokens,
+			Model:      GenAIRequestModel,
+			In:         GenAIUsageInputTokens,
+			Out:        GenAIUsageOutputTokens,
+			CacheRead:  GenAIUsageCacheReadInputTokens,
+			CacheWrite: GenAIUsageCacheCreationInputTokens,
 		},
 		DefaultPricing: LLMPricingRuleProcessorDefaultPricing{
 			Rules: pricingRules,
@@ -98,7 +97,7 @@ func buildProcessorConfig(rules []*LLMPricingRule) *LLMPricingRuleProcessorConfi
 			Out:        SignozGenAICostOutput,
 			CacheRead:  SignozGenAICostCacheRead,
 			CacheWrite: SignozGenAICostCacheWrite,
-			Total:      telemetrytypes.SignozGenAITotalCost,
+			Total:      SignozGenAITotalCost,
 		},
 	}
 }

pkg/types/querybuildertypes/querybuildertypesv5/query_type.go

@@ -9,7 +9,6 @@ type QueryType struct {
 var (
 	QueryTypeUnknown       = QueryType{valuer.NewString("unknown")}
 	QueryTypeBuilder       = QueryType{valuer.NewString("builder_query")}
-	QueryTypeBuilderAI     = QueryType{valuer.NewString("builder_ai_query")}
 	QueryTypeFormula       = QueryType{valuer.NewString("builder_formula")}
 	QueryTypeSubQuery      = QueryType{valuer.NewString("builder_sub_query")}
 	QueryTypeJoin          = QueryType{valuer.NewString("builder_join")}
@@ -22,7 +21,6 @@ var (
 func (QueryType) Enum() []any {
 	return []any{
 		QueryTypeBuilder,
-		QueryTypeBuilderAI,
 		QueryTypeFormula,
 		// Not yet supported.
 		// QueryTypeSubQuery,

pkg/types/querybuildertypes/querybuildertypesv5/req.go

@@ -62,13 +62,6 @@ type queryEnvelopeBuilder struct {
 	Spec builderQuerySpec `json:"spec" description:"The builder query specification."`
 }
 
-// queryEnvelopeBuilderAI is the OpenAPI schema for a builder_ai_query QueryEnvelope.
-// The spec is always a traces builder query (the signal is implied by the type).
-type queryEnvelopeBuilderAI struct {
-	Type QueryType                           `json:"type" required:"true" description:"The type of the query."`
-	Spec QueryBuilderQuery[TraceAggregation] `json:"spec" description:"The AI builder query specification."`
-}
-
 // queryEnvelopeFormula is the OpenAPI schema for a QueryEnvelope with type=builder_formula.
 type queryEnvelopeFormula struct {
 	Type QueryType           `json:"type" required:"true" description:"The type of the query."`
@@ -107,7 +100,6 @@ var _ jsonschema.OneOfExposer = QueryEnvelope{}
 func (QueryEnvelope) JSONSchemaOneOf() []any {
 	return []any{
 		queryEnvelopeBuilder{},
-		queryEnvelopeBuilderAI{},
 		queryEnvelopeFormula{},
 		// queryEnvelopeJoin{}, // deferred — see commented queryEnvelopeJoin above
 		queryEnvelopeTraceOperator{},
@@ -128,7 +120,6 @@ func (QueryEnvelope) PrepareJSONSchema(s *jsonschema.Schema) error {
 		"propertyName": "type",
 		"mapping": map[string]string{
 			QueryTypeBuilder.StringValue():       "#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilder",
-			QueryTypeBuilderAI.StringValue():     "#/components/schemas/Querybuildertypesv5QueryEnvelopeBuilderAI",
 			QueryTypeFormula.StringValue():       "#/components/schemas/Querybuildertypesv5QueryEnvelopeFormula",
 			QueryTypeTraceOperator.StringValue(): "#/components/schemas/Querybuildertypesv5QueryEnvelopeTraceOperator",
 			QueryTypePromQL.StringValue():        "#/components/schemas/Querybuildertypesv5QueryEnvelopePromQL",
@@ -159,16 +150,6 @@ func (q *QueryEnvelope) UnmarshalJSON(data []byte) error {
 		}
 		q.Spec = spec
 
-	case QueryTypeBuilderAI:
-		// A dedicated AI query is always a traces builder query; the signal is
-		// implied by the type, so pin it rather than requiring the caller to send it.
-		var spec QueryBuilderQuery[TraceAggregation]
-		if err := json.Unmarshal(shadow.Spec, &spec); err != nil {
-			return err
-		}
-		spec.Signal = telemetrytypes.SignalTraces
-		q.Spec = spec
-
 	case QueryTypeFormula:
 		var spec QueryBuilderFormula
 		if err := json.Unmarshal(shadow.Spec, &spec); err != nil {
@@ -213,7 +194,7 @@ func (q *QueryEnvelope) UnmarshalJSON(data []byte) error {
 			"unknown query type %q",
 			shadow.Type,
 		).WithAdditional(
-			"Valid query types are: builder_query, builder_ai_query, builder_sub_query, builder_formula, builder_join, builder_trace_operator, promql, clickhouse_sql",
+			"Valid query types are: builder_query, builder_sub_query, builder_formula, builder_join, builder_trace_operator, promql, clickhouse_sql",
 		).WithSuggestions(errors.NewValidReferences(errors.NounQueryTypes, QueryType{}.Enum()...))
 	}
 

pkg/types/querybuildertypes/querybuildertypesv5/resp.go

@@ -492,6 +492,75 @@ func (t TimeSeriesValue) MarshalJSON() ([]byte, error) {
 	})
 }
 
+// UnmarshalJSON inverts MarshalJSON, which renders non-finite floats as the
+// strings "NaN"/"Inf"/"-Inf". The bucket cache serializes through this type,
+// so a value it cannot read back costs it the whole cached entry.
+func (t *TimeSeriesValue) UnmarshalJSON(data []byte) error {
+	type Alias TimeSeriesValue
+
+	aux := &struct {
+		*Alias
+		Value  json.RawMessage   `json:"value"`
+		Values []json.RawMessage `json:"values,omitempty"`
+	}{
+		Alias: (*Alias)(t),
+	}
+	if err := json.Unmarshal(data, aux); err != nil {
+		return err
+	}
+
+	var err error
+	if t.Value, err = parseFloatOrNonFinite(aux.Value); err != nil {
+		return err
+	}
+	t.Values, err = parseFloatsOrNonFinite(aux.Values)
+	return err
+}
+
+// parseFloatOrNonFinite inverts sanitizeValue for one float: a JSON number, or
+// a sentinel string standing in for a value JSON cannot represent.
+func parseFloatOrNonFinite(raw json.RawMessage) (float64, error) {
+	if len(raw) == 0 || string(raw) == "null" {
+		return 0, nil
+	}
+
+	var f float64
+	if err := json.Unmarshal(raw, &f); err == nil {
+		return f, nil
+	}
+
+	var s string
+	if err := json.Unmarshal(raw, &s); err != nil {
+		return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "value %s is neither a number nor a non-finite sentinel", raw)
+	}
+	switch s {
+	case "NaN":
+		return math.NaN(), nil
+	case "Inf", "+Inf":
+		return math.Inf(1), nil
+	case "-Inf":
+		return math.Inf(-1), nil
+	default:
+		return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "unrecognized non-finite value %q", s)
+	}
+}
+
+// parseFloatsOrNonFinite does the same for Values, keeping the nil/empty
+// distinction MarshalJSON preserves.
+func parseFloatsOrNonFinite(raw []json.RawMessage) ([]float64, error) {
+	if raw == nil {
+		return nil, nil
+	}
+	values := make([]float64, len(raw))
+	for idx := range raw {
+		var err error
+		if values[idx], err = parseFloatOrNonFinite(raw[idx]); err != nil {
+			return nil, err
+		}
+	}
+	return values, nil
+}
+
 func (r RawData) MarshalJSON() ([]byte, error) {
 	type Alias RawData
 	return json.Marshal((*Alias)(&r))

pkg/types/querybuildertypes/querybuildertypesv5/resp_test.go

@@ -428,3 +428,111 @@ func TestRoundToNonZeroDecimals(t *testing.T) {
 	assert.Equal(t, math.Inf(1), roundToNonZeroDecimals(math.Inf(1), 3))
 	assert.Equal(t, math.Inf(-1), roundToNonZeroDecimals(math.Inf(-1), 3))
 }
+
+func TestTimeSeriesValueUnmarshalJSONNonFinite(t *testing.T) {
+	cases := []struct {
+		description   string
+		encoded       string
+		expectedValue float64
+		expectError   bool
+	}{
+		{
+			description:   "finite value decodes as a number",
+			encoded:       `{"timestamp":1000,"value":1.5}`,
+			expectedValue: 1.5,
+		},
+		{
+			description:   "NaN sentinel decodes back to NaN",
+			encoded:       `{"timestamp":1000,"value":"NaN"}`,
+			expectedValue: math.NaN(),
+		},
+		{
+			description:   "Inf sentinel decodes back to positive infinity",
+			encoded:       `{"timestamp":1000,"value":"Inf"}`,
+			expectedValue: math.Inf(1),
+		},
+		{
+			description:   "negative Inf sentinel decodes back to negative infinity",
+			encoded:       `{"timestamp":1000,"value":"-Inf"}`,
+			expectedValue: math.Inf(-1),
+		},
+		{
+			description:   "null decodes to zero",
+			encoded:       `{"timestamp":1000,"value":null}`,
+			expectedValue: 0,
+		},
+		{
+			description: "an unrecognized string is still an error",
+			encoded:     `{"timestamp":1000,"value":"banana"}`,
+			expectError: true,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.description, func(t *testing.T) {
+			var got TimeSeriesValue
+			err := json.Unmarshal([]byte(c.encoded), &got)
+			if c.expectError {
+				assert.Error(t, err)
+				return
+			}
+			assert.NoError(t, err)
+			assert.EqualValues(t, 1000, got.Timestamp)
+			if math.IsNaN(c.expectedValue) {
+				assert.True(t, math.IsNaN(got.Value))
+				return
+			}
+			assert.Equal(t, c.expectedValue, got.Value)
+		})
+	}
+}
+
+func TestTimeSeriesValueRoundTripsNonFiniteValues(t *testing.T) {
+	original := &TimeSeries{
+		Labels: []*Label{
+			{Key: telemetrytypes.TelemetryFieldKey{Name: "job_name"}, Value: "dbBloatMonitorJob"},
+		},
+		Values: []*TimeSeriesValue{
+			{Timestamp: 1000, Value: 11.524},
+			{Timestamp: 2000, Value: math.NaN()},
+			{Timestamp: 3000, Value: math.Inf(1)},
+			{Timestamp: 4000, Value: math.Inf(-1)},
+			{Timestamp: 5000, Value: 456.7},
+		},
+	}
+
+	encoded, err := json.Marshal(original)
+	assert.NoError(t, err)
+
+	var decoded *TimeSeries
+	err = json.Unmarshal(encoded, &decoded)
+	assert.NoError(t, err, "a response containing non-finite values must survive the round trip")
+	assert.Len(t, decoded.Values, 5)
+	assert.Equal(t, 11.524, decoded.Values[0].Value)
+	assert.True(t, math.IsNaN(decoded.Values[1].Value))
+	assert.True(t, math.IsInf(decoded.Values[2].Value, 1))
+	assert.True(t, math.IsInf(decoded.Values[3].Value, -1))
+	assert.Equal(t, 456.7, decoded.Values[4].Value)
+}
+
+func TestTimeSeriesValueRoundTripsHeatmapValues(t *testing.T) {
+	original := &TimeSeriesValue{
+		Timestamp: 1000,
+		Value:     2.5,
+		Values:    []float64{1.5, math.NaN(), 3.5},
+		Bucket:    &Bucket{Step: 10},
+	}
+
+	encoded, err := json.Marshal(original)
+	assert.NoError(t, err)
+
+	var decoded TimeSeriesValue
+	err = json.Unmarshal(encoded, &decoded)
+	assert.NoError(t, err)
+	assert.Equal(t, 2.5, decoded.Value)
+	assert.Len(t, decoded.Values, 3)
+	assert.Equal(t, 1.5, decoded.Values[0])
+	assert.True(t, math.IsNaN(decoded.Values[1]))
+	assert.Equal(t, 3.5, decoded.Values[2])
+	assert.Equal(t, float64(10), decoded.Bucket.Step)
+}

pkg/types/querybuildertypes/querybuildertypesv5/validation.go

@@ -641,7 +641,7 @@ func (r *QueryRangeRequest) ValidateRequestScope() ([]ValidationOption, error) {
 	// Builder query names must be unique across the composite query.
 	queryNames := make(map[string]bool)
 	for _, envelope := range r.CompositeQuery.Queries {
-		if envelope.Type == QueryTypeBuilder || envelope.Type == QueryTypeSubQuery || envelope.Type == QueryTypeBuilderAI {
+		if envelope.Type == QueryTypeBuilder || envelope.Type == QueryTypeSubQuery {
 			name := envelope.GetQueryName()
 			if name != "" {
 				if queryNames[name] {
@@ -710,7 +710,7 @@ func (c *CompositeQuery) Validate(opts ...ValidationOption) error {
 		}
 
 		// Check name uniqueness for builder queries
-		if envelope.Type == QueryTypeBuilder || envelope.Type == QueryTypeSubQuery || envelope.Type == QueryTypeBuilderAI {
+		if envelope.Type == QueryTypeBuilder || envelope.Type == QueryTypeSubQuery {
 			name := envelope.GetQueryName()
 			if name != "" {
 				if queryNames[name] {
@@ -744,15 +744,6 @@ func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) err
 				"unknown query spec type",
 			)
 		}
-	case QueryTypeBuilderAI:
-		spec, ok := envelope.Spec.(QueryBuilderQuery[TraceAggregation])
-		if !ok {
-			return errors.NewInvalidInputf(
-				errors.CodeInvalidInput,
-				"invalid AI builder query spec",
-			)
-		}
-		return spec.Validate(opts...)
 	case QueryTypeFormula:
 		spec, ok := envelope.Spec.(QueryBuilderFormula)
 		if !ok {
@@ -822,7 +813,7 @@ func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) err
 			"unknown query type: %s",
 			envelope.Type,
 		).WithAdditional(
-			"Valid query types are: builder_query, builder_ai_query, builder_sub_query, builder_formula, builder_join, promql, clickhouse_sql, trace_operator",
+			"Valid query types are: builder_query, builder_sub_query, builder_formula, builder_join, promql, clickhouse_sql, trace_operator",
 		).WithSuggestions(errors.NewValidReferences(errors.NounQueryTypes, QueryType{}.Enum()...))
 	}
 }

pkg/types/telemetrytypes/field_context.go

@@ -76,7 +76,6 @@ var (
 		"log":        FieldContextLog,
 		"metric":     FieldContextMetric,
 		"tracefield": FieldContextTrace,
-		"trace":      FieldContextTrace,
 	}
 )
 

pkg/types/telemetrytypes/genai_semconv.go

@@ -1,43 +0,0 @@
-package telemetrytypes
-
-// OpenTelemetry gen_ai semantic-convention attribute keys. Single source of truth
-// shared by the AI query builder and the LLM pricing pipeline.
-const (
-	GenAIRequestModel = "gen_ai.request.model"
-	GenAIToolName     = "gen_ai.tool.name"
-	GenAIAgentName    = "gen_ai.agent.name"
-	GenAIProviderName = "gen_ai.provider.name"
-
-	GenAIUsageInputTokens              = "gen_ai.usage.input_tokens"
-	GenAIUsageOutputTokens             = "gen_ai.usage.output_tokens"
-	GenAIUsageCacheReadInputTokens     = "gen_ai.usage.cache_read.input_tokens"
-	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
-
-	GenAIInputMessages  = "gen_ai.input.messages"
-	GenAIOutputMessages = "gen_ai.output.messages"
-
-	// SignozGenAITotalCost is not OTel semconv: it is the per-span total cost the
-	// SigNoz LLM pricing processor computes and attaches (see llmpricingruletypes).
-	SignozGenAITotalCost = "_signoz.gen_ai.total_cost"
-)
-
-// GenAIFieldDefinitions are the gen_ai semantic-convention span attributes the AI
-// query builder relies on. They are surfaced by the metadata store for trace
-// queries regardless of whether they have been ingested yet, so the AI gate/columns
-// resolve on a fresh install (mirrors intrinsic metric keys). String keys are the
-// gate; the usage keys are numeric.
-var GenAIFieldDefinitions = map[string]TelemetryFieldKey{
-	GenAIRequestModel: {Name: GenAIRequestModel, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIToolName:     {Name: GenAIToolName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIAgentName:    {Name: GenAIAgentName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIProviderName: {Name: GenAIProviderName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-
-	GenAIUsageInputTokens:              {Name: GenAIUsageInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageOutputTokens:             {Name: GenAIUsageOutputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageCacheReadInputTokens:     {Name: GenAIUsageCacheReadInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageCacheCreationInputTokens: {Name: GenAIUsageCacheCreationInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	SignozGenAITotalCost:               {Name: SignozGenAITotalCost, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-
-	GenAIInputMessages:  {Name: GenAIInputMessages, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIOutputMessages: {Name: GenAIOutputMessages, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-}

pkg/types/telemetrytypes/selector.go

@@ -10,7 +10,6 @@ const wildcardSelector = "*"
 
 var telemetryGrantQueryTypes = map[string]bool{
 	"builder_query":     true,
-	"builder_ai_query":  true,
 	"builder_sub_query": true,
 	"promql":            false,
 	"clickhouse_sql":    false,

tests/conftest.py

@@ -24,6 +24,7 @@ pytest_plugins = [
     "fixtures.keycloak",
     "fixtures.idp",
     "fixtures.notification_channel",
+    "fixtures.maildev",
     "fixtures.alerts",
     "fixtures.cloudintegrations",
     "fixtures.jsontypes",

tests/fixtures/alerts.py

@@ -1,11 +1,13 @@
 import base64
 import json
+import re
 import time
 import urllib.parse
 import uuid
 from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
+from urllib.parse import urlparse
 
 import pytest
 import requests
@@ -15,6 +17,7 @@ from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.fs import get_testdata_file_path
 from fixtures.logger import setup_logger
 from fixtures.logs import Logs
+from fixtures.maildev import get_all_mails, verify_email_received
 from fixtures.metrics import Metrics
 from fixtures.traces import Traces
 
@@ -311,3 +314,145 @@ def update_rule_channel_name(rule_data: dict, channel_name: str):
         # loop over all the sepcs and update the channels
         for spec in thresholds["spec"]:
             spec["channels"] = [channel_name]
+
+
+def _is_json_subset(subset, superset) -> bool:
+    """Check if subset is contained within superset recursively.
+    - For dicts: all keys in subset must exist in superset with matching values
+    - For lists: all items in subset must be present in superset
+    - For scalars: exact equality
+    """
+    if isinstance(subset, dict):
+        if not isinstance(superset, dict):
+            return False
+        return all(key in superset and _is_json_subset(value, superset[key]) for key, value in subset.items())
+    if isinstance(subset, list):
+        if not isinstance(superset, list):
+            return False
+        return all(any(_is_json_subset(sub_item, sup_item) for sup_item in superset) for sub_item in subset)
+    if isinstance(subset, re.Pattern):
+        return isinstance(superset, str) and subset.search(superset) is not None
+    return subset == superset
+
+
+def verify_webhook_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    validation_data: dict,
+) -> bool:
+    """Check if wiremock received a request at the given path
+    whose JSON body is a superset of the expected json_body."""
+    path = validation_data["path"]
+    json_body = validation_data["json_body"]
+
+    url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+    try:
+        res = requests.post(url, json={"method": "POST", "url": path}, timeout=10)
+    except requests.exceptions.RequestException:
+        return False
+    if res.status_code != HTTPStatus.OK:
+        return False
+
+    for req in res.json()["requests"]:
+        body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+        if _is_json_subset(json_body, body):
+            return True
+    return False
+
+
+def _check_notification_validation(
+    validation: types.NotificationValidation,
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+) -> bool:
+    """Dispatch a single validation check to the appropriate verifier."""
+    if validation.destination_type == "webhook":
+        return verify_webhook_notification_expectation(notification_channel, validation.validation_data)
+    if validation.destination_type == "email":
+        return verify_email_received(maildev, validation.validation_data)
+    raise ValueError(f"Invalid destination type: {validation.destination_type}")
+
+
+def verify_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    expected_notification: types.AMNotificationExpectation,
+) -> bool:
+    """Poll for expected notifications across webhook and email channels."""
+    time_to_wait = datetime.now() + timedelta(seconds=expected_notification.wait_time_seconds)
+
+    while datetime.now() < time_to_wait:
+        all_found = all(_check_notification_validation(v, notification_channel, maildev) for v in expected_notification.notification_validations)
+
+        if expected_notification.should_notify and all_found:
+            logger.info("All expected notifications found")
+            return True
+
+        time.sleep(1)
+
+    # Timeout reached
+    if not expected_notification.should_notify:
+        # Verify no notifications were received
+        for validation in expected_notification.notification_validations:
+            found = _check_notification_validation(validation, notification_channel, maildev)
+            assert not found, f"Expected no notification but found one for {validation.destination_type} with data {validation.validation_data}"
+        logger.info("No notifications found, as expected")
+        return True
+
+    missing = [v for v in expected_notification.notification_validations if not _check_notification_validation(v, notification_channel, maildev)]
+    assert len(missing) == 0, f"Expected all notifications to be found but missing: {missing}, received: {_received_notifications(notification_channel, maildev, missing)}"
+    return True
+
+
+def _received_notifications(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    missing: list[types.NotificationValidation],
+) -> dict:
+    received = {}
+    if any(v.destination_type == "webhook" for v in missing):
+        webhook_bodies = []
+        for validation in missing:
+            if validation.destination_type != "webhook":
+                continue
+            url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+            try:
+                res = requests.post(url, json={"method": "POST", "url": validation.validation_data["path"]}, timeout=10)
+                webhook_bodies.extend(json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8")) for req in res.json()["requests"])
+            except requests.exceptions.RequestException as exc:
+                webhook_bodies.append(f"<failed to fetch wiremock journal: {exc}>")
+        received["webhook"] = webhook_bodies
+    if any(v.destination_type == "email" for v in missing):
+        received["email"] = get_all_mails(maildev)
+    return received
+
+
+def update_raw_channel_config(
+    channel_config: dict,
+    channel_name: str,
+    notification_channel: types.TestContainerDocker,
+) -> dict:
+    """
+    Updates the channel config to point to the given wiremock
+    notification_channel container to receive notifications.
+    """
+    config = channel_config.copy()
+
+    config["name"] = channel_name
+
+    url_field_map = {
+        "slack_configs": "api_url",
+        "msteamsv2_configs": "webhook_url",
+        "webhook_configs": "url",
+        "pagerduty_configs": "url",
+        "opsgenie_configs": "api_url",
+    }
+
+    for config_key, url_field in url_field_map.items():
+        if config_key in config:
+            for entry in config[config_key]:
+                if url_field in entry:
+                    original_url = entry[url_field]
+                    path = urlparse(original_url).path
+                    entry[url_field] = notification_channel.container_configs["8080"].get(path)
+
+    return config

tests/fixtures/auth.py

@@ -77,7 +77,7 @@ def register_admin(
             timeout=5,
         )
 
-        assert response.status_code == HTTPStatus.OK
+        assert response.status_code == HTTPStatus.OK, f"failed to register admin: {response.status_code} {response.text}"
 
         return types.Operation(name="create_user_admin")
 

tests/fixtures/http.py

@@ -125,13 +125,19 @@ def gateway(
 
 @pytest.fixture(name="make_http_mocks", scope="function")
 def make_http_mocks() -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
+    mocked_containers = []
+
     def _make_http_mocks(container: types.TestContainerDocker, mappings: list[Mapping]) -> None:
         Config.base_url = container.host_configs["8080"].get("/__admin")
 
         for mapping in mappings:
             Mappings.create_mapping(mapping=mapping)
 
+        mocked_containers.append(container)
+
     yield _make_http_mocks
 
-    Mappings.delete_all_mappings()
-    Requests.reset_request_journal()
+    for container in mocked_containers:
+        Config.base_url = container.host_configs["8080"].get("/__admin")
+        Mappings.delete_all_mappings()
+        Requests.reset_request_journal()

tests/fixtures/maildev.py

@@ -0,0 +1,143 @@
+import re
+from http import HTTPStatus
+
+import docker
+import docker.errors
+import pytest
+import requests
+from testcontainers.core.container import DockerContainer, Network
+
+from fixtures import reuse, types
+from fixtures.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+MAILDEV_INCOMING_USER = "apikey"
+MAILDEV_INCOMING_PASS = "integration-smtp-secret"
+
+SMTP_TEST_FROM = "alertmanager@integration.test"
+
+OLD_PROVIDER_SMTP_PASS = "old-provider-smtp-secret"
+NEW_PROVIDER_SMTP_PASS = "new-provider-smtp-secret"
+
+
+def signoz_smtp_env(maildev: "types.TestContainerDocker", password: str = MAILDEV_INCOMING_PASS) -> dict:
+    return {
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__SMARTHOST": f"{maildev.container_configs['1025'].address}:{maildev.container_configs['1025'].port}",
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__FROM": SMTP_TEST_FROM,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__AUTH__USERNAME": MAILDEV_INCOMING_USER,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__AUTH__PASSWORD": password,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__REQUIRE__TLS": "false",
+    }
+
+
+def create_maildev(
+    network: Network,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    cache_key: str = "maildev",
+    incoming_user: str = MAILDEV_INCOMING_USER,
+    incoming_pass: str = MAILDEV_INCOMING_PASS,
+) -> types.TestContainerDocker:
+    def create() -> types.TestContainerDocker:
+        container = DockerContainer(image="maildev/maildev:2.2.1")
+        container.with_env("MAILDEV_INCOMING_USER", incoming_user)
+        container.with_env("MAILDEV_INCOMING_PASS", incoming_pass)
+        container.with_exposed_ports(1025, 1080)
+        container.with_network(network=network)
+        container.start()
+
+        return types.TestContainerDocker(
+            id=container.get_wrapped_container().id,
+            host_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1025),
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1080),
+                ),
+            },
+            container_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_wrapped_container().name,
+                    port=1025,
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_wrapped_container().name,
+                    port=1080,
+                ),
+            },
+        )
+
+    def delete(container: types.TestContainerDocker):
+        client = docker.from_env()
+        try:
+            client.containers.get(container_id=container.id).stop()
+            client.containers.get(container_id=container.id).remove(v=True)
+        except docker.errors.NotFound:
+            logger.info(
+                "Skipping removal of MailDev, MailDev(%s) not found. Maybe it was manually removed?",
+                {"id": container.id},
+            )
+
+    def restore(cache: dict) -> types.TestContainerDocker:
+        return types.TestContainerDocker.from_cache(cache)
+
+    return reuse.wrap(
+        request,
+        pytestconfig,
+        cache_key,
+        lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
+        create,
+        delete,
+        restore,
+    )
+
+
+@pytest.fixture(name="maildev", scope="package")
+def maildev(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig)
+
+
+def get_all_mails(_maildev: types.TestContainerDocker) -> list[dict]:
+    url = _maildev.host_configs["1080"].get("/email")
+    response = requests.get(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to fetch emails from MailDev, status code: {response.status_code}, response: {response.text}"
+
+    def addresses(entries: list[dict]) -> str:
+        return ",".join(sorted(entry.get("address", "") for entry in entries))
+
+    return [
+        {
+            "subject": email.get("subject", ""),
+            "html": email.get("html", ""),
+            "text": email.get("text", ""),
+            "from": addresses(email.get("from", [])),
+            "to": addresses(email.get("to", [])),
+        }
+        for email in response.json()
+    ]
+
+
+def verify_email_received(_maildev: types.TestContainerDocker, filters: dict) -> bool:
+    def matches(expected, actual: str) -> bool:
+        if isinstance(expected, re.Pattern):
+            return expected.search(actual) is not None
+        return expected == actual
+
+    for email in get_all_mails(_maildev):
+        if all(key in email and matches(filter_value, email[key]) for key, filter_value in filters.items()):
+            return True
+    return False
+
+
+def delete_all_mails(_maildev: types.TestContainerDocker) -> None:
+    url = _maildev.host_configs["1080"].get("/email/all")
+    response = requests.delete(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to delete emails from MailDev, status code: {response.status_code}, response: {response.text}"

tests/fixtures/notification_channel.py

@@ -1,3 +1,6 @@
+# pylint: disable=line-too-long
+import json
+import time
 from collections.abc import Callable
 from http import HTTPStatus
 
@@ -11,10 +14,116 @@ from wiremock.testing.testcontainer import WireMockContainer
 from fixtures import reuse, types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.logger import setup_logger
+from fixtures.maildev import MAILDEV_INCOMING_PASS, SMTP_TEST_FROM
 
 logger = setup_logger(__name__)
 
 
+EMAIL_TRANSPORT_KEYS = [
+    "from",
+    "hello",
+    "smarthost",
+    "auth_username",
+    "auth_password",
+    "auth_password_file",
+    "auth_secret",
+    "auth_secret_file",
+    "auth_identity",
+    "require_tls",
+    "tls_config",
+    "force_implicit_tls",
+]
+
+
+def assert_email_channel_payload_clean(payload: str) -> None:
+    receiver = json.loads(payload)
+    for email_config in receiver["email_configs"]:
+        transport_keys = set(email_config.keys()) & set(EMAIL_TRANSPORT_KEYS)
+        transport_keys -= {"smarthost"} if email_config.get("smarthost", "") == "" else set()
+        assert not transport_keys, f"email channel payload carries transport keys {transport_keys}: {payload}"
+
+    assert MAILDEV_INCOMING_PASS not in payload
+    assert SMTP_TEST_FROM not in payload
+
+
+"""
+Default notification channel configs shared across alertmanager tests.
+"""
+slack_default_config = {
+    # channel name configured on runtime
+    "slack_configs": [
+        {
+            "api_url": "services/TEAM_ID/BOT_ID/TOKEN_ID",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+msteams_default_config = {
+    "msteamsv2_configs": [
+        {
+            "webhook_url": "msteams/webhook_url",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+pagerduty_default_config = {
+    "pagerduty_configs": [
+        {
+            "routing_key": "PagerDutyRoutingKey",
+            "url": "v2/enqueue",  # base_url configured on runtime
+            "client": "SigNoz Alert Manager",
+            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+            "description": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n\t{{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n\t {{" "}}(\n\t {{- with .CommonLabels.Remove .GroupLabels.Names }}\n\t\t{{- range $index, $label := .SortedPairs -}}\n\t\t {{ if $index }}, {{ end }}\n\t\t {{- $label.Name }}="{{ $label.Value -}}"\n\t\t{{- end }}\n\t {{- end -}}\n\t )\n\t{{- end }}',
+            "details": {
+                "firing": '{{ template "pagerduty.default.instances" .Alerts.Firing }}',
+                "num_firing": "{{ .Alerts.Firing | len }}",
+                "num_resolved": "{{ .Alerts.Resolved | len }}",
+                "resolved": '{{ template "pagerduty.default.instances" .Alerts.Resolved }}',
+            },
+            "source": "SigNoz Alert Manager",
+            "severity": "{{ (index .Alerts 0).Labels.severity }}",
+        }
+    ],
+}
+
+opsgenie_default_config = {
+    "opsgenie_configs": [
+        {
+            "api_key": "OpsGenieAPIKey",
+            "api_url": "/",  # base_url configured on runtime
+            "description": '{{ if gt (len .Alerts.Firing) 0 -}}\r\n\tAlerts Firing:\r\n\t{{ range .Alerts.Firing }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n\tAlerts Resolved:\r\n\t{{ range .Alerts.Resolved }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}',
+            "priority": '{{ if eq (index .Alerts 0).Labels.severity "critical" }}P1{{ else if eq (index .Alerts 0).Labels.severity "warning" }}P2{{ else if eq (index .Alerts 0).Labels.severity "info" }}P3{{ else }}P4{{ end }}',
+            "message": "{{ .CommonLabels.alertname }}",
+            "details": {},
+        }
+    ],
+}
+
+webhook_default_config = {
+    "webhook_configs": [
+        {
+            "url": "webhook/webhook_url",  # base_url configured on runtime
+        }
+    ],
+}
+
+email_default_config = {
+    "email_configs": [
+        {
+            "to": "test@example.com",
+            "html": '<html><body>{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}</body></html>',
+            "headers": {
+                "Subject": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}'
+            },
+        }
+    ],
+}
+
+
 @pytest.fixture(name="notification_channel", scope="package")
 def notification_channel(
     network: Network,
@@ -67,6 +176,40 @@ def notification_channel(
     )
 
 
+@pytest.fixture(name="create_notification_channel", scope="function")
+def create_notification_channel(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> Callable[[dict], str]:
+    admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    channel_ids = []
+
+    def _create_notification_channel(channel_config: dict) -> str:
+        response = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels"),
+            json=channel_config,
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        assert response.status_code == HTTPStatus.CREATED, f"Failed to create channel, Response: {response.text} Response status: {response.status_code}"
+        channel_id = response.json()["data"]["id"]
+        channel_ids.append(channel_id)
+        return channel_id
+
+    yield _create_notification_channel
+
+    for channel_id in channel_ids:
+        response = requests.delete(
+            signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"),
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        if response.status_code != HTTPStatus.NO_CONTENT:
+            logger.error("Failed to delete channel: %s", {"channel_id": channel_id, "status": response.status_code, "response": response.text})
+
+
 @pytest.fixture(name="create_webhook_notification_channel", scope="function")
 def create_webhook_notification_channel(
     signoz: types.SigNoz,
@@ -103,3 +246,19 @@ def create_webhook_notification_channel(
         return channel_id
 
     return _create_webhook_notification_channel
+
+
+def send_test_notification(signoz: types.SigNoz, token: str, receiver: dict, wait_seconds: int = 90) -> None:
+    deadline = time.time() + wait_seconds
+    last = None
+    while time.time() < deadline:
+        last = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels/test"),
+            json=receiver,
+            headers={"Authorization": f"Bearer {token}"},
+            timeout=30,
+        )
+        if last.status_code == HTTPStatus.NO_CONTENT:
+            return
+        time.sleep(2)
+    raise AssertionError(f"test notification did not succeed within {wait_seconds}s, last response: {last.status_code} {last.text}")

tests/fixtures/querier.py

@@ -78,15 +78,12 @@ class BuilderQuery:
     signal: str
     name: str = "A"
     source: str | None = None
-    query_type: str = "builder_query"
     limit: int | None = None
     offset: int | None = None
     filter_expression: str | None = None
-    having_expression: str | None = None
     select_fields: list[TelemetryFieldKey] | None = None
     order: list[OrderBy] | None = None
     aggregations: list[Aggregation | MetricAggregation] | None = None
-    group_by: list[TelemetryFieldKey] | None = None
     step_interval: int | None = None
 
     def to_dict(self) -> dict:
@@ -102,20 +99,16 @@ class BuilderQuery:
             spec["offset"] = self.offset
         if self.filter_expression:
             spec["filter"] = {"expression": self.filter_expression}
-        if self.having_expression:
-            spec["having"] = {"expression": self.having_expression}
         if self.select_fields:
             spec["selectFields"] = [f.to_dict() for f in self.select_fields]
         if self.order:
             spec["order"] = [o.to_dict() if hasattr(o, "to_dict") else o for o in self.order]
         if self.aggregations:
             spec["aggregations"] = [agg.to_dict() if hasattr(agg, "to_dict") else agg for agg in self.aggregations]
-        if self.group_by:
-            spec["groupBy"] = [k.to_dict() for k in self.group_by]
         if self.step_interval is not None:
             spec["stepInterval"] = self.step_interval
 
-        return {"type": self.query_type, "spec": spec}
+        return {"type": "builder_query", "spec": spec}
 
 
 @dataclass

tests/fixtures/querierai.py

@@ -1,129 +0,0 @@
-"""
-Trace builders for the querierai suite (query_type="builder_ai_query").
-
-Every builder pins its spans a few seconds before the given `now` so
-`query_window(now)` covers them, and tags them with the caller's service name so
-tests do not interfere with each other's data.
-"""
-
-from datetime import datetime, timedelta
-
-from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
-
-
-def query_window(now: datetime) -> tuple[int, int]:
-    """[now-10min, now+1min) in epoch millis — wide enough for every builder here."""
-    return (
-        int((now - timedelta(minutes=10)).timestamp() * 1000),
-        int((now + timedelta(minutes=1)).timestamp() * 1000),
-    )
-
-
-def root_span(*, now: datetime, trace_id: str, span_id: str, resources: dict[str, str], duration_s: float) -> Traces:
-    """The non-gen_ai entry span every AI trace hangs off. On its own (no gen_ai
-    children) it is exactly the trace the AI gate must exclude."""
-    return Traces(
-        timestamp=now - timedelta(seconds=5),
-        duration=timedelta(seconds=duration_s),
-        trace_id=trace_id,
-        span_id=span_id,
-        parent_span_id="",
-        name="POST /api/chat",
-        kind=TracesKind.SPAN_KIND_SERVER,
-        status_code=TracesStatusCode.STATUS_CODE_OK,
-        resources=resources,
-        attributes={"http.request.method": "POST"},
-    )
-
-
-def ai_trace(
-    *,
-    now: datetime,
-    service: str,
-    user: str,
-    in_tokens: int | None,
-    out_tokens: int,
-    cost: float,
-    model: str = "gpt-4o-mini",
-    environment: str = "production",
-) -> list[Traces]:
-    """A minimal AI trace: root span + one LLM span with gen_ai attributes.
-    in_tokens=None omits the input-tokens attribute entirely (not zero)."""
-    trace_id = TraceIdGenerator.trace_id()
-    root_id = TraceIdGenerator.span_id()
-    resources = {"service.name": service, "deployment.environment": environment}
-
-    attributes = {
-        "gen_ai.request.model": model,
-        "gen_ai.system": "openai",
-        "gen_ai.user.id": user,
-        # numeric values land in attributes_number
-        "gen_ai.usage.output_tokens": out_tokens,
-        "_signoz.gen_ai.total_cost": cost,
-    }
-    if in_tokens is not None:
-        attributes["gen_ai.usage.input_tokens"] = in_tokens
-
-    llm = Traces(
-        timestamp=now - timedelta(seconds=4),
-        duration=timedelta(seconds=1),
-        trace_id=trace_id,
-        span_id=TraceIdGenerator.span_id(),
-        parent_span_id=root_id,
-        name="chat gpt-4o-mini",
-        kind=TracesKind.SPAN_KIND_CLIENT,
-        status_code=TracesStatusCode.STATUS_CODE_OK,
-        resources=resources,
-        attributes=attributes,
-    )
-    return [
-        root_span(now=now, trace_id=trace_id, span_id=root_id, resources=resources, duration_s=1.1),
-        llm,
-    ]
-
-
-def ai_trace_mixed_spans(*, now: datetime, service: str, user: str) -> list[Traces]:
-    """
-    Root + one LLM span + one tool span + one agent span. The gate matches all three
-    child spans, but only the LLM span carries gen_ai.request.model.
-    """
-    trace_id = TraceIdGenerator.trace_id()
-    root_id = TraceIdGenerator.span_id()
-    resources = {"service.name": service, "deployment.environment": "production"}
-
-    def child(name: str, kind: TracesKind, attributes: dict, offset_s: float) -> Traces:
-        return Traces(
-            timestamp=now - timedelta(seconds=offset_s),
-            duration=timedelta(seconds=0.5),
-            trace_id=trace_id,
-            span_id=TraceIdGenerator.span_id(),
-            parent_span_id=root_id,
-            name=name,
-            kind=kind,
-            status_code=TracesStatusCode.STATUS_CODE_OK,
-            resources=resources,
-            attributes=attributes,
-        )
-
-    return [
-        root_span(now=now, trace_id=trace_id, span_id=root_id, resources=resources, duration_s=4),
-        child(
-            "chat gpt-4o-mini",
-            TracesKind.SPAN_KIND_CLIENT,
-            {
-                "gen_ai.request.model": "gpt-4o-mini",
-                "gen_ai.system": "openai",
-                "gen_ai.user.id": user,
-                "gen_ai.usage.input_tokens": 100,
-                "gen_ai.usage.output_tokens": 20,
-            },
-            4,
-        ),
-        child(
-            "execute_tool",
-            TracesKind.SPAN_KIND_INTERNAL,
-            {"gen_ai.tool.name": "get_weather", "gen_ai.tool.type": "function"},
-            3,
-        ),
-        child("agent.step", TracesKind.SPAN_KIND_INTERNAL, {"gen_ai.agent.name": "chat-agent"}, 2),
-    ]

tests/fixtures/types.py

@@ -197,3 +197,40 @@ class AlertTestCase:
     alert_data: list[AlertData]
     # list of alert expectations for the test case
     alert_expectation: AlertExpectation
+
+
+@dataclass(frozen=True)
+class NotificationValidation:
+    # destination type of the notification, either webhook or email
+    # slack, msteams, pagerduty, opsgenie, webhook channels send notifications through webhook
+    # email channels send notifications through email
+    destination_type: Literal["webhook", "email"]
+    # validation data for validating the received notification payload
+    validation_data: dict[str, any]
+
+
+@dataclass(frozen=True)
+class AMNotificationExpectation:
+    # whether we expect any notifications to be fired or not, false when testing downtime scenarios
+    # or don't expect any notifications to be fired in given time period
+    should_notify: bool
+    # seconds to wait for the notifications to be fired, if no
+    # notifications are fired in the expected time, the test will fail
+    wait_time_seconds: int
+    # list of notifications to expect, as a single rule can trigger multiple notifications
+    # spanning across different notifiers
+    notification_validations: list[NotificationValidation]
+
+
+@dataclass(frozen=True)
+class AlertManagerNotificationTestCase:
+    # name of the test case
+    name: str
+    # path to the rule file in testdata directory
+    rule_path: str
+    # list of alert data that will be inserted into the database for the rule to be triggered
+    alert_data: list[AlertData]
+    # configuration for the notification channel
+    channel_config: dict[str, any]
+    # notification expectations for the test case
+    notification_expectation: AMNotificationExpectation

tests/integration/bootstrap/setup.py

@@ -39,5 +39,7 @@ def test_teardown(
     idp: types.TestContainerIDP,  # pylint: disable=unused-argument
     create_user_admin: types.Operation,  # pylint: disable=unused-argument
     migrator: types.Operation,  # pylint: disable=unused-argument
+    maildev: types.TestContainerDocker,  # pylint: disable=unused-argument
+    notification_channel: types.TestContainerDocker,  # pylint: disable=unused-argument
 ) -> None:
     pass

tests/integration/testdata/alertmanager/content_templating/logs_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "User login successful", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Database connection established", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "API request received", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Response sent to client", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }

tests/integration/testdata/alertmanager/content_templating/logs_rule.json

@@ -0,0 +1,69 @@
+{
+  "alert": "content_templating_logs",
+  "ruleType": "threshold_rule",
+  "alertType": "LOGS_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 0,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "logs",
+            "filter": {
+              "expression": "body CONTAINS 'payment failure'"
+            },
+            "aggregations": [
+              {
+                "expression": "count()"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Payment failure spike detected on $service_name",
+    "summary": "Payment failures elevated on $service_name"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/metrics_data.jsonl

@@ -0,0 +1,12 @@
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:01:00+00:00","value":80,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:02:00+00:00","value":95,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:03:00+00:00","value":110,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:04:00+00:00","value":120,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:05:00+00:00","value":125,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:06:00+00:00","value":130,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:07:00+00:00","value":135,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:08:00+00:00","value":140,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:09:00+00:00","value":145,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:10:00+00:00","value":150,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:11:00+00:00","value":155,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:12:00+00:00","value":160,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}

tests/integration/testdata/alertmanager/content_templating/metrics_rule.json

@@ -0,0 +1,72 @@
+{
+  "alert": "content_templating_metrics",
+  "ruleType": "threshold_rule",
+  "alertType": "METRIC_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 100,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "metrics",
+            "aggregations": [
+              {
+                "metricName": "container_memory_bytes_content_templating",
+                "timeAggregation": "avg",
+                "spaceAggregation": "max"
+              }
+            ],
+            "groupBy": [
+              {"name": "namespace", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "pod", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "container", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "node", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "severity", "fieldContext": "attribute", "fieldDataType": "string"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Container $container in pod $pod ($namespace) exceeded memory threshold",
+    "summary": "High container memory in $namespace/$pod"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/traces_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "duration": "PT1.2S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a1", "span_id": "c1b2c3d4e5f6a7b8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "duration": "PT1.4S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a2", "span_id": "c2b3c4d5e6f7a8b9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "duration": "PT1.6S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a3", "span_id": "c3b4c5d6e7f8a9b0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "duration": "PT1.8S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a4", "span_id": "c4b5c6d7e8f9a0b1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "duration": "PT2.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a5", "span_id": "c5b6c7d8e9f0a1b2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "duration": "PT2.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a6", "span_id": "c6b7c8d9e0f1a2b3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "duration": "PT2.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a7", "span_id": "c7b8c9d0e1f2a3b4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "duration": "PT2.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a8", "span_id": "c8b9c0d1e2f3a4b5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "duration": "PT2.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a9", "span_id": "c9b0c1d2e3f4a5b6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "duration": "PT3.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b1", "span_id": "d1c2d3e4f5a6b7c8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "duration": "PT3.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b2", "span_id": "d2c3d4e5f6a7b8c9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "duration": "PT3.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b3", "span_id": "d3c4d5e6f7a8b9c0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "duration": "PT3.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b4", "span_id": "d4c5d6e7f8a9b0c1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "duration": "PT3.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b5", "span_id": "d5c6d7e8f9a0b1c2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "duration": "PT4.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b6", "span_id": "d6c7d8e9f0a1b2c3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "duration": "PT4.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b7", "span_id": "d7c8d9e0f1a2b3c4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "duration": "PT4.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b8", "span_id": "d8c9d0e1f2a3b4c5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "duration": "PT4.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b9", "span_id": "d9c0d1e2f3a4b5c6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "duration": "PT4.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c1", "span_id": "e1d2e3f4a5b6c7d8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "duration": "PT5.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c2", "span_id": "e2d3e4f5a6b7c8d9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }

tests/integration/testdata/alertmanager/content_templating/traces_rule.json

@@ -0,0 +1,71 @@
+{
+  "alert": "content_templating_traces",
+  "ruleType": "threshold_rule",
+  "alertType": "TRACES_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 1,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ],
+          "targetUnit": "s"
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "unit": "ns",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "traces",
+            "filter": {
+              "expression": "http.request.path = '/checkout'"
+            },
+            "aggregations": [
+              {
+                "expression": "p90(duration_nano)"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "p90 latency high on $service_name",
+    "summary": "p90 latency exceeded threshold on $service_name"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/tests/alertmanager/01_channels.py

@@ -0,0 +1,268 @@
+import time
+import uuid
+from collections.abc import Callable
+from http import HTTPStatus
+
+import pytest
+import requests
+from sqlalchemy import text
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.maildev import (
+    MAILDEV_INCOMING_PASS,
+    SMTP_TEST_FROM,
+    delete_all_mails,
+    verify_email_received,
+)
+from fixtures.notification_channel import assert_email_channel_payload_clean, send_test_notification
+
+TIMEOUT = 10
+
+
+CHANNEL_TYPE_CASES = [
+    (
+        "webhook",
+        lambda sink: {"webhook_configs": [{"url": sink.container_configs["8080"].get("/webhook/crud-original"), "send_resolved": True}]},
+        lambda sink: {"webhook_configs": [{"url": sink.container_configs["8080"].get("/webhook/crud-updated"), "send_resolved": True}]},
+        "crud-original",
+        "crud-updated",
+    ),
+    (
+        "slack",
+        lambda sink: {"slack_configs": [{"api_url": sink.container_configs["8080"].get("/services/T/B/X"), "channel": "#crud-original"}]},
+        lambda sink: {"slack_configs": [{"api_url": sink.container_configs["8080"].get("/services/T/B/X"), "channel": "#crud-updated"}]},
+        "#crud-original",
+        "#crud-updated",
+    ),
+    (
+        "pagerduty",
+        lambda sink: {"pagerduty_configs": [{"routing_key": "crud-original-routing-key"}]},
+        lambda sink: {"pagerduty_configs": [{"routing_key": "crud-updated-routing-key"}]},
+        "crud-original-routing-key",
+        "crud-updated-routing-key",
+    ),
+    (
+        "opsgenie",
+        lambda sink: {"opsgenie_configs": [{"api_key": "crud-original-api-key", "message": "{{ .CommonLabels.alertname }}"}]},
+        lambda sink: {"opsgenie_configs": [{"api_key": "crud-updated-api-key", "message": "{{ .CommonLabels.alertname }}"}]},
+        "crud-original-api-key",
+        "crud-updated-api-key",
+    ),
+    (
+        "msteamsv2",
+        lambda sink: {"msteamsv2_configs": [{"webhook_url": sink.container_configs["8080"].get("/msteams/crud-original")}]},
+        lambda sink: {"msteamsv2_configs": [{"webhook_url": sink.container_configs["8080"].get("/msteams/crud-updated")}]},
+        "crud-original",
+        "crud-updated",
+    ),
+    (
+        "email",
+        lambda sink: {"email_configs": [{"to": "crud-original@integration.test"}]},
+        lambda sink: {"email_configs": [{"to": "crud-updated@integration.test"}]},
+        "crud-original@integration.test",
+        "crud-updated@integration.test",
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "channel_type,make_config,make_updated_config,created_marker,updated_marker",
+    CHANNEL_TYPE_CASES,
+    ids=[case[0] for case in CHANNEL_TYPE_CASES],
+)
+def test_channel_crud(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    notification_channel: types.TestContainerDocker,
+    channel_type: str,
+    make_config: Callable[[types.TestContainerDocker], dict],
+    make_updated_config: Callable[[types.TestContainerDocker], dict],
+    created_marker: str,
+    updated_marker: str,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"crud-{channel_type}-{uuid.uuid4()}"
+
+    config = {"name": name, **make_config(notification_channel)}
+    response = requests.post(signoz.self.host_configs["8080"].get("/api/v1/channels"), json=config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.CREATED, response.text
+    created = response.json()["data"]
+    channel_id = created["id"]
+    assert created["name"] == name
+    assert created["type"] == channel_type
+
+    response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    listed = {channel["name"]: channel for channel in response.json()["data"]}
+    assert name in listed
+    assert listed[name]["type"] == channel_type
+
+    response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert created_marker in response.json()["data"]["data"]
+
+    updated_config = {"name": name, **make_updated_config(notification_channel)}
+    response = requests.put(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), json=updated_config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NO_CONTENT, response.text
+
+    response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    data = response.json()["data"]["data"]
+    assert updated_marker in data
+    assert created_marker not in data
+
+    response = requests.delete(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NO_CONTENT, response.text
+
+    response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NOT_FOUND, response.text
+
+
+def test_create_rejects_duplicate_name(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    create_notification_channel: Callable[[dict], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"duplicate-{uuid.uuid4()}"
+
+    create_notification_channel({"name": name, "email_configs": [{"to": "first@integration.test"}]})
+
+    response = requests.post(
+        signoz.self.host_configs["8080"].get("/api/v1/channels"),
+        json={"name": name, "email_configs": [{"to": "second@integration.test"}]},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "unique" in response.text
+
+
+def test_create_rejects_channel_without_configs(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = requests.post(
+        signoz.self.host_configs["8080"].get("/api/v1/channels"),
+        json={"name": f"empty-{uuid.uuid4()}"},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "notification configuration" in response.text
+
+
+def test_update_rejects_name_change(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    create_notification_channel: Callable[[dict], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"rename-{uuid.uuid4()}"
+    channel_id = create_notification_channel({"name": name, "email_configs": [{"to": "rename@integration.test"}]})
+
+    response = requests.put(
+        signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"),
+        json={"name": f"{name}-renamed", "email_configs": [{"to": "rename@integration.test"}]},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "cannot update channel name" in response.text
+
+
+def test_channels_require_authentication(signoz: types.SigNoz) -> None:
+    response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.UNAUTHORIZED, response.text
+
+
+def test_email_channel_never_stores_or_serves_smtp_settings(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    hostile_name = f"hostile-email-{uuid.uuid4()}"
+    hostile_config = {
+        "name": hostile_name,
+        "email_configs": [
+            {
+                "to": "hostile@integration.test",
+                "from": "spoofed@integration.test",
+                "hello": "attacker.test",
+                "smarthost": "smtp.attacker.test:2525",
+                "auth_username": "attacker",
+                "auth_password": "tenant-posted-secret",
+                "require_tls": False,
+                "headers": {"Subject": "hostile subject"},
+            }
+        ],
+    }
+
+    response = requests.post(signoz.self.host_configs["8080"].get("/api/v1/channels"), json=hostile_config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.CREATED, response.text
+    created = response.json()["data"]
+    assert_email_channel_payload_clean(created["data"])
+
+    response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{created['id']}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    served = response.json()["data"]["data"]
+    assert_email_channel_payload_clean(served)
+    assert "hostile@integration.test" in served
+    assert "hostile subject" in served
+    assert "smtp.attacker.test" not in served
+    assert "tenant-posted-secret" not in served
+
+    response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert "tenant-posted-secret" not in response.text
+    assert MAILDEV_INCOMING_PASS not in response.text
+
+    with signoz.sqlstore.conn.connect() as conn:
+        stored = conn.execute(
+            text("SELECT data FROM notification_channel WHERE name = :name"),
+            {"name": hostile_name},
+        ).fetchone()
+        assert stored is not None
+        assert_email_channel_payload_clean(stored[0])
+        assert "tenant-posted-secret" not in stored[0]
+
+        configs = conn.execute(text("SELECT config FROM alertmanager_config")).fetchall()
+        assert len(configs) > 0
+        for (config_raw,) in configs:
+            assert MAILDEV_INCOMING_PASS not in config_raw
+            assert "tenant-posted-secret" not in config_raw
+            assert '"smtp_auth_password"' not in config_raw
+            assert '"auth_password"' not in config_raw
+
+
+def test_email_test_channel_delivers_via_env_transport(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    maildev: types.TestContainerDocker,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    delete_all_mails(maildev)
+
+    recipient = f"delivery-{uuid.uuid4()}@integration.test"
+    send_test_notification(
+        signoz,
+        token,
+        {"name": f"delivery-{uuid.uuid4()}", "email_configs": [{"to": recipient}]},
+    )
+
+    deadline = time.time() + 30
+    while time.time() < deadline:
+        if verify_email_received(maildev, {"to": recipient, "from": SMTP_TEST_FROM}):
+            return
+        time.sleep(1)
+    raise AssertionError(f"no email delivered to {recipient} from {SMTP_TEST_FROM}")

tests/integration/tests/alertmanager/02_notifiers.py

@@ -0,0 +1,360 @@
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    email_default_config,
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+logger = setup_logger(__name__)
+
+NOTIFIERS_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="slack_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            # extra wait for alertmanager server setup
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="msteams_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                                        "type": "AdaptiveCard",
+                                        "version": "1.2",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Alerts",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                                "wrap": True,
+                                                "color": "Attention",
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Labels",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "threshold.name",
+                                                        "value": "critical",
+                                                    }
+                                                ],
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Annotations",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "description",
+                                                        "value": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                                    },
+                                                ],
+                                            },
+                                        ],
+                                        "msteams": {"width": "full"},
+                                        "actions": [
+                                            {
+                                                "type": "Action.OpenUrl",
+                                                "title": "View Alert",
+                                            }
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "event_action": "trigger",
+                            "payload": {
+                                "source": "SigNoz Alert Manager",
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Annotations": [
+                                            {"description = This alert is fired when the defined metric (current value": "15) crosses the threshold (10)"},
+                                        ],
+                                        "Labels": [
+                                            "alertname = threshold_above_at_least_once",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "threshold_above_at_least_once",
+                            "details": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "threshold_above_at_least_once",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                        "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "commonAnnotations": {
+                                "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="email_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=email_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="email",
+                    validation_data={
+                        "subject": re.compile(r'\[FIRING:1\] threshold_above_at_least_once for  \(alertname="threshold_above_at_least_once", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'),
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "notifier_test_case",
+    NOTIFIERS_TEST,
+    ids=lambda notifier_test_case: notifier_test_case.name,
+)
+def test_notifier_templating(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    create_alert_rule: Callable[[dict], str],
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    maildev: types.TestContainerDocker,
+    notifier_test_case: types.AlertManagerNotificationTestCase,
+):
+    channel_name = str(uuid.uuid4())
+
+    channel_config = update_raw_channel_config(notifier_test_case.channel_config, channel_name, notification_channel)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    webhook_validations = [v for v in notifier_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    if any(v.destination_type == "email" for v in notifier_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    time.sleep(12)
+
+    insert_alert_data(
+        notifier_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=5),
+    )
+
+    rule_path = get_testdata_file_path(notifier_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        notifier_test_case.notification_expectation,
+    )

tests/integration/tests/alertmanager/03_content_templating.py

@@ -0,0 +1,332 @@
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+logger = setup_logger(__name__)
+
+
+CONTENT_TEMPLATING_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="msteams_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "type": "AdaptiveCard",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": re.compile(
+                                                    r'\[FIRING:1\] content_templating_metrics for  \(alertname="content_templating_metrics", container="checkout", namespace="production", node="ip-10-0-1-23", pod="checkout-7d9c8b5f4-x2k9p", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'
+                                                ),
+                                            },
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "content_templating_metrics",
+                            "details": {
+                                "alertname": "content_templating_metrics",
+                                "container": "checkout",
+                                "namespace": "production",
+                                "node": "ip-10-0-1-23",
+                                "pod": "checkout-7d9c8b5f4-x2k9p",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "payload": {
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Labels": [
+                                            "alertname = content_templating_metrics",
+                                            "container = checkout",
+                                            "namespace = production",
+                                            "node = ip-10-0-1-23",
+                                            "pod = checkout-7d9c8b5f4-x2k9p",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="slack_logs_default_templating",
+        rule_path="alertmanager/content_templating/logs_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="logs",
+                data_path="alertmanager/content_templating/logs_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="slack_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "content_templating_metrics",
+                                        "container": "checkout",
+                                        "namespace": "production",
+                                        "node": "ip-10-0-1-23",
+                                        "pod": "checkout-7d9c8b5f4-x2k9p",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "description": "Container checkout in pod checkout-7d9c8b5f4-x2k9p (production) exceeded memory threshold",
+                                        "summary": "High container memory in production/checkout-7d9c8b5f4-x2k9p",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "content_templating_metrics",
+                                "container": "checkout",
+                                "namespace": "production",
+                                "node": "ip-10-0-1-23",
+                                "pod": "checkout-7d9c8b5f4-x2k9p",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "content_templating_test_case",
+    CONTENT_TEMPLATING_TEST,
+    ids=lambda content_templating_test_case: content_templating_test_case.name,
+)
+def test_content_templating(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    create_alert_rule: Callable[[dict], str],
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    maildev: types.TestContainerDocker,
+    content_templating_test_case: types.AlertManagerNotificationTestCase,
+):
+    channel_name = str(uuid.uuid4())
+
+    channel_config = update_raw_channel_config(content_templating_test_case.channel_config, channel_name, notification_channel)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    webhook_validations = [v for v in content_templating_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    if any(v.destination_type == "email" for v in content_templating_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    time.sleep(12)
+
+    insert_alert_data(
+        content_templating_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=10),
+    )
+
+    rule_path = get_testdata_file_path(content_templating_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        content_templating_test_case.notification_expectation,
+    )

tests/integration/tests/alertmanager/conftest.py

@@ -0,0 +1,35 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.maildev import signoz_smtp_env
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev: types.TestContainerDocker,
+    notification_channel: types.TestContainerDocker,
+) -> types.SigNoz:
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_alertmanager",
+        env_overrides={
+            **signoz_smtp_env(maildev),
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_PAGERDUTY__URL": notification_channel.container_configs["8080"].get("/v2/enqueue"),
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_OPSGENIE__API__URL": notification_channel.container_configs["8080"].get("/"),
+        },
+    )

tests/integration/tests/alertmanagerrotation/01_smtp_rotation.py

@@ -0,0 +1,99 @@
+import time
+import uuid
+from collections.abc import Callable
+from http import HTTPStatus
+
+import docker
+import pytest
+import requests
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, token_getter
+from fixtures.logger import setup_logger
+from fixtures.maildev import (
+    NEW_PROVIDER_SMTP_PASS,
+    SMTP_TEST_FROM,
+    delete_all_mails,
+    get_all_mails,
+    signoz_smtp_env,
+    verify_email_received,
+)
+from fixtures.notification_channel import send_test_notification
+from fixtures.signoz import create_signoz
+
+logger = setup_logger(__name__)
+
+
+def wait_for_email(maildev: types.TestContainerDocker, filters: dict, wait_seconds: int = 30) -> None:
+    deadline = time.time() + wait_seconds
+    while time.time() < deadline:
+        if verify_email_received(maildev, filters):
+            return
+        time.sleep(1)
+    raise AssertionError(f"no email matching {filters} within {wait_seconds}s, inbox: {get_all_mails(maildev)}")
+
+
+def test_smtp_rotation_applies_to_existing_channels(  # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    maildev_old: types.TestContainerDocker,
+    maildev_new: types.TestContainerDocker,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    channel_name = f"rotation-{uuid.uuid4()}"
+    recipient = f"rotation-{uuid.uuid4()}@integration.test"
+
+    response = requests.post(
+        signoz.self.host_configs["8080"].get("/api/v1/channels"),
+        json={"name": channel_name, "email_configs": [{"to": recipient}]},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=10,
+    )
+    assert response.status_code == HTTPStatus.CREATED, response.text
+
+    delete_all_mails(maildev_old)
+    recipient_old_probe = f"probe-old-{uuid.uuid4()}@integration.test"
+    send_test_notification(signoz, token, {"name": f"probe-{uuid.uuid4()}", "email_configs": [{"to": recipient_old_probe}]})
+    wait_for_email(maildev_old, {"to": recipient_old_probe, "from": SMTP_TEST_FROM})
+    logger.info("Delivery through the old provider verified")
+
+    docker.from_env().containers.get(signoz.self.id).stop()
+    logger.info("Stopped signoz running against the old provider")
+
+    signoz_new = create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_smtp_rotation_new",
+        env_overrides=signoz_smtp_env(maildev_new, password=NEW_PROVIDER_SMTP_PASS),
+    )
+    token_new = token_getter(signoz_new)(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = requests.get(
+        signoz_new.self.host_configs["8080"].get("/api/v1/channels"),
+        headers={"Authorization": f"Bearer {token_new}"},
+        timeout=10,
+    )
+    assert response.status_code == HTTPStatus.OK, response.text
+    listed = {channel["name"]: channel for channel in response.json()["data"]}
+    assert channel_name in listed
+
+    delete_all_mails(maildev_new)
+    mails_at_old_provider = len(get_all_mails(maildev_old))
+    recipient_new_probe = f"probe-new-{uuid.uuid4()}@integration.test"
+    send_test_notification(signoz_new, token_new, {"name": f"probe-{uuid.uuid4()}", "email_configs": [{"to": recipient_new_probe}]})
+    wait_for_email(maildev_new, {"to": recipient_new_probe, "from": SMTP_TEST_FROM})
+    assert len(get_all_mails(maildev_old)) == mails_at_old_provider, "old provider must receive nothing after rotation"

tests/integration/tests/alertmanagerrotation/conftest.py

@@ -0,0 +1,40 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.maildev import NEW_PROVIDER_SMTP_PASS, OLD_PROVIDER_SMTP_PASS, create_maildev, signoz_smtp_env
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="maildev_old", scope="package")
+def maildev_old(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig, cache_key="maildev_smtp_old", incoming_pass=OLD_PROVIDER_SMTP_PASS)
+
+
+@pytest.fixture(name="maildev_new", scope="package")
+def maildev_new(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig, cache_key="maildev_smtp_new", incoming_pass=NEW_PROVIDER_SMTP_PASS)
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev_old: types.TestContainerDocker,
+) -> types.SigNoz:
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_smtp_rotation",
+        env_overrides=signoz_smtp_env(maildev_old, password=OLD_PROVIDER_SMTP_PASS),
+    )

tests/integration/tests/promqlconformance/03_nonfinite_cache.py

@@ -0,0 +1,72 @@
+"""
+Regression test for caching PromQL results that contain non-finite values.
+
+A ratio yields NaN where both sides are zero, and NaN marshals as the string
+"NaN". Before the fix the cached bucket could not be read back, so a second
+identical request returned only the window edges.
+"""
+
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from http import HTTPStatus
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.metrics import Metrics
+from fixtures.querier import get_all_series, make_query_request
+
+SUM_METRIC = "job_duration_sum"
+COUNT_METRIC = "job_duration_count"
+HOUR_MS = 3_600_000
+SAMPLE_INTERVAL_MS = 60_000
+
+
+def test_cached_promql_result_with_nan_matches_uncached(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_metrics: Callable[[list[Metrics]], None],
+) -> None:
+    # 12h ending on an hour boundary 15m ago — old enough to be cached.
+    end_ms = (int((datetime.now(tz=UTC) - timedelta(minutes=15)).timestamp() * 1000) // HOUR_MS) * HOUR_MS
+    start_ms = end_ms - 12 * HOUR_MS
+
+    # active_job divides finite; idle_job is 0/0, the NaN the cache must survive.
+    series = {"active_job": (100.0, 4.0), "idle_job": (0.0, 0.0)}
+    metrics: list[Metrics] = []
+    for job_name, (sum_value, count_value) in series.items():
+        for ts_ms in range(start_ms, end_ms + 1, SAMPLE_INTERVAL_MS):
+            timestamp = datetime.fromtimestamp(ts_ms / 1000, tz=UTC)
+            metrics.append(Metrics(metric_name=SUM_METRIC, labels={"job_name": job_name}, timestamp=timestamp, value=sum_value))
+            metrics.append(Metrics(metric_name=COUNT_METRIC, labels={"job_name": job_name}, timestamp=timestamp, value=count_value))
+    insert_metrics(metrics)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    promql = f"sum by (job_name) ({SUM_METRIC}) / sum by (job_name) ({COUNT_METRIC})"
+
+    def run() -> tuple[dict[str, dict[int, object]], int]:
+        query = {"type": "promql", "spec": {"name": "A", "query": promql}}
+        response = make_query_request(signoz, token, start_ms, end_ms, [query], no_cache=False)
+        assert response.status_code == HTTPStatus.OK, response.text[:300]
+        body = response.json()
+        out: dict[str, dict[int, object]] = {}
+        for entry in get_all_series(body, "A") or []:
+            labels = {l["key"]["name"]: str(l["value"]) for l in entry.get("labels") or []}
+            out[labels["job_name"]] = {v["timestamp"]: v["value"] for v in entry.get("values") or []}
+        return out, int(body["data"]["meta"]["stepIntervals"]["A"])
+
+    # First populates the cache, second must be served from it.
+    first, step_seconds = run()
+    second, _ = run()
+
+    expected_points = (end_ms - start_ms) // (step_seconds * 1000) + 1
+    assert set(first) == set(series), sorted(first)
+    assert set(first["idle_job"].values()) == {"NaN"}, sorted(set(first["idle_job"].values()))
+    assert set(first["active_job"].values()) == {25.0}, sorted(set(first["active_job"].values()))
+    assert len(first["idle_job"]) == expected_points, f"expected {expected_points} points, got {len(first['idle_job'])}"
+
+    # The cached read excludes end_ms, the one legitimate difference.
+    assert set(second) == set(first), sorted(second)
+    for job_name, points in first.items():
+        expected = {ts: value for ts, value in points.items() if ts < end_ms}
+        assert second[job_name] == expected, f"{job_name}: got {len(second[job_name])} of {len(expected)} points"

tests/integration/tests/querierai/01_ai_traces.py

@@ -1,681 +0,0 @@
-"""
-Integration tests for query_type="builder_ai_query" over the traces signal.
-
-Data shape (generic OTel gen_ai semantic conventions):
-  - a root span (no gen_ai attributes)
-  - an LLM span carrying gen_ai.request.model (str) and numeric usage attributes
-    (gen_ai.usage.input_tokens / output_tokens / cost) plus gen_ai.user.id
-Each test tags its spans with a unique service.name and filters on it, so tests do
-not interfere with each other's data. Builders shared across the suite (query window,
-ai_trace, ai_trace_mixed_spans) live in fixtures/querierai.py; one-off shapes are
-built right above the test that uses them.
-"""
-
-import json
-from collections.abc import Callable
-from datetime import UTC, datetime, timedelta
-from http import HTTPStatus
-
-import pytest
-
-from fixtures import types
-from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.querier import (
-    BuilderQuery,
-    OrderBy,
-    RequestType,
-    TelemetryFieldKey,
-    make_query_request,
-)
-from fixtures.querierai import (
-    ai_trace,
-    ai_trace_mixed_spans,
-    query_window,
-    root_span,
-)
-from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
-
-
-def test_ai_list_excludes_non_ai(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    Trace-list panel (requestType="trace"): returns AI traces and excludes the
-    non-AI trace. Asserts on the raw response payload to stay agnostic to the exact
-    row schema.
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-list"
-
-    ai = ai_trace(now=now, service=service, user="alice", in_tokens=100, out_tokens=20, cost=0.5)
-    # a lone root span, i.e. a trace with no gen_ai spans at all
-    non_ai = root_span(
-        now=now,
-        trace_id=TraceIdGenerator.trace_id(),
-        span_id=TraceIdGenerator.span_id(),
-        resources={"service.name": service},
-        duration_s=1,
-    )
-    ai_trace_id = ai[0].trace_id
-    insert_traces([*ai, non_ai])
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression=f"service.name = '{service}'",
-        limit=10,
-    )
-
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    body = json.dumps(response.json())
-    assert ai_trace_id in body, f"expected AI trace {ai_trace_id} in list response"
-    assert non_ai.trace_id not in body, f"non-AI trace {non_ai.trace_id} should be excluded by the gate"
-
-
-def test_ai_list_having_aggregate_filter(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    Aggregate filter written in the SAME filter box: the span-level predicate narrows
-    to the service, the trace-level `output_tokens > 100` keeps the large-token
-    trace and drops the small one (split internally into WHERE + HAVING). Both
-    spellings of a trace-level aggregate — bare and `trace.` — behave identically
-    (unit tests pin them to byte-identical SQL; this covers the wiring once
-    end-to-end). An output-only aggregate is rejected under either spelling.
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-having"
-
-    small = ai_trace(now=now, service=service, user="alice", in_tokens=10, out_tokens=20, cost=0.1)
-    large = ai_trace(now=now, service=service, user="bob", in_tokens=10, out_tokens=500, cost=0.2)
-    small_id = small[0].trace_id
-    large_id = large[0].trace_id
-    insert_traces(small + large)
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    for spelling in ("output_tokens", "trace.output_tokens"):
-        query = BuilderQuery(
-            signal="traces",
-            query_type="builder_ai_query",
-            name="A",
-            filter_expression=f"service.name = '{service}' AND {spelling} > 100",
-            limit=10,
-        )
-        response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-        assert response.status_code == HTTPStatus.OK, f"{spelling}: {response.text}"
-
-        body = json.dumps(response.json())
-        assert large_id in body, f"{spelling}: trace with 500 out-tokens should pass > 100"
-        assert small_id not in body, f"{spelling}: trace with 20 out-tokens should be filtered out by HAVING"
-
-    # output-only aggregate gets the targeted rejection.
-    bad = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression="trace.span_count > 3",
-        limit=10,
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [bad.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
-    assert "cannot be used" in response.text
-
-
-def test_ai_list_order_limit_offset(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """Trace list honors order by (aggregate column) + limit + offset (pagination)."""
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-order"
-
-    traces: list[Traces] = []
-    for out in (100, 200, 300, 400, 500):
-        traces += ai_trace(now=now, service=service, user="u", in_tokens=10, out_tokens=out, cost=0.1)
-    insert_traces(traces)
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    def page(offset: int) -> list[int]:
-        query = BuilderQuery(
-            signal="traces",
-            query_type="builder_ai_query",
-            name="A",
-            filter_expression=f"service.name = '{service}'",
-            order=[OrderBy(key=TelemetryFieldKey(name="output_tokens"), direction="desc")],
-            limit=2,
-            offset=offset,
-        )
-        resp = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-        assert resp.status_code == HTTPStatus.OK, resp.text
-        rows = resp.json()["data"]["data"]["results"][0]["rows"]
-        return [int(r["data"]["output_tokens"]) for r in rows]
-
-    assert page(0) == [500, 400], "first page: highest output_tokens, desc"
-    assert page(2) == [300, 200], "second page (offset 2): next two, desc"
-
-
-def test_ai_span_list_limit(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """Span list honors limit (delegated raw path): 6 gen_ai spans available, capped to 4."""
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-spanlimit"
-    insert_traces(ai_trace_mixed_spans(now=now, service=service, user="a") + ai_trace_mixed_spans(now=now, service=service, user="b"))
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression=f"service.name = '{service}'",
-        limit=4,
-    )
-    resp = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type=RequestType.RAW)
-    assert resp.status_code == HTTPStatus.OK, resp.text
-    rows = resp.json()["data"]["data"]["results"][0]["rows"]
-    assert len(rows) == 4, f"limit should cap at 4 (6 gen_ai spans available), got {len(rows)}"
-
-
-def test_ai_span_list_excludes_non_gen_ai_spans(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    Span list (requestType=raw): returns only the gen_ai spans (LLM/tool/agent); the
-    root span of the same trace (no gen_ai attributes) is excluded by the span-level gate.
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-spanlist"
-    insert_traces(ai_trace_mixed_spans(now=now, service=service, user="alice"))
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression=f"service.name = '{service}'",
-        select_fields=[TelemetryFieldKey(name="name", field_context="span")],
-        limit=50,
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type=RequestType.RAW)
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    rows = response.json()["data"]["data"]["results"][0]["rows"]
-    names = sorted(r["data"]["name"] for r in rows)
-    assert names == ["agent.step", "chat gpt-4o-mini", "execute_tool"], names
-    assert "POST /api/chat" not in names  # root span excluded
-
-
-def test_ai_list_having_or_aggregates(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    Two trace-level aggregates OR-ed within the filter box (regression guard for OR-group
-    whitespace handling): output_tokens > 100 OR input_tokens > 1000 keeps only the
-    large-output trace (input_tokens is 10 for both, so that branch never matches).
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-having-or"
-
-    small = ai_trace(now=now, service=service, user="a", in_tokens=10, out_tokens=20, cost=0.1)
-    large = ai_trace(now=now, service=service, user="b", in_tokens=10, out_tokens=500, cost=0.2)
-    small_id, large_id = small[0].trace_id, large[0].trace_id
-    insert_traces(small + large)
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression=f"service.name = '{service}' AND (output_tokens > 100 OR input_tokens > 1000)",
-        limit=10,
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    body = json.dumps(response.json())
-    assert large_id in body
-    assert small_id not in body
-
-
-def test_ai_list_resource_filter_isolates_by_fingerprint(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    A resource attribute in the filter is pulled into the __resource_filter fingerprint
-    CTE (see maybeAttachResourceFilter). Two traces on the same service but different
-    deployment.environment: `resource.deployment.environment = 'production'` must keep
-    the production trace and drop the staging one — the fingerprint prune isolates by
-    the resource, not by any span attribute.
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-resfilter"
-
-    prod = ai_trace(now=now, service=service, user="a", in_tokens=10, out_tokens=20, cost=0.1, environment="production")
-    stag = ai_trace(now=now, service=service, user="b", in_tokens=10, out_tokens=20, cost=0.1, environment="staging")
-    prod_id, stag_id = prod[0].trace_id, stag[0].trace_id
-    insert_traces(prod + stag)
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression=(f"resource.service.name = '{service}' AND resource.deployment.environment = 'production'"),
-        limit=10,
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    body = json.dumps(response.json())
-    assert prod_id in body, "production trace should match the resource filter"
-    assert stag_id not in body, "staging trace should be excluded by the resource fingerprint prune"
-
-
-def test_ai_list_rejects_aggregate_or_span_filter(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    Aggregate (HAVING) columns may not be OR-ed with span-level keys in the trace
-    list; a span-OR-span filter is fine.
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-orfilter"
-    # seed a trace so service.name resolves as a known key in this window (resource
-    # keys are discovered from ingested data).
-    insert_traces(ai_trace(now=now, service=service, user="a", in_tokens=10, out_tokens=20, cost=0.1))
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    # aggregate OR span -> rejected
-    bad = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        limit=10,
-        filter_expression=f"output_tokens > 1000 OR service.name = '{service}'",
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [bad.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
-    assert "cannot be combined" in response.text
-
-    # span OR span -> accepted (result content doesn't matter; just not an error)
-    ok = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        limit=10,
-        filter_expression=f"service.name = '{service}' OR has_error = true",
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [ok.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.OK, response.text
-
-
-def test_ai_list_nested_group_span_or_and_aggregate(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    A complex filter that mixes all three routing paths in one expression:
-        service.name = X AND (has_error = true OR gen_ai.request.model = 'gpt-4o') AND total_tokens > 100
-    The nested (span OR span) group must not flatten (precedence), the span predicates
-    go to WHERE as a trace-existence check, and the new `total_tokens` aggregate goes to
-    HAVING. Three traces isolate each discriminator:
-      - t_ok:      gpt-4o, out=500 -> OR matches (model) AND total_tokens>100  -> IN
-      - t_or_miss: gpt-4o-mini, out=500 -> OR fails (no error, wrong model)    -> OUT
-      - t_agg_miss: gpt-4o, out=20  -> OR matches but total_tokens<=100         -> OUT
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-nested"
-
-    t_ok = ai_trace(now=now, service=service, user="a", model="gpt-4o", in_tokens=10, out_tokens=500, cost=0.1)
-    t_or_miss = ai_trace(now=now, service=service, user="b", model="gpt-4o-mini", in_tokens=10, out_tokens=500, cost=0.1)
-    t_agg_miss = ai_trace(now=now, service=service, user="c", model="gpt-4o", in_tokens=10, out_tokens=20, cost=0.1)
-    insert_traces(t_ok + t_or_miss + t_agg_miss)
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression=(f"service.name = '{service}' AND (has_error = true OR gen_ai.request.model = 'gpt-4o') AND total_tokens > 100"),
-        limit=10,
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    body = json.dumps(response.json())
-    assert t_ok[0].trace_id in body
-    assert t_or_miss[0].trace_id not in body, "nested (span OR span) group must exclude the wrong-model, no-error trace"
-    assert t_agg_miss[0].trace_id not in body, "HAVING total_tokens > 100 must exclude the low-token trace"
-
-
-def test_ai_list_rejects_unknown_aggregate_key(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-) -> None:
-    """A trace-level filter on an unknown aggregate name is rejected, not silently run."""
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        limit=10,
-        filter_expression="trace.bogus_tokens > 1",
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
-
-
-def test_ai_list_rejects_order_by_span_attribute(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-) -> None:
-    """Only gen_ai-scoped aggregates are orderable; ordering by a span/resource key errors."""
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        limit=5,
-        order=[OrderBy(key=TelemetryFieldKey(name="service.name"), direction="asc")],
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
-    assert "order key" in response.text
-
-
-def test_ai_list_total_tokens_output_only(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    A trace whose LLM span carries only output tokens (no input-tokens attribute at
-    all) must still total: total_tokens is coalesce(sum(in),0)+coalesce(sum(out),0),
-    since sum over an absent attribute is NULL and NULL + n = NULL in ClickHouse.
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-total-coalesce"
-    insert_traces(ai_trace(now=now, service=service, user="a", in_tokens=None, out_tokens=300, cost=0.1))
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression=f"service.name = '{service}'",
-        limit=10,
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    rows = response.json()["data"]["data"]["results"][0]["rows"]
-    assert len(rows) == 1, f"expected one trace, got: {rows}"
-    data = rows[0]["data"]
-    assert data["input_tokens"] is None, data  # attribute absent -> NULL, not 0
-    assert data["output_tokens"] == 300, data
-    assert data["total_tokens"] == 300, f"total must coalesce the missing input side: {data}"
-
-
-def test_ai_list_variable_in_aggregate_filter(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """A query variable in a trace-level condition is substituted into the HAVING."""
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-having-var"
-
-    small = ai_trace(now=now, service=service, user="a", in_tokens=10, out_tokens=20, cost=0.1)
-    large = ai_trace(now=now, service=service, user="b", in_tokens=10, out_tokens=500, cost=0.2)
-    small_id, large_id = small[0].trace_id, large[0].trace_id
-    insert_traces(small + large)
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        filter_expression=f"service.name = '{service}' AND trace.output_tokens > $threshold",
-        limit=10,
-    )
-    response = make_query_request(
-        signoz,
-        token,
-        start_ms,
-        end_ms,
-        [query.to_dict()],
-        request_type="trace",
-        variables={"threshold": {"type": "custom", "value": 100}},
-    )
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    body = json.dumps(response.json())
-    assert large_id in body
-    assert small_id not in body
-
-
-def test_ai_list_messages_first_input_last_output(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    `input` is the FIRST LLM span's prompt (argMin over timestamp) and `output` is the
-    LAST LLM span's answer (argMax) — the question -> final-answer preview.
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-messages"
-    trace_id = TraceIdGenerator.trace_id()
-    root_id = TraceIdGenerator.span_id()
-    resources = {"service.name": service}
-
-    def llm(offset_s: float, prompt: str, answer: str) -> Traces:
-        return Traces(
-            timestamp=now - timedelta(seconds=offset_s),
-            duration=timedelta(seconds=1),
-            trace_id=trace_id,
-            span_id=TraceIdGenerator.span_id(),
-            parent_span_id=root_id,
-            name="chat",
-            kind=TracesKind.SPAN_KIND_CLIENT,
-            status_code=TracesStatusCode.STATUS_CODE_OK,
-            resources=resources,
-            attributes={
-                "gen_ai.request.model": "gpt-4o-mini",
-                "gen_ai.input.messages": prompt,
-                "gen_ai.output.messages": answer,
-            },
-        )
-
-    # earlier call is the "first" (its input is the prompt), later call is the "last"
-    # (its output is the final answer).
-    insert_traces(
-        [
-            root_span(now=now, trace_id=trace_id, span_id=root_id, resources=resources, duration_s=4),
-            llm(4, "first prompt", "first answer"),
-            llm(2, "second prompt", "second answer"),
-        ]
-    )
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        limit=10,
-        filter_expression=f"service.name = '{service}'",
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    rows = response.json()["data"]["data"]["results"][0]["rows"]
-    assert len(rows) == 1, f"expected one trace, got: {rows}"
-    data = rows[0]["data"]
-    assert data["input"] == "first prompt", f"input should be the earliest call's prompt: {data}"
-    assert data["output"] == "second answer", f"output should be the latest call's answer: {data}"
-
-
-def test_ai_list_enrichment_values(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-) -> None:
-    """
-    End-to-end values of the derived per-trace columns (only integration can check that
-    ClickHouse computes uniqIf / sum+sum / countIf(predicate) correctly, not just that
-    the SQL is shaped right). One trace: root + 1 errored LLM + 3 tool spans
-    (get_weather x2, get_time x1) + 1 agent span. The tool and agent spans are in the
-    gen_ai gate but carry no request.model, so llm_call_count stays 1 while span_count
-    counts them all.
-    """
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    service = "ai-it-metrics"
-    trace_id = TraceIdGenerator.trace_id()
-    root_id = TraceIdGenerator.span_id()
-    resources = {"service.name": service}
-
-    def tool(name: str, offset_s: float) -> Traces:
-        return Traces(
-            timestamp=now - timedelta(seconds=offset_s),
-            duration=timedelta(seconds=0.2),
-            trace_id=trace_id,
-            span_id=TraceIdGenerator.span_id(),
-            parent_span_id=root_id,
-            name="execute_tool",
-            kind=TracesKind.SPAN_KIND_INTERNAL,
-            status_code=TracesStatusCode.STATUS_CODE_OK,
-            resources=resources,
-            attributes={"gen_ai.tool.name": name, "gen_ai.tool.type": "function"},
-        )
-
-    llm = Traces(
-        timestamp=now - timedelta(seconds=4),
-        duration=timedelta(seconds=2),
-        trace_id=trace_id,
-        span_id=TraceIdGenerator.span_id(),
-        parent_span_id=root_id,
-        name="chat gpt-4o-mini",
-        kind=TracesKind.SPAN_KIND_CLIENT,
-        status_code=TracesStatusCode.STATUS_CODE_ERROR,  # -> has_error, drives error_count
-        resources=resources,
-        attributes={
-            "gen_ai.request.model": "gpt-4o-mini",
-            "gen_ai.usage.input_tokens": 100,
-            "gen_ai.usage.output_tokens": 20,
-            "_signoz.gen_ai.total_cost": 0.5,
-        },
-    )
-    agent = Traces(
-        timestamp=now - timedelta(seconds=1),
-        duration=timedelta(seconds=0.5),
-        trace_id=trace_id,
-        span_id=TraceIdGenerator.span_id(),
-        parent_span_id=root_id,
-        name="agent.step",
-        kind=TracesKind.SPAN_KIND_INTERNAL,
-        status_code=TracesStatusCode.STATUS_CODE_OK,
-        resources=resources,
-        attributes={"gen_ai.agent.name": "chat-agent"},
-    )
-    insert_traces(
-        [
-            root_span(now=now, trace_id=trace_id, span_id=root_id, resources=resources, duration_s=4),
-            llm,
-            tool("get_weather", 3),
-            tool("get_weather", 2.5),
-            tool("get_time", 2),
-            agent,
-        ]
-    )
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    start_ms, end_ms = query_window(now)
-
-    query = BuilderQuery(
-        signal="traces",
-        query_type="builder_ai_query",
-        name="A",
-        limit=10,
-        filter_expression=f"service.name = '{service}'",
-    )
-    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
-    assert response.status_code == HTTPStatus.OK, response.text
-
-    rows = response.json()["data"]["data"]["results"][0]["rows"]
-    assert len(rows) == 1, f"expected one trace, got: {rows}"
-    data = rows[0]["data"]
-
-    assert data["span_count"] == 6, data  # root + llm + 3 tools + agent
-    assert data["llm_call_count"] == 1, data  # only the request.model span, not tool/agent
-    assert data["tool_call_count"] == 3, data  # all three tool spans
-    assert data["distinct_tool_count"] == 2, data  # get_weather, get_time
-    assert data["input_tokens"] == 100, data
-    assert data["output_tokens"] == 20, data
-    assert data["total_tokens"] == 120, data  # input + output
-    assert data["estimated_total_cost"] == pytest.approx(0.5), data
-    assert data["error_count"] == 1, data  # the errored LLM span
-    assert data["max_llm_duration_nano"] > 0, data  # scoped max over LLM spans

tests/pyproject.toml

@@ -75,3 +75,7 @@ ignore = [
 
 [tool.ruff.format]
 # Defaults align with black (double quotes, 4-space indent).
+
+[tool.ruff.lint.per-file-ignores]
+"fixtures/notification_channel.py" = ["E501"]
+"integration/tests/alertmanager/*" = ["E501"]