fix(tests): wait until the browser has left the idp before returning from idp_login

Leaving the login page is not enough: keycloak's SAML flow returns an
auto-submitting interstitial on the idp host, and the POST it fires is what
actually creates the user in signoz. Returning at the interstitial let tests
query the user before the callback ran — flaky "user not found" in
test_idp_initiated_saml_authn.

Compare hosts instead of full URLs: intra-keycloak hops (login-actions, the
interstitial, a re-rendered form on bad credentials) keep waiting; the wait
ends only once signoz has handled the callback and redirected.

.github/workflows/integrationci.yaml

@@ -39,6 +39,8 @@ jobs:
       matrix:
         suite:
           - alerts
+          - alertmanager
+          - alertmanagerrotation
           - basepath
           - callbackauthn
           - cloudintegrations

frontend/src/pages/DashboardsListPageV2/utils/dslTokenizer.test.ts

@@ -114,6 +114,23 @@ describe('getCaretContext — stage detection', () => {
 		expect(ctx.partial).toBe('');
 	});
 
+	it('never replaces past the caret when it sits before the operator', () => {
+		const ctx = getCaretContext("env  = 'prod'", 4);
+		expect(ctx.stage).toBe('operator');
+		expect(ctx.partial).toBe('');
+		expect(ctx.replaceStart).toBe(4);
+		expect(ctx.replaceEnd).toBe(4);
+	});
+
+	it('never replaces past the caret when it sits before the value', () => {
+		const ctx = getCaretContext("env =  'prod'", 6);
+		expect(ctx.stage).toBe('value');
+		expect(ctx.operator).toBe('=');
+		expect(ctx.partial).toBe('');
+		expect(ctx.replaceStart).toBe(6);
+		expect(ctx.replaceEnd).toBe(6);
+	});
+
 	it('detects the stage of the term under a mid-string caret', () => {
 		const q = "env =  AND team = 'core'";
 		// caret right after the first `env ` (index 4) is the operator stage
@@ -142,6 +159,13 @@ describe('spliceAtCaret', () => {
 		expect(next).toBe("env = 'prod'");
 	});
 
+	it('inserts (without duplicating text) at a caret parked before a token', () => {
+		const q = "env  = 'prod'";
+		const ctx = getCaretContext(q, 4);
+		const { next } = spliceAtCaret(q, ctx, '!= ');
+		expect(next).toBe("env !=  = 'prod'");
+	});
+
 	it('preserves text after the caret', () => {
 		const q = "env  AND team = 'core'";
 		const ctx = getCaretContext(q, 4); // operator gap after `env`

frontend/src/pages/DashboardsListPageV2/utils/dslTokenizer.ts

@@ -328,7 +328,7 @@ export const getCaretContext = (query: string, caret: number): CaretContext => {
 		fieldKey: scan.key ? scan.key.text : '',
 		operator: slot.operator,
 		partial: slot.partial,
-		replaceStart: term.start + slot.replaceStartRel,
+		replaceStart: Math.min(term.start + slot.replaceStartRel, pos),
 		replaceEnd: pos,
 	};
 };

pkg/alertmanager/alertmanagerserver/server.go

@@ -241,9 +241,12 @@ func (server *Server) PutAlerts(ctx context.Context, postableAlerts alertmanager
 }
 
 func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertmanagertypes.Config) error {
-	config := alertmanagerConfig.AlertmanagerConfig()
+	resolved, err := alertmanagerConfig.Resolved()
+	if err != nil {
+		return err
+	}
+	config := resolved.AlertmanagerConfig()
 
-	var err error
 	// Load SigNoz's alertmanager notification templates from the configured
 	// globs. The upstream default templates (default.tmpl, email.tmpl) are
 	// always loaded from the embedded alertmanager assets inside FromGlobs, so
@@ -275,7 +278,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 			server.logger.InfoContext(ctx, "skipping creation of receiver not referenced by any route", slog.String("receiver", rcv.Name))
 			continue
 		}
-		extendedRcv, err := alertmanagerConfig.GetReceiver(rcv.Name)
+		extendedRcv, err := resolved.GetReceiver(rcv.Name)
 		if err != nil {
 			return err
 		}
@@ -350,7 +353,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 	go server.dispatcher.Run()
 	go server.inhibitor.Run()
 
-	server.alertmanagerConfig = alertmanagerConfig
+	server.alertmanagerConfig = resolved
 	return nil
 }
 

pkg/alertmanager/config.go

@@ -1,7 +1,6 @@
 package alertmanager
 
 import (
-	"net/url"
 	"strings"
 	"time"
 
@@ -26,11 +25,6 @@ type Signoz struct {
 	alertmanagerserver.Config `mapstructure:",squash" yaml:",squash"`
 }
 
-type Legacy struct {
-	// ApiURL is the URL of the legacy signoz alertmanager.
-	ApiURL *url.URL `mapstructure:"api_url"`
-}
-
 func NewConfigFactory() factory.ConfigFactory {
 	return factory.NewConfigFactory(factory.MustNewName("alertmanager"), newConfig)
 }

pkg/alertmanager/signozalertmanager/provider.go

@@ -167,6 +167,10 @@ func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgI
 		return err
 	}
 
+	if err := config.SetGlobalConfig(provider.config.Signoz.Global); err != nil {
+		return err
+	}
+
 	if err := config.UpdateReceiver(receiver); err != nil {
 		return err
 	}
@@ -217,6 +221,10 @@ func (provider *provider) CreateChannel(ctx context.Context, orgID string, recei
 		return nil, err
 	}
 
+	if err := config.SetGlobalConfig(provider.config.Signoz.Global); err != nil {
+		return nil, err
+	}
+
 	if err := config.CreateReceiver(receiver); err != nil {
 		return nil, err
 	}

pkg/modules/cloudintegration/implcloudintegration/definitionstore_test.go

@@ -0,0 +1,40 @@
+package implcloudintegration
+
+import (
+	"context"
+	"testing"
+
+	citypes "github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestServiceDefinitionsAreValid(t *testing.T) {
+	store := NewServiceDefinitionStore()
+
+	for _, provider := range []citypes.CloudProviderType{
+		citypes.CloudProviderTypeAWS,
+		citypes.CloudProviderTypeAzure,
+		citypes.CloudProviderTypeGCP,
+	} {
+		t.Run(provider.StringValue(), func(t *testing.T) {
+			defs, err := store.List(context.Background(), provider)
+			require.NoError(t, err, "all embedded definitions must load and validate")
+			require.NotEmpty(t, defs, "provider should ship at least one service definition")
+
+			for _, def := range defs {
+				assert.NotEmpty(t, def.ID, "service definition must have an id")
+				assert.NotEmpty(t, def.Title, "service %q must have a title", def.ID)
+
+				// Get() must agree with List() for every service it advertises.
+				serviceID, err := citypes.NewServiceID(provider, def.ID)
+				if !assert.NoError(t, err, "service id %q must be registered in serviceid.go", def.ID) {
+					continue
+				}
+				got, err := store.Get(context.Background(), provider, serviceID)
+				require.NoError(t, err, "service %q listed but not gettable", def.ID)
+				assert.Equal(t, def.ID, got.ID)
+			}
+		})
+	}
+}

pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/assets/dashboards/overview.json

@@ -621,7 +621,7 @@
                       {
                         "metricName": "cloudsql.googleapis.com/database/postgresql/deadlock_count",
                         "temporality": "",
-                        "timeAggregation": "max",
+                        "timeAggregation": "rate",
                         "spaceAggregation": "sum",
                         "reduceTo": "avg"
                       }
@@ -882,7 +882,7 @@
                       {
                         "metricName": "cloudsql.googleapis.com/database/postgresql/transaction_count",
                         "temporality": "",
-                        "timeAggregation": "max",
+                        "timeAggregation": "rate",
                         "spaceAggregation": "sum",
                         "reduceTo": "avg"
                       }

pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.json

@@ -54,13 +54,13 @@
       {
         "name": "cloudsql.googleapis.com/database/postgresql/transaction_count",
         "unit": "Count",
-        "type": "Gauge",
+        "type": "Sum",
         "description": ""
       },
       {
         "name": "cloudsql.googleapis.com/database/postgresql/deadlock_count",
         "unit": "Count",
-        "type": "Gauge",
+        "type": "Sum",
         "description": ""
       },
       {

pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/assets/dashboards/overview.json

@@ -925,7 +925,7 @@
                         "metricName": "compute.googleapis.com/instance/disk/average_io_latency",
                         "temporality": "",
                         "timeAggregation": "avg",
-                        "spaceAggregation": "sum",
+                        "spaceAggregation": "max",
                         "reduceTo": "avg"
                       }
                     ],

pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/assets/dashboards/overview.json

@@ -1207,8 +1207,8 @@
                       {
                         "metricName": "kubernetes.io/container/restart_count",
                         "temporality": "",
-                        "timeAggregation": "max",
-                        "spaceAggregation": "max",
+                        "timeAggregation": "increase",
+                        "spaceAggregation": "sum",
                         "reduceTo": "avg"
                       }
                     ],

pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/assets/dashboards/overview.json

@@ -601,8 +601,8 @@
                       {
                         "metricName": "redis.googleapis.com/stats/cache_hit_ratio",
                         "temporality": "",
-                        "timeAggregation": "max",
-                        "spaceAggregation": "sum",
+                        "timeAggregation": "min",
+                        "spaceAggregation": "min",
                         "reduceTo": "avg"
                       }
                     ],
@@ -788,7 +788,7 @@
                         "metricName": "redis.googleapis.com/commands/usec_per_call",
                         "temporality": "",
                         "timeAggregation": "max",
-                        "spaceAggregation": "sum",
+                        "spaceAggregation": "max",
                         "reduceTo": "avg"
                       }
                     ],
@@ -945,7 +945,7 @@
                       {
                         "metricName": "redis.googleapis.com/commands/calls",
                         "temporality": "",
-                        "timeAggregation": "max",
+                        "timeAggregation": "rate",
                         "spaceAggregation": "sum",
                         "reduceTo": "avg"
                       }
@@ -1044,8 +1044,8 @@
                       {
                         "metricName": "redis.googleapis.com/stats/reject_connections_count",
                         "temporality": "",
-                        "timeAggregation": "avg",
-                        "spaceAggregation": "avg",
+                        "timeAggregation": "rate",
+                        "spaceAggregation": "sum",
                         "reduceTo": "avg"
                       }
                     ],
@@ -1213,4 +1213,4 @@
     "refreshInterval": "",
     "links": []
   }
-}
+}

pkg/signoz/provider.go

@@ -233,6 +233,7 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewFillDashboardMeterSourceFactory(sqlstore, dashboardStore),
 		sqlmigration.NewUpdateRoleTransactionGroupsFactory(),
 		sqlmigration.NewFillDashboardSpecCollectionsFactory(sqlstore, dashboardStore),
+		sqlmigration.NewScrubEmailChannelTransportFactory(sqlstore),
 	)
 }
 

pkg/sqlmigration/107_scrub_email_channel_transport.go

@@ -0,0 +1,274 @@
+package sqlmigration
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/sqlstore"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/migrate"
+)
+
+type scrubEmailChannelTransport struct {
+	sqlstore sqlstore.SQLStore
+	logger   *slog.Logger
+}
+
+type alertmanagerConfigScrubRow struct {
+	bun.BaseModel `bun:"table:alertmanager_config"`
+
+	ID     string `bun:"id"`
+	Config string `bun:"config"`
+}
+
+type notificationChannelScrubRow struct {
+	bun.BaseModel `bun:"table:notification_channel"`
+
+	ID   string `bun:"id"`
+	Data string `bun:"data"`
+}
+
+var emailTransportKeys = []string{
+	"from",
+	"hello",
+	"smarthost",
+	"auth_username",
+	"auth_password",
+	"auth_password_file",
+	"auth_secret",
+	"auth_secret_file",
+	"auth_identity",
+	"require_tls",
+	"tls_config",
+	"force_implicit_tls",
+}
+
+var globalSMTPKeys = []string{
+	"smtp_from",
+	"smtp_hello",
+	"smtp_smarthost",
+	"smtp_auth_username",
+	"smtp_auth_password",
+	"smtp_auth_password_file",
+	"smtp_auth_secret",
+	"smtp_auth_secret_file",
+	"smtp_auth_identity",
+	"smtp_require_tls",
+	"smtp_tls_config",
+	"smtp_force_implicit_tls",
+}
+
+func NewScrubEmailChannelTransportFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
+	return factory.NewProviderFactory(
+		factory.MustNewName("scrub_email_channel_transport"),
+		func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
+			return &scrubEmailChannelTransport{sqlstore: sqlstore, logger: ps.Logger}, nil
+		},
+	)
+}
+
+func (migration *scrubEmailChannelTransport) Register(migrations *migrate.Migrations) error {
+	if err := migrations.Register(migration.Up, migration.Down); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (migration *scrubEmailChannelTransport) Up(ctx context.Context, db *bun.DB) error {
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = tx.Rollback()
+	}()
+
+	if err := migration.scrubConfigs(ctx, tx); err != nil {
+		return err
+	}
+
+	if err := migration.scrubChannels(ctx, tx); err != nil {
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func (migration *scrubEmailChannelTransport) scrubConfigs(ctx context.Context, tx bun.Tx) error {
+	rows := make([]*alertmanagerConfigScrubRow, 0)
+	if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		cfg := make(map[string]json.RawMessage)
+		if err := json.Unmarshal([]byte(row.Config), &cfg); err != nil {
+			migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable config", slog.String("config_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		changed := false
+
+		if globalRaw, ok := cfg["global"]; ok && string(globalRaw) != "null" {
+			global := make(map[string]json.RawMessage)
+			if err := json.Unmarshal(globalRaw, &global); err != nil {
+				migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable global", slog.String("config_id", row.ID), errors.Attr(err))
+				continue
+			}
+			if deleteKeys(global, globalSMTPKeys) {
+				newGlobal, err := json.Marshal(global)
+				if err != nil {
+					migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal global", slog.String("config_id", row.ID), errors.Attr(err))
+					continue
+				}
+				cfg["global"] = newGlobal
+				changed = true
+			}
+		}
+
+		if receiversRaw, ok := cfg["receivers"]; ok && string(receiversRaw) != "null" {
+			receivers := make([]map[string]json.RawMessage, 0)
+			if err := json.Unmarshal(receiversRaw, &receivers); err != nil {
+				migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable receivers", slog.String("config_id", row.ID), errors.Attr(err))
+				continue
+			}
+
+			receiversChanged := false
+			unreadable := false
+			for _, receiver := range receivers {
+				scrubbed, err := scrubEmailConfigs(receiver)
+				if err != nil {
+					migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable email configs", slog.String("config_id", row.ID), errors.Attr(err))
+					unreadable = true
+					break
+				}
+				receiversChanged = receiversChanged || scrubbed
+			}
+			if unreadable {
+				continue
+			}
+
+			if receiversChanged {
+				newReceivers, err := json.Marshal(receivers)
+				if err != nil {
+					migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal receivers", slog.String("config_id", row.ID), errors.Attr(err))
+					continue
+				}
+				cfg["receivers"] = newReceivers
+				changed = true
+			}
+		}
+
+		if !changed {
+			continue
+		}
+
+		newConfig, err := json.Marshal(cfg)
+		if err != nil {
+			migration.logger.WarnContext(ctx, "skipping alertmanager config, cannot marshal config", slog.String("config_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		if _, err := tx.NewUpdate().
+			Model((*alertmanagerConfigScrubRow)(nil)).
+			Set("config = ?", string(newConfig)).
+			Set("hash = ?", fmt.Sprintf("%x", md5.Sum(newConfig))).
+			Where("id = ?", row.ID).
+			Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (migration *scrubEmailChannelTransport) scrubChannels(ctx context.Context, tx bun.Tx) error {
+	rows := make([]*notificationChannelScrubRow, 0)
+	if err := tx.NewSelect().Model(&rows).Where("type = ?", "email").Scan(ctx); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		receiver := make(map[string]json.RawMessage)
+		if err := json.Unmarshal([]byte(row.Data), &receiver); err != nil {
+			migration.logger.WarnContext(ctx, "skipping notification channel with unreadable data", slog.String("channel_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		scrubbed, err := scrubEmailConfigs(receiver)
+		if err != nil {
+			migration.logger.WarnContext(ctx, "skipping notification channel with unreadable email configs", slog.String("channel_id", row.ID), errors.Attr(err))
+			continue
+		}
+		if !scrubbed {
+			continue
+		}
+
+		newData, err := json.Marshal(receiver)
+		if err != nil {
+			migration.logger.WarnContext(ctx, "skipping notification channel, cannot marshal data", slog.String("channel_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		if _, err := tx.NewUpdate().
+			Model((*notificationChannelScrubRow)(nil)).
+			Set("data = ?", string(newData)).
+			Where("id = ?", row.ID).
+			Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func scrubEmailConfigs(receiver map[string]json.RawMessage) (bool, error) {
+	emailConfigsRaw, ok := receiver["email_configs"]
+	if !ok || string(emailConfigsRaw) == "null" {
+		return false, nil
+	}
+
+	emailConfigs := make([]map[string]json.RawMessage, 0)
+	if err := json.Unmarshal(emailConfigsRaw, &emailConfigs); err != nil {
+		return false, err
+	}
+
+	changed := false
+	for _, emailConfig := range emailConfigs {
+		changed = deleteKeys(emailConfig, emailTransportKeys) || changed
+	}
+
+	if !changed {
+		return false, nil
+	}
+
+	newEmailConfigs, err := json.Marshal(emailConfigs)
+	if err != nil {
+		return false, err
+	}
+	receiver["email_configs"] = newEmailConfigs
+
+	return true, nil
+}
+
+func deleteKeys(m map[string]json.RawMessage, keys []string) bool {
+	deleted := false
+	for _, key := range keys {
+		if _, ok := m[key]; ok {
+			delete(m, key)
+			deleted = true
+		}
+	}
+
+	return deleted
+}
+
+func (migration *scrubEmailChannelTransport) Down(context.Context, *bun.DB) error {
+	return nil
+}

pkg/types/alertmanagertypes/channel_test.go

@@ -35,12 +35,8 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"email_configs": []any{map[string]any{
 						"send_resolved": false,
 						"to":            "test@example.com",
-						"from":          "alerts@example.com",
-						"hello":         "localhost",
-						"smarthost":     "smtp.example.com:587",
-						"require_tls":   true,
+						"smarthost":     "",
 						"html":          "{{ template \"email.default.html\" . }}",
-						"tls_config":    map[string]any{"insecure_skip_verify": false},
 						"threading":     map[string]any{},
 					}},
 				},
@@ -63,7 +59,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"slack_configs": []any{map[string]any{
 						"send_resolved": true,
 						"api_url":       "https://slack.com/api/test",
-						"app_url":       "https://slack.com/api/chat.postMessage",
 						"channel":       "#alerts",
 						"callback_id":   "{{ template \"slack.default.callbackid\" . }}",
 						"color":         "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",
@@ -77,12 +72,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"title":         "{{ template \"slack.default.title\" . }}",
 						"title_link":    "{{ template \"slack.default.titlelink\" . }}",
 						"username":      "{{ template \"slack.default.username\" . }}",
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},
@@ -104,7 +93,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"pagerduty_configs": []any{map[string]any{
 						"send_resolved": false,
 						"service_key":   "test",
-						"url":           "https://events.pagerduty.com/v2/enqueue",
 						"client":        "{{ template \"pagerduty.default.client\" . }}",
 						"client_url":    "{{ template \"pagerduty.default.clientURL\" . }}",
 						"description":   "{{ template \"pagerduty.default.description\" .}}",
@@ -116,12 +104,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 							"num_resolved": "{{ .Alerts.Resolved | len }}",
 							"resolved":     "{{ .Alerts.Resolved | toJson }}",
 						},
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},
@@ -148,7 +130,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"pagerduty_configs": []any{map[string]any{
 						"send_resolved": false,
 						"service_key":   "test",
-						"url":           "https://events.pagerduty.com/v2/enqueue",
 						"client":        "{{ template \"pagerduty.default.client\" . }}",
 						"client_url":    "{{ template \"pagerduty.default.clientURL\" . }}",
 						"description":   "{{ template \"pagerduty.default.description\" .}}",
@@ -160,12 +141,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 							"num_resolved": "{{ .Alerts.Resolved | len }}",
 							"resolved":     "{{ .Alerts.Resolved | toJson }}",
 						},
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 				{
@@ -173,7 +148,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"slack_configs": []any{map[string]any{
 						"send_resolved": true,
 						"api_url":       "https://slack.com/api/test",
-						"app_url":       "https://slack.com/api/chat.postMessage",
 						"channel":       "#alerts",
 						"callback_id":   "{{ template \"slack.default.callbackid\" . }}",
 						"color":         "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",
@@ -187,12 +161,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"title":         "{{ template \"slack.default.title\" . }}",
 						"title_link":    "{{ template \"slack.default.titlelink\" . }}",
 						"username":      "{{ template \"slack.default.username\" . }}",
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},

pkg/types/alertmanagertypes/config.go

@@ -117,6 +117,11 @@ func NewConfigFromStoreableConfig(sc *StoreableConfig) (*Config, error) {
 		return nil, err
 	}
 
+	// It must be replaced with an empty, non-nil global, upstream swaps nil for
+	// DefaultGlobalConfig, which would let a path that skips SetGlobalConfig pass
+	// validation and fail silently at delivery instead of failing fast here.
+	alertmanagerConfig.Global = &config.GlobalConfig{}
+
 	return &Config{
 		alertmanagerConfig: alertmanagerConfig,
 		customConfigs:      customConfigs,
@@ -174,7 +179,7 @@ func newConfigFromString(s string) (*config.Config, map[string]customReceiverCon
 	return amConfig, customConfigs, nil
 }
 
-func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverConfigs) []byte {
+func extendedReceivers(c *config.Config, customConfigs map[string]customReceiverConfigs) []*Receiver {
 	receivers := make([]*Receiver, len(c.Receivers))
 	for i := range c.Receivers {
 		base := c.Receivers[i]
@@ -185,7 +190,14 @@ func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverC
 		}
 	}
 
-	b, err := json.Marshal(storedConfig{Config: c, Receivers: receivers})
+	return receivers
+}
+
+func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverConfigs) []byte {
+	persistable := *c
+	persistable.Global = nil
+
+	b, err := json.Marshal(storedConfig{Config: &persistable, Receivers: extendedReceivers(c, customConfigs)})
 	if err != nil {
 		// Taking inspiration from the upstream. This is never expected to happen.
 		return []byte(fmt.Sprintf("<error creating config string: %s>", err))
@@ -206,6 +218,37 @@ func (c *Config) flush() {
 	c.storeableConfig.UpdatedAt = time.Now()
 }
 
+func (c *Config) Resolved() (*Config, error) {
+	raw, err := json.Marshal(storedConfig{Config: c.alertmanagerConfig, Receivers: extendedReceivers(c.alertmanagerConfig, c.customConfigs)})
+	if err != nil {
+		return nil, err
+	}
+
+	alertmanagerConfig, customConfigs, err := newConfigFromString(string(raw))
+	if err != nil {
+		return nil, err
+	}
+
+	if err := alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+		return nil, err
+	}
+
+	storeableConfig := *c.storeableConfig
+	resolved := &Config{
+		alertmanagerConfig: alertmanagerConfig,
+		customConfigs:      customConfigs,
+		storeableConfig:    &storeableConfig,
+	}
+	resolved.applyNativeDefaults()
+
+	return resolved, nil
+}
+
+func (c *Config) validate() error {
+	_, err := c.Resolved()
+	return err
+}
+
 func (c *Config) CopyWithReset() (*Config, error) {
 	newConfig, err := NewDefaultConfig(
 		*c.alertmanagerConfig.Global,
@@ -271,6 +314,15 @@ func (c *Config) StoreableConfig() *StoreableConfig {
 	return c.storeableConfig
 }
 
+func cloneReceiver(receiver *Receiver) (*Receiver, error) {
+	raw, err := json.Marshal(receiver)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewReceiver(string(raw))
+}
+
 func (c *Config) CreateReceiver(receiver *Receiver) error {
 	// check that receiver name is not already used
 	for _, existingReceiver := range c.alertmanagerConfig.Receivers {
@@ -279,16 +331,21 @@ func (c *Config) CreateReceiver(receiver *Receiver) error {
 		}
 	}
 
-	route, err := NewRouteFromReceiver(receiver)
+	owned, err := cloneReceiver(receiver)
+	if err != nil {
+		return err
+	}
+
+	route, err := NewRouteFromReceiver(owned)
 	if err != nil {
 		return err
 	}
 
 	c.alertmanagerConfig.Route.Routes = append(c.alertmanagerConfig.Route.Routes, route)
-	c.alertmanagerConfig.Receivers = append(c.alertmanagerConfig.Receivers, *receiver.Receiver)
-	c.setCustomConfigs(receiver)
+	c.alertmanagerConfig.Receivers = append(c.alertmanagerConfig.Receivers, *owned.Receiver)
+	c.setCustomConfigs(owned)
 
-	if err := c.alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+	if err := c.validate(); err != nil {
 		return err
 	}
 	c.applyNativeDefaults()
@@ -313,16 +370,21 @@ func (c *Config) GetReceiver(name string) (*Receiver, error) {
 }
 
 func (c *Config) UpdateReceiver(receiver *Receiver) error {
+	owned, err := cloneReceiver(receiver)
+	if err != nil {
+		return err
+	}
+
 	// find and update receiver
 	for i, existingReceiver := range c.alertmanagerConfig.Receivers {
-		if existingReceiver.Name == receiver.Name {
-			c.alertmanagerConfig.Receivers[i] = *receiver.Receiver
-			c.setCustomConfigs(receiver)
+		if existingReceiver.Name == owned.Name {
+			c.alertmanagerConfig.Receivers[i] = *owned.Receiver
+			c.setCustomConfigs(owned)
 			break
 		}
 	}
 
-	if err := c.alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+	if err := c.validate(); err != nil {
 		return err
 	}
 	c.applyNativeDefaults()

pkg/types/alertmanagertypes/config_test.go

@@ -330,6 +330,150 @@ func TestSetGlobalConfigPreservesSMTPRequireTLS(t *testing.T) {
 	}
 }
 
+func newSMTPGlobalConfig() GlobalConfig {
+	return GlobalConfig{
+		SMTPFrom:         "alerts@example.com",
+		SMTPHello:        "example.com",
+		SMTPSmarthost:    config.HostPort{Host: "smtp.sendgrid.net", Port: "587"},
+		SMTPAuthUsername: "apikey",
+		SMTPAuthPassword: "operator-secret",
+		SMTPRequireTLS:   true,
+	}
+}
+
+func newEmailTestConfig(t *testing.T) *Config {
+	t.Helper()
+
+	cfg, err := NewDefaultConfig(
+		newSMTPGlobalConfig(),
+		RouteConfig{GroupInterval: time.Minute, GroupWait: time.Minute, RepeatInterval: time.Minute},
+		"1",
+	)
+	require.NoError(t, err)
+
+	receiver, err := NewReceiver(`{"name":"email-receiver","email_configs":[{"to":"team@example.com"}]}`)
+	require.NoError(t, err)
+	require.NoError(t, cfg.CreateReceiver(receiver))
+
+	return cfg
+}
+
+func TestStoreableConfigCarriesNoSMTPSettings(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	raw := cfg.StoreableConfig().Config
+	assert.NotContains(t, raw, "operator-secret")
+	assert.NotContains(t, raw, "smtp.sendgrid.net")
+	assert.NotContains(t, raw, "apikey")
+	assert.NotContains(t, raw, "alerts@example.com")
+
+	assert.Equal(t, "operator-secret", string(cfg.alertmanagerConfig.Global.SMTPAuthPassword))
+}
+
+func TestStoreableConfigCarriesNoGlobal(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	stored := map[string]json.RawMessage{}
+	require.NoError(t, json.Unmarshal([]byte(cfg.StoreableConfig().Config), &stored))
+	assert.NotContains(t, stored, "global")
+}
+
+func TestSetGlobalConfigDoesNotChangeStoreableHash(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+	hash := cfg.StoreableConfig().Hash
+
+	require.NoError(t, cfg.SetGlobalConfig(GlobalConfig{SMTPSmarthost: config.HostPort{Host: "smtp.other.net", Port: "2525"}, SMTPAuthPassword: "rotated-secret"}))
+
+	assert.Equal(t, hash, cfg.StoreableConfig().Hash)
+	assert.NotContains(t, cfg.StoreableConfig().Config, "rotated-secret")
+}
+
+func TestNewConfigFromStoreableConfigDiscardsStoredGlobal(t *testing.T) {
+	stored := &StoreableConfig{
+		Config: `{"global":{"resolve_timeout":"5m","smtp_smarthost":"email-smtp.us-east-1.amazonaws.com:587","smtp_auth_password":"old-secret","slack_api_url":"https://hooks.slack.com/services/T/B/X"},"route":{"receiver":"default-receiver"},"receivers":[{"name":"default-receiver"}]}`,
+		OrgID:  "1",
+	}
+
+	cfg, err := NewConfigFromStoreableConfig(stored)
+	require.NoError(t, err)
+
+	assert.Equal(t, &config.GlobalConfig{}, cfg.alertmanagerConfig.Global)
+}
+
+func TestResolvedFillsEmailTransportFromGlobal(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Len(t, receiver.EmailConfigs, 1)
+
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "team@example.com", got.To)
+	assert.Equal(t, "smtp.sendgrid.net:587", got.Smarthost.String())
+	assert.Equal(t, "alerts@example.com", got.From)
+	assert.Equal(t, "apikey", got.AuthUsername)
+	assert.Equal(t, "operator-secret", string(got.AuthPassword))
+	require.NotNil(t, got.RequireTLS)
+	assert.True(t, *got.RequireTLS)
+
+	stored, err := cfg.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	assert.Empty(t, stored.EmailConfigs[0].Smarthost.String())
+	assert.NotContains(t, cfg.StoreableConfig().Config, "operator-secret")
+}
+
+func TestStaleStoredSMTPSettingsAreReplacedOnLoad(t *testing.T) {
+	stored := &StoreableConfig{
+		Config: `{"global":{"resolve_timeout":"5m","smtp_from":"old@example.com","smtp_hello":"localhost","smtp_smarthost":"email-smtp.us-east-1.amazonaws.com:587","smtp_auth_username":"old-user","smtp_auth_password":"old-secret","smtp_require_tls":true},"route":{"receiver":"default-receiver","group_by":["ruleId"],"routes":[{"receiver":"email-receiver","continue":true,"matchers":["ruleId=~\"-1\""]}],"group_wait":"30s","group_interval":"5m","repeat_interval":"4h"},"receivers":[{"name":"default-receiver"},{"name":"email-receiver","email_configs":[{"send_resolved":false,"to":"team@example.com","from":"old@example.com","hello":"localhost","smarthost":"email-smtp.us-east-1.amazonaws.com:587","auth_username":"old-user","auth_password":"old-secret","require_tls":true}]}]}`,
+		OrgID:  "1",
+	}
+
+	cfg, err := NewConfigFromStoreableConfig(stored)
+	require.NoError(t, err)
+
+	loaded, err := cfg.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Len(t, loaded.EmailConfigs, 1)
+	assert.Empty(t, loaded.EmailConfigs[0].Smarthost.String())
+	assert.Empty(t, string(loaded.EmailConfigs[0].AuthPassword))
+
+	require.NoError(t, cfg.SetGlobalConfig(newSMTPGlobalConfig()))
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "smtp.sendgrid.net:587", got.Smarthost.String())
+	assert.Equal(t, "operator-secret", string(got.AuthPassword))
+	assert.Equal(t, "alerts@example.com", got.From)
+
+	assert.NotContains(t, cfg.StoreableConfig().Config, "old-secret")
+	assert.NotContains(t, cfg.StoreableConfig().Config, "amazonaws.com")
+	assert.NotContains(t, cfg.StoreableConfig().Config, "operator-secret")
+}
+
+func TestCreateReceiverDoesNotMutateCaller(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Equal(t, "smtp.sendgrid.net:587", receiver.EmailConfigs[0].Smarthost.String())
+
+	throwaway, err := cfg.CopyWithReset()
+	require.NoError(t, err)
+	require.NoError(t, throwaway.CreateReceiver(receiver))
+
+	assert.Equal(t, "smtp.sendgrid.net:587", receiver.EmailConfigs[0].Smarthost.String())
+	assert.Equal(t, "operator-secret", string(receiver.EmailConfigs[0].AuthPassword))
+}
+
 // Round-trip: create → serialize → reload → GetReceiver still has the configs.
 func TestConfigPreservesGoogleChatConfigs(t *testing.T) {
 	webhookURL, err := url.Parse("https://chat.googleapis.com/v1/spaces/test/messages")

pkg/types/alertmanagertypes/receiver.go

@@ -37,6 +37,7 @@ func NewReceiver(input string) (*Receiver, error) {
 	if err != nil {
 		return nil, err
 	}
+	stripEmailTransport(withDefaults)
 	receiver.Receiver = withDefaults
 
 	// Extend this block when adding another native notifier type.
@@ -53,6 +54,23 @@ func NewReceiver(input string) (*Receiver, error) {
 	return receiver, nil
 }
 
+func stripEmailTransport(base *config.Receiver) {
+	for _, ec := range base.EmailConfigs {
+		ec.From = ""
+		ec.Hello = ""
+		ec.Smarthost = config.HostPort{}
+		ec.AuthUsername = ""
+		ec.AuthPassword = ""
+		ec.AuthPasswordFile = ""
+		ec.AuthSecret = ""
+		ec.AuthSecretFile = ""
+		ec.AuthIdentity = ""
+		ec.RequireTLS = nil
+		ec.TLSConfig = nil
+		ec.ForceImplicitTLS = nil
+	}
+}
+
 func defaultedBaseReceiver(base *config.Receiver) (*config.Receiver, error) {
 	bytes, err := yaml.Marshal(base)
 	if err != nil {
@@ -102,7 +120,12 @@ func TestReceiver(ctx context.Context, receiver *Receiver, receiverIntegrationsF
 		return err
 	}
 
-	defaultedReceiver, err := testConfig.GetReceiver(receiver.Name)
+	resolvedConfig, err := testConfig.Resolved()
+	if err != nil {
+		return err
+	}
+
+	defaultedReceiver, err := resolvedConfig.GetReceiver(receiver.Name)
 	if err != nil {
 		return err
 	}

pkg/types/alertmanagertypes/receiver_test.go

@@ -46,6 +46,31 @@ func TestNewReceiver(t *testing.T) {
 	}
 }
 
+func TestNewReceiverStripsEmailTransport(t *testing.T) {
+	receiver, err := NewReceiver(`{"name":"email","email_configs":[{"to":"team@example.com","from":"attacker@example.com","hello":"example.com","smarthost":"smtp.example.com:587","auth_username":"user","auth_password":"supersecret","auth_secret":"alsosecret","auth_identity":"id","require_tls":false,"headers":{"Subject":"custom"}}]}`)
+	require.NoError(t, err)
+	require.Len(t, receiver.EmailConfigs, 1)
+
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "team@example.com", got.To)
+	assert.Equal(t, map[string]string{"Subject": "custom"}, got.Headers)
+
+	assert.Empty(t, got.From)
+	assert.Empty(t, got.Hello)
+	assert.Empty(t, got.Smarthost.String())
+	assert.Empty(t, got.AuthUsername)
+	assert.Empty(t, string(got.AuthPassword))
+	assert.Empty(t, string(got.AuthSecret))
+	assert.Empty(t, got.AuthIdentity)
+	assert.Nil(t, got.RequireTLS)
+	assert.Nil(t, got.TLSConfig)
+
+	bytes, err := json.Marshal(receiver)
+	require.NoError(t, err)
+	assert.NotContains(t, string(bytes), "supersecret")
+	assert.NotContains(t, string(bytes), "smtp.example.com")
+}
+
 // Omitted fields fall back to DefaultGoogleChatReceiverConfig.
 func TestNewReceiverGoogleChatAppliesDefaults(t *testing.T) {
 	receiver, err := NewReceiver(`{"name":"googlechat","googlechat_configs":[{"webhook_url":"https://chat.googleapis.com/v1/spaces/test/messages"}]}`)

tests/conftest.py

@@ -24,6 +24,7 @@ pytest_plugins = [
     "fixtures.keycloak",
     "fixtures.idp",
     "fixtures.notification_channel",
+    "fixtures.maildev",
     "fixtures.alerts",
     "fixtures.cloudintegrations",
     "fixtures.jsontypes",

tests/fixtures/alerts.py

@@ -1,11 +1,13 @@
 import base64
 import json
+import re
 import time
 import urllib.parse
 import uuid
 from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
+from urllib.parse import urlparse
 
 import pytest
 import requests
@@ -15,6 +17,7 @@ from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.fs import get_testdata_file_path
 from fixtures.logger import setup_logger
 from fixtures.logs import Logs
+from fixtures.maildev import get_all_mails, verify_email_received
 from fixtures.metrics import Metrics
 from fixtures.traces import Traces
 
@@ -311,3 +314,145 @@ def update_rule_channel_name(rule_data: dict, channel_name: str):
         # loop over all the sepcs and update the channels
         for spec in thresholds["spec"]:
             spec["channels"] = [channel_name]
+
+
+def _is_json_subset(subset, superset) -> bool:
+    """Check if subset is contained within superset recursively.
+    - For dicts: all keys in subset must exist in superset with matching values
+    - For lists: all items in subset must be present in superset
+    - For scalars: exact equality
+    """
+    if isinstance(subset, dict):
+        if not isinstance(superset, dict):
+            return False
+        return all(key in superset and _is_json_subset(value, superset[key]) for key, value in subset.items())
+    if isinstance(subset, list):
+        if not isinstance(superset, list):
+            return False
+        return all(any(_is_json_subset(sub_item, sup_item) for sup_item in superset) for sub_item in subset)
+    if isinstance(subset, re.Pattern):
+        return isinstance(superset, str) and subset.search(superset) is not None
+    return subset == superset
+
+
+def verify_webhook_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    validation_data: dict,
+) -> bool:
+    """Check if wiremock received a request at the given path
+    whose JSON body is a superset of the expected json_body."""
+    path = validation_data["path"]
+    json_body = validation_data["json_body"]
+
+    url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+    try:
+        res = requests.post(url, json={"method": "POST", "url": path}, timeout=10)
+    except requests.exceptions.RequestException:
+        return False
+    if res.status_code != HTTPStatus.OK:
+        return False
+
+    for req in res.json()["requests"]:
+        body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+        if _is_json_subset(json_body, body):
+            return True
+    return False
+
+
+def _check_notification_validation(
+    validation: types.NotificationValidation,
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+) -> bool:
+    """Dispatch a single validation check to the appropriate verifier."""
+    if validation.destination_type == "webhook":
+        return verify_webhook_notification_expectation(notification_channel, validation.validation_data)
+    if validation.destination_type == "email":
+        return verify_email_received(maildev, validation.validation_data)
+    raise ValueError(f"Invalid destination type: {validation.destination_type}")
+
+
+def verify_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    expected_notification: types.AMNotificationExpectation,
+) -> bool:
+    """Poll for expected notifications across webhook and email channels."""
+    time_to_wait = datetime.now() + timedelta(seconds=expected_notification.wait_time_seconds)
+
+    while datetime.now() < time_to_wait:
+        all_found = all(_check_notification_validation(v, notification_channel, maildev) for v in expected_notification.notification_validations)
+
+        if expected_notification.should_notify and all_found:
+            logger.info("All expected notifications found")
+            return True
+
+        time.sleep(1)
+
+    # Timeout reached
+    if not expected_notification.should_notify:
+        # Verify no notifications were received
+        for validation in expected_notification.notification_validations:
+            found = _check_notification_validation(validation, notification_channel, maildev)
+            assert not found, f"Expected no notification but found one for {validation.destination_type} with data {validation.validation_data}"
+        logger.info("No notifications found, as expected")
+        return True
+
+    missing = [v for v in expected_notification.notification_validations if not _check_notification_validation(v, notification_channel, maildev)]
+    assert len(missing) == 0, f"Expected all notifications to be found but missing: {missing}, received: {_received_notifications(notification_channel, maildev, missing)}"
+    return True
+
+
+def _received_notifications(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    missing: list[types.NotificationValidation],
+) -> dict:
+    received = {}
+    if any(v.destination_type == "webhook" for v in missing):
+        webhook_bodies = []
+        for validation in missing:
+            if validation.destination_type != "webhook":
+                continue
+            url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+            try:
+                res = requests.post(url, json={"method": "POST", "url": validation.validation_data["path"]}, timeout=10)
+                webhook_bodies.extend(json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8")) for req in res.json()["requests"])
+            except requests.exceptions.RequestException as exc:
+                webhook_bodies.append(f"<failed to fetch wiremock journal: {exc}>")
+        received["webhook"] = webhook_bodies
+    if any(v.destination_type == "email" for v in missing):
+        received["email"] = get_all_mails(maildev)
+    return received
+
+
+def update_raw_channel_config(
+    channel_config: dict,
+    channel_name: str,
+    notification_channel: types.TestContainerDocker,
+) -> dict:
+    """
+    Updates the channel config to point to the given wiremock
+    notification_channel container to receive notifications.
+    """
+    config = channel_config.copy()
+
+    config["name"] = channel_name
+
+    url_field_map = {
+        "slack_configs": "api_url",
+        "msteamsv2_configs": "webhook_url",
+        "webhook_configs": "url",
+        "pagerduty_configs": "url",
+        "opsgenie_configs": "api_url",
+    }
+
+    for config_key, url_field in url_field_map.items():
+        if config_key in config:
+            for entry in config[config_key]:
+                if url_field in entry:
+                    original_url = entry[url_field]
+                    path = urlparse(original_url).path
+                    entry[url_field] = notification_channel.container_configs["8080"].get(path)
+
+    return config

tests/fixtures/auth.py

@@ -77,7 +77,7 @@ def register_admin(
             timeout=5,
         )
 
-        assert response.status_code == HTTPStatus.OK
+        assert response.status_code == HTTPStatus.OK, f"failed to register admin: {response.status_code} {response.text}"
 
         return types.Operation(name="create_user_admin")
 

tests/fixtures/http.py

@@ -125,13 +125,19 @@ def gateway(
 
 @pytest.fixture(name="make_http_mocks", scope="function")
 def make_http_mocks() -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
+    mocked_containers = []
+
     def _make_http_mocks(container: types.TestContainerDocker, mappings: list[Mapping]) -> None:
         Config.base_url = container.host_configs["8080"].get("/__admin")
 
         for mapping in mappings:
             Mappings.create_mapping(mapping=mapping)
 
+        mocked_containers.append(container)
+
     yield _make_http_mocks
 
-    Mappings.delete_all_mappings()
-    Requests.reset_request_journal()
+    for container in mocked_containers:
+        Config.base_url = container.host_configs["8080"].get("/__admin")
+        Mappings.delete_all_mappings()
+        Requests.reset_request_journal()

tests/fixtures/idp.py

@@ -7,6 +7,7 @@ import pytest
 import requests
 from keycloak import KeycloakAdmin
 from selenium import webdriver
+from selenium.common.exceptions import WebDriverException
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.wait import WebDriverWait
@@ -370,11 +371,20 @@ def idp_login(driver: webdriver.Chrome) -> Callable[[str, str], None]:
         password_field.send_keys(password)
 
         # Click the login button
+        idp_host = urlparse(driver.current_url).netloc
         login_button = wait.until(EC.element_to_be_clickable((By.ID, "kc-login")))
         login_button.click()
 
-        # Wait till kc-login element has vanished from the page, which means that a redirection is taking place.
-        wait.until(EC.invisibility_of_element((By.ID, "kc-login")))
+        # Wait till the browser has left the idp host — not just the login page: keycloak's SAML flow inserts an
+        # auto-submitting interstitial on the idp whose POST is what creates the user in signoz. The button is
+        # re-queried per poll; a mid-navigation WebDriverException (detached node) just retries the poll.
+        def _left_idp(drv: webdriver.Chrome) -> bool:
+            try:
+                return urlparse(drv.current_url).netloc != idp_host and not drv.find_elements(By.ID, "kc-login")
+            except WebDriverException:
+                return False
+
+        wait.until(_left_idp)
 
     return _idp_login
 

tests/fixtures/maildev.py

@@ -0,0 +1,143 @@
+import re
+from http import HTTPStatus
+
+import docker
+import docker.errors
+import pytest
+import requests
+from testcontainers.core.container import DockerContainer, Network
+
+from fixtures import reuse, types
+from fixtures.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+MAILDEV_INCOMING_USER = "apikey"
+MAILDEV_INCOMING_PASS = "integration-smtp-secret"
+
+SMTP_TEST_FROM = "alertmanager@integration.test"
+
+OLD_PROVIDER_SMTP_PASS = "old-provider-smtp-secret"
+NEW_PROVIDER_SMTP_PASS = "new-provider-smtp-secret"
+
+
+def signoz_smtp_env(maildev: "types.TestContainerDocker", password: str = MAILDEV_INCOMING_PASS) -> dict:
+    return {
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__SMARTHOST": f"{maildev.container_configs['1025'].address}:{maildev.container_configs['1025'].port}",
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__FROM": SMTP_TEST_FROM,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__AUTH__USERNAME": MAILDEV_INCOMING_USER,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__AUTH__PASSWORD": password,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__REQUIRE__TLS": "false",
+    }
+
+
+def create_maildev(
+    network: Network,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    cache_key: str = "maildev",
+    incoming_user: str = MAILDEV_INCOMING_USER,
+    incoming_pass: str = MAILDEV_INCOMING_PASS,
+) -> types.TestContainerDocker:
+    def create() -> types.TestContainerDocker:
+        container = DockerContainer(image="maildev/maildev:2.2.1")
+        container.with_env("MAILDEV_INCOMING_USER", incoming_user)
+        container.with_env("MAILDEV_INCOMING_PASS", incoming_pass)
+        container.with_exposed_ports(1025, 1080)
+        container.with_network(network=network)
+        container.start()
+
+        return types.TestContainerDocker(
+            id=container.get_wrapped_container().id,
+            host_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1025),
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1080),
+                ),
+            },
+            container_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_wrapped_container().name,
+                    port=1025,
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_wrapped_container().name,
+                    port=1080,
+                ),
+            },
+        )
+
+    def delete(container: types.TestContainerDocker):
+        client = docker.from_env()
+        try:
+            client.containers.get(container_id=container.id).stop()
+            client.containers.get(container_id=container.id).remove(v=True)
+        except docker.errors.NotFound:
+            logger.info(
+                "Skipping removal of MailDev, MailDev(%s) not found. Maybe it was manually removed?",
+                {"id": container.id},
+            )
+
+    def restore(cache: dict) -> types.TestContainerDocker:
+        return types.TestContainerDocker.from_cache(cache)
+
+    return reuse.wrap(
+        request,
+        pytestconfig,
+        cache_key,
+        lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
+        create,
+        delete,
+        restore,
+    )
+
+
+@pytest.fixture(name="maildev", scope="package")
+def maildev(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig)
+
+
+def get_all_mails(_maildev: types.TestContainerDocker) -> list[dict]:
+    url = _maildev.host_configs["1080"].get("/email")
+    response = requests.get(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to fetch emails from MailDev, status code: {response.status_code}, response: {response.text}"
+
+    def addresses(entries: list[dict]) -> str:
+        return ",".join(sorted(entry.get("address", "") for entry in entries))
+
+    return [
+        {
+            "subject": email.get("subject", ""),
+            "html": email.get("html", ""),
+            "text": email.get("text", ""),
+            "from": addresses(email.get("from", [])),
+            "to": addresses(email.get("to", [])),
+        }
+        for email in response.json()
+    ]
+
+
+def verify_email_received(_maildev: types.TestContainerDocker, filters: dict) -> bool:
+    def matches(expected, actual: str) -> bool:
+        if isinstance(expected, re.Pattern):
+            return expected.search(actual) is not None
+        return expected == actual
+
+    for email in get_all_mails(_maildev):
+        if all(key in email and matches(filter_value, email[key]) for key, filter_value in filters.items()):
+            return True
+    return False
+
+
+def delete_all_mails(_maildev: types.TestContainerDocker) -> None:
+    url = _maildev.host_configs["1080"].get("/email/all")
+    response = requests.delete(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to delete emails from MailDev, status code: {response.status_code}, response: {response.text}"

tests/fixtures/notification_channel.py

@@ -1,3 +1,6 @@
+# pylint: disable=line-too-long
+import json
+import time
 from collections.abc import Callable
 from http import HTTPStatus
 
@@ -11,10 +14,116 @@ from wiremock.testing.testcontainer import WireMockContainer
 from fixtures import reuse, types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.logger import setup_logger
+from fixtures.maildev import MAILDEV_INCOMING_PASS, SMTP_TEST_FROM
 
 logger = setup_logger(__name__)
 
 
+EMAIL_TRANSPORT_KEYS = [
+    "from",
+    "hello",
+    "smarthost",
+    "auth_username",
+    "auth_password",
+    "auth_password_file",
+    "auth_secret",
+    "auth_secret_file",
+    "auth_identity",
+    "require_tls",
+    "tls_config",
+    "force_implicit_tls",
+]
+
+
+def assert_email_channel_payload_clean(payload: str) -> None:
+    receiver = json.loads(payload)
+    for email_config in receiver["email_configs"]:
+        transport_keys = set(email_config.keys()) & set(EMAIL_TRANSPORT_KEYS)
+        transport_keys -= {"smarthost"} if email_config.get("smarthost", "") == "" else set()
+        assert not transport_keys, f"email channel payload carries transport keys {transport_keys}: {payload}"
+
+    assert MAILDEV_INCOMING_PASS not in payload
+    assert SMTP_TEST_FROM not in payload
+
+
+"""
+Default notification channel configs shared across alertmanager tests.
+"""
+slack_default_config = {
+    # channel name configured on runtime
+    "slack_configs": [
+        {
+            "api_url": "services/TEAM_ID/BOT_ID/TOKEN_ID",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+msteams_default_config = {
+    "msteamsv2_configs": [
+        {
+            "webhook_url": "msteams/webhook_url",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+pagerduty_default_config = {
+    "pagerduty_configs": [
+        {
+            "routing_key": "PagerDutyRoutingKey",
+            "url": "v2/enqueue",  # base_url configured on runtime
+            "client": "SigNoz Alert Manager",
+            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+            "description": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n\t{{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n\t {{" "}}(\n\t {{- with .CommonLabels.Remove .GroupLabels.Names }}\n\t\t{{- range $index, $label := .SortedPairs -}}\n\t\t {{ if $index }}, {{ end }}\n\t\t {{- $label.Name }}="{{ $label.Value -}}"\n\t\t{{- end }}\n\t {{- end -}}\n\t )\n\t{{- end }}',
+            "details": {
+                "firing": '{{ template "pagerduty.default.instances" .Alerts.Firing }}',
+                "num_firing": "{{ .Alerts.Firing | len }}",
+                "num_resolved": "{{ .Alerts.Resolved | len }}",
+                "resolved": '{{ template "pagerduty.default.instances" .Alerts.Resolved }}',
+            },
+            "source": "SigNoz Alert Manager",
+            "severity": "{{ (index .Alerts 0).Labels.severity }}",
+        }
+    ],
+}
+
+opsgenie_default_config = {
+    "opsgenie_configs": [
+        {
+            "api_key": "OpsGenieAPIKey",
+            "api_url": "/",  # base_url configured on runtime
+            "description": '{{ if gt (len .Alerts.Firing) 0 -}}\r\n\tAlerts Firing:\r\n\t{{ range .Alerts.Firing }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n\tAlerts Resolved:\r\n\t{{ range .Alerts.Resolved }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}',
+            "priority": '{{ if eq (index .Alerts 0).Labels.severity "critical" }}P1{{ else if eq (index .Alerts 0).Labels.severity "warning" }}P2{{ else if eq (index .Alerts 0).Labels.severity "info" }}P3{{ else }}P4{{ end }}',
+            "message": "{{ .CommonLabels.alertname }}",
+            "details": {},
+        }
+    ],
+}
+
+webhook_default_config = {
+    "webhook_configs": [
+        {
+            "url": "webhook/webhook_url",  # base_url configured on runtime
+        }
+    ],
+}
+
+email_default_config = {
+    "email_configs": [
+        {
+            "to": "test@example.com",
+            "html": '<html><body>{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}</body></html>',
+            "headers": {
+                "Subject": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}'
+            },
+        }
+    ],
+}
+
+
 @pytest.fixture(name="notification_channel", scope="package")
 def notification_channel(
     network: Network,
@@ -67,6 +176,40 @@ def notification_channel(
     )
 
 
+@pytest.fixture(name="create_notification_channel", scope="function")
+def create_notification_channel(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> Callable[[dict], str]:
+    admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    channel_ids = []
+
+    def _create_notification_channel(channel_config: dict) -> str:
+        response = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels"),
+            json=channel_config,
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        assert response.status_code == HTTPStatus.CREATED, f"Failed to create channel, Response: {response.text} Response status: {response.status_code}"
+        channel_id = response.json()["data"]["id"]
+        channel_ids.append(channel_id)
+        return channel_id
+
+    yield _create_notification_channel
+
+    for channel_id in channel_ids:
+        response = requests.delete(
+            signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"),
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        if response.status_code != HTTPStatus.NO_CONTENT:
+            logger.error("Failed to delete channel: %s", {"channel_id": channel_id, "status": response.status_code, "response": response.text})
+
+
 @pytest.fixture(name="create_webhook_notification_channel", scope="function")
 def create_webhook_notification_channel(
     signoz: types.SigNoz,
@@ -103,3 +246,19 @@ def create_webhook_notification_channel(
         return channel_id
 
     return _create_webhook_notification_channel
+
+
+def send_test_notification(signoz: types.SigNoz, token: str, receiver: dict, wait_seconds: int = 90) -> None:
+    deadline = time.time() + wait_seconds
+    last = None
+    while time.time() < deadline:
+        last = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels/test"),
+            json=receiver,
+            headers={"Authorization": f"Bearer {token}"},
+            timeout=30,
+        )
+        if last.status_code == HTTPStatus.NO_CONTENT:
+            return
+        time.sleep(2)
+    raise AssertionError(f"test notification did not succeed within {wait_seconds}s, last response: {last.status_code} {last.text}")

tests/fixtures/types.py

@@ -197,3 +197,40 @@ class AlertTestCase:
     alert_data: list[AlertData]
     # list of alert expectations for the test case
     alert_expectation: AlertExpectation
+
+
+@dataclass(frozen=True)
+class NotificationValidation:
+    # destination type of the notification, either webhook or email
+    # slack, msteams, pagerduty, opsgenie, webhook channels send notifications through webhook
+    # email channels send notifications through email
+    destination_type: Literal["webhook", "email"]
+    # validation data for validating the received notification payload
+    validation_data: dict[str, any]
+
+
+@dataclass(frozen=True)
+class AMNotificationExpectation:
+    # whether we expect any notifications to be fired or not, false when testing downtime scenarios
+    # or don't expect any notifications to be fired in given time period
+    should_notify: bool
+    # seconds to wait for the notifications to be fired, if no
+    # notifications are fired in the expected time, the test will fail
+    wait_time_seconds: int
+    # list of notifications to expect, as a single rule can trigger multiple notifications
+    # spanning across different notifiers
+    notification_validations: list[NotificationValidation]
+
+
+@dataclass(frozen=True)
+class AlertManagerNotificationTestCase:
+    # name of the test case
+    name: str
+    # path to the rule file in testdata directory
+    rule_path: str
+    # list of alert data that will be inserted into the database for the rule to be triggered
+    alert_data: list[AlertData]
+    # configuration for the notification channel
+    channel_config: dict[str, any]
+    # notification expectations for the test case
+    notification_expectation: AMNotificationExpectation

tests/integration/bootstrap/setup.py

@@ -39,5 +39,7 @@ def test_teardown(
     idp: types.TestContainerIDP,  # pylint: disable=unused-argument
     create_user_admin: types.Operation,  # pylint: disable=unused-argument
     migrator: types.Operation,  # pylint: disable=unused-argument
+    maildev: types.TestContainerDocker,  # pylint: disable=unused-argument
+    notification_channel: types.TestContainerDocker,  # pylint: disable=unused-argument
 ) -> None:
     pass

tests/integration/testdata/alertmanager/content_templating/logs_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "User login successful", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Database connection established", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "API request received", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Response sent to client", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }

tests/integration/testdata/alertmanager/content_templating/logs_rule.json

@@ -0,0 +1,69 @@
+{
+  "alert": "content_templating_logs",
+  "ruleType": "threshold_rule",
+  "alertType": "LOGS_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 0,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "logs",
+            "filter": {
+              "expression": "body CONTAINS 'payment failure'"
+            },
+            "aggregations": [
+              {
+                "expression": "count()"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Payment failure spike detected on $service_name",
+    "summary": "Payment failures elevated on $service_name"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/metrics_data.jsonl

@@ -0,0 +1,12 @@
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:01:00+00:00","value":80,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:02:00+00:00","value":95,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:03:00+00:00","value":110,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:04:00+00:00","value":120,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:05:00+00:00","value":125,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:06:00+00:00","value":130,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:07:00+00:00","value":135,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:08:00+00:00","value":140,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:09:00+00:00","value":145,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:10:00+00:00","value":150,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:11:00+00:00","value":155,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:12:00+00:00","value":160,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}

tests/integration/testdata/alertmanager/content_templating/metrics_rule.json

@@ -0,0 +1,72 @@
+{
+  "alert": "content_templating_metrics",
+  "ruleType": "threshold_rule",
+  "alertType": "METRIC_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 100,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "metrics",
+            "aggregations": [
+              {
+                "metricName": "container_memory_bytes_content_templating",
+                "timeAggregation": "avg",
+                "spaceAggregation": "max"
+              }
+            ],
+            "groupBy": [
+              {"name": "namespace", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "pod", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "container", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "node", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "severity", "fieldContext": "attribute", "fieldDataType": "string"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Container $container in pod $pod ($namespace) exceeded memory threshold",
+    "summary": "High container memory in $namespace/$pod"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/traces_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "duration": "PT1.2S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a1", "span_id": "c1b2c3d4e5f6a7b8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "duration": "PT1.4S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a2", "span_id": "c2b3c4d5e6f7a8b9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "duration": "PT1.6S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a3", "span_id": "c3b4c5d6e7f8a9b0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "duration": "PT1.8S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a4", "span_id": "c4b5c6d7e8f9a0b1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "duration": "PT2.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a5", "span_id": "c5b6c7d8e9f0a1b2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "duration": "PT2.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a6", "span_id": "c6b7c8d9e0f1a2b3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "duration": "PT2.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a7", "span_id": "c7b8c9d0e1f2a3b4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "duration": "PT2.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a8", "span_id": "c8b9c0d1e2f3a4b5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "duration": "PT2.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a9", "span_id": "c9b0c1d2e3f4a5b6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "duration": "PT3.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b1", "span_id": "d1c2d3e4f5a6b7c8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "duration": "PT3.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b2", "span_id": "d2c3d4e5f6a7b8c9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "duration": "PT3.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b3", "span_id": "d3c4d5e6f7a8b9c0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "duration": "PT3.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b4", "span_id": "d4c5d6e7f8a9b0c1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "duration": "PT3.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b5", "span_id": "d5c6d7e8f9a0b1c2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "duration": "PT4.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b6", "span_id": "d6c7d8e9f0a1b2c3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "duration": "PT4.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b7", "span_id": "d7c8d9e0f1a2b3c4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "duration": "PT4.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b8", "span_id": "d8c9d0e1f2a3b4c5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "duration": "PT4.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b9", "span_id": "d9c0d1e2f3a4b5c6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "duration": "PT4.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c1", "span_id": "e1d2e3f4a5b6c7d8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "duration": "PT5.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c2", "span_id": "e2d3e4f5a6b7c8d9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }

tests/integration/testdata/alertmanager/content_templating/traces_rule.json

@@ -0,0 +1,71 @@
+{
+  "alert": "content_templating_traces",
+  "ruleType": "threshold_rule",
+  "alertType": "TRACES_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 1,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ],
+          "targetUnit": "s"
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "unit": "ns",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "traces",
+            "filter": {
+              "expression": "http.request.path = '/checkout'"
+            },
+            "aggregations": [
+              {
+                "expression": "p90(duration_nano)"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "p90 latency high on $service_name",
+    "summary": "p90 latency exceeded threshold on $service_name"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/tests/alertmanager/01_channels.py

@@ -0,0 +1,268 @@
+import time
+import uuid
+from collections.abc import Callable
+from http import HTTPStatus
+
+import pytest
+import requests
+from sqlalchemy import text
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.maildev import (
+    MAILDEV_INCOMING_PASS,
+    SMTP_TEST_FROM,
+    delete_all_mails,
+    verify_email_received,
+)
+from fixtures.notification_channel import assert_email_channel_payload_clean, send_test_notification
+
+TIMEOUT = 10
+
+
+CHANNEL_TYPE_CASES = [
+    (
+        "webhook",
+        lambda sink: {"webhook_configs": [{"url": sink.container_configs["8080"].get("/webhook/crud-original"), "send_resolved": True}]},
+        lambda sink: {"webhook_configs": [{"url": sink.container_configs["8080"].get("/webhook/crud-updated"), "send_resolved": True}]},
+        "crud-original",
+        "crud-updated",
+    ),
+    (
+        "slack",
+        lambda sink: {"slack_configs": [{"api_url": sink.container_configs["8080"].get("/services/T/B/X"), "channel": "#crud-original"}]},
+        lambda sink: {"slack_configs": [{"api_url": sink.container_configs["8080"].get("/services/T/B/X"), "channel": "#crud-updated"}]},
+        "#crud-original",
+        "#crud-updated",
+    ),
+    (
+        "pagerduty",
+        lambda sink: {"pagerduty_configs": [{"routing_key": "crud-original-routing-key"}]},
+        lambda sink: {"pagerduty_configs": [{"routing_key": "crud-updated-routing-key"}]},
+        "crud-original-routing-key",
+        "crud-updated-routing-key",
+    ),
+    (
+        "opsgenie",
+        lambda sink: {"opsgenie_configs": [{"api_key": "crud-original-api-key", "message": "{{ .CommonLabels.alertname }}"}]},
+        lambda sink: {"opsgenie_configs": [{"api_key": "crud-updated-api-key", "message": "{{ .CommonLabels.alertname }}"}]},
+        "crud-original-api-key",
+        "crud-updated-api-key",
+    ),
+    (
+        "msteamsv2",
+        lambda sink: {"msteamsv2_configs": [{"webhook_url": sink.container_configs["8080"].get("/msteams/crud-original")}]},
+        lambda sink: {"msteamsv2_configs": [{"webhook_url": sink.container_configs["8080"].get("/msteams/crud-updated")}]},
+        "crud-original",
+        "crud-updated",
+    ),
+    (
+        "email",
+        lambda sink: {"email_configs": [{"to": "crud-original@integration.test"}]},
+        lambda sink: {"email_configs": [{"to": "crud-updated@integration.test"}]},
+        "crud-original@integration.test",
+        "crud-updated@integration.test",
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "channel_type,make_config,make_updated_config,created_marker,updated_marker",
+    CHANNEL_TYPE_CASES,
+    ids=[case[0] for case in CHANNEL_TYPE_CASES],
+)
+def test_channel_crud(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    notification_channel: types.TestContainerDocker,
+    channel_type: str,
+    make_config: Callable[[types.TestContainerDocker], dict],
+    make_updated_config: Callable[[types.TestContainerDocker], dict],
+    created_marker: str,
+    updated_marker: str,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"crud-{channel_type}-{uuid.uuid4()}"
+
+    config = {"name": name, **make_config(notification_channel)}
+    response = requests.post(signoz.self.host_configs["8080"].get("/api/v1/channels"), json=config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.CREATED, response.text
+    created = response.json()["data"]
+    channel_id = created["id"]
+    assert created["name"] == name
+    assert created["type"] == channel_type
+
+    response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    listed = {channel["name"]: channel for channel in response.json()["data"]}
+    assert name in listed
+    assert listed[name]["type"] == channel_type
+
+    response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert created_marker in response.json()["data"]["data"]
+
+    updated_config = {"name": name, **make_updated_config(notification_channel)}
+    response = requests.put(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), json=updated_config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NO_CONTENT, response.text
+
+    response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    data = response.json()["data"]["data"]
+    assert updated_marker in data
+    assert created_marker not in data
+
+    response = requests.delete(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NO_CONTENT, response.text
+
+    response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NOT_FOUND, response.text
+
+
+def test_create_rejects_duplicate_name(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    create_notification_channel: Callable[[dict], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"duplicate-{uuid.uuid4()}"
+
+    create_notification_channel({"name": name, "email_configs": [{"to": "first@integration.test"}]})
+
+    response = requests.post(
+        signoz.self.host_configs["8080"].get("/api/v1/channels"),
+        json={"name": name, "email_configs": [{"to": "second@integration.test"}]},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "unique" in response.text
+
+
+def test_create_rejects_channel_without_configs(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = requests.post(
+        signoz.self.host_configs["8080"].get("/api/v1/channels"),
+        json={"name": f"empty-{uuid.uuid4()}"},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "notification configuration" in response.text
+
+
+def test_update_rejects_name_change(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    create_notification_channel: Callable[[dict], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"rename-{uuid.uuid4()}"
+    channel_id = create_notification_channel({"name": name, "email_configs": [{"to": "rename@integration.test"}]})
+
+    response = requests.put(
+        signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"),
+        json={"name": f"{name}-renamed", "email_configs": [{"to": "rename@integration.test"}]},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "cannot update channel name" in response.text
+
+
+def test_channels_require_authentication(signoz: types.SigNoz) -> None:
+    response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.UNAUTHORIZED, response.text
+
+
+def test_email_channel_never_stores_or_serves_smtp_settings(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    hostile_name = f"hostile-email-{uuid.uuid4()}"
+    hostile_config = {
+        "name": hostile_name,
+        "email_configs": [
+            {
+                "to": "hostile@integration.test",
+                "from": "spoofed@integration.test",
+                "hello": "attacker.test",
+                "smarthost": "smtp.attacker.test:2525",
+                "auth_username": "attacker",
+                "auth_password": "tenant-posted-secret",
+                "require_tls": False,
+                "headers": {"Subject": "hostile subject"},
+            }
+        ],
+    }
+
+    response = requests.post(signoz.self.host_configs["8080"].get("/api/v1/channels"), json=hostile_config, headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.CREATED, response.text
+    created = response.json()["data"]
+    assert_email_channel_payload_clean(created["data"])
+
+    response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/channels/{created['id']}"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    served = response.json()["data"]["data"]
+    assert_email_channel_payload_clean(served)
+    assert "hostile@integration.test" in served
+    assert "hostile subject" in served
+    assert "smtp.attacker.test" not in served
+    assert "tenant-posted-secret" not in served
+
+    response = requests.get(signoz.self.host_configs["8080"].get("/api/v1/channels"), headers={"Authorization": f"Bearer {token}"}, timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert "tenant-posted-secret" not in response.text
+    assert MAILDEV_INCOMING_PASS not in response.text
+
+    with signoz.sqlstore.conn.connect() as conn:
+        stored = conn.execute(
+            text("SELECT data FROM notification_channel WHERE name = :name"),
+            {"name": hostile_name},
+        ).fetchone()
+        assert stored is not None
+        assert_email_channel_payload_clean(stored[0])
+        assert "tenant-posted-secret" not in stored[0]
+
+        configs = conn.execute(text("SELECT config FROM alertmanager_config")).fetchall()
+        assert len(configs) > 0
+        for (config_raw,) in configs:
+            assert MAILDEV_INCOMING_PASS not in config_raw
+            assert "tenant-posted-secret" not in config_raw
+            assert '"smtp_auth_password"' not in config_raw
+            assert '"auth_password"' not in config_raw
+
+
+def test_email_test_channel_delivers_via_env_transport(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    maildev: types.TestContainerDocker,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    delete_all_mails(maildev)
+
+    recipient = f"delivery-{uuid.uuid4()}@integration.test"
+    send_test_notification(
+        signoz,
+        token,
+        {"name": f"delivery-{uuid.uuid4()}", "email_configs": [{"to": recipient}]},
+    )
+
+    deadline = time.time() + 30
+    while time.time() < deadline:
+        if verify_email_received(maildev, {"to": recipient, "from": SMTP_TEST_FROM}):
+            return
+        time.sleep(1)
+    raise AssertionError(f"no email delivered to {recipient} from {SMTP_TEST_FROM}")

tests/integration/tests/alertmanager/02_notifiers.py

@@ -0,0 +1,360 @@
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    email_default_config,
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+logger = setup_logger(__name__)
+
+NOTIFIERS_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="slack_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            # extra wait for alertmanager server setup
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="msteams_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                                        "type": "AdaptiveCard",
+                                        "version": "1.2",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Alerts",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                                "wrap": True,
+                                                "color": "Attention",
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Labels",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "threshold.name",
+                                                        "value": "critical",
+                                                    }
+                                                ],
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Annotations",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "description",
+                                                        "value": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                                    },
+                                                ],
+                                            },
+                                        ],
+                                        "msteams": {"width": "full"},
+                                        "actions": [
+                                            {
+                                                "type": "Action.OpenUrl",
+                                                "title": "View Alert",
+                                            }
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "event_action": "trigger",
+                            "payload": {
+                                "source": "SigNoz Alert Manager",
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Annotations": [
+                                            {"description = This alert is fired when the defined metric (current value": "15) crosses the threshold (10)"},
+                                        ],
+                                        "Labels": [
+                                            "alertname = threshold_above_at_least_once",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "threshold_above_at_least_once",
+                            "details": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "threshold_above_at_least_once",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                        "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "commonAnnotations": {
+                                "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="email_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=email_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="email",
+                    validation_data={
+                        "subject": re.compile(r'\[FIRING:1\] threshold_above_at_least_once for  \(alertname="threshold_above_at_least_once", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'),
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "notifier_test_case",
+    NOTIFIERS_TEST,
+    ids=lambda notifier_test_case: notifier_test_case.name,
+)
+def test_notifier_templating(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    create_alert_rule: Callable[[dict], str],
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    maildev: types.TestContainerDocker,
+    notifier_test_case: types.AlertManagerNotificationTestCase,
+):
+    channel_name = str(uuid.uuid4())
+
+    channel_config = update_raw_channel_config(notifier_test_case.channel_config, channel_name, notification_channel)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    webhook_validations = [v for v in notifier_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    if any(v.destination_type == "email" for v in notifier_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    time.sleep(12)
+
+    insert_alert_data(
+        notifier_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=5),
+    )
+
+    rule_path = get_testdata_file_path(notifier_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        notifier_test_case.notification_expectation,
+    )

tests/integration/tests/alertmanager/03_content_templating.py

@@ -0,0 +1,332 @@
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+logger = setup_logger(__name__)
+
+
+CONTENT_TEMPLATING_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="msteams_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "type": "AdaptiveCard",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": re.compile(
+                                                    r'\[FIRING:1\] content_templating_metrics for  \(alertname="content_templating_metrics", container="checkout", namespace="production", node="ip-10-0-1-23", pod="checkout-7d9c8b5f4-x2k9p", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'
+                                                ),
+                                            },
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "content_templating_metrics",
+                            "details": {
+                                "alertname": "content_templating_metrics",
+                                "container": "checkout",
+                                "namespace": "production",
+                                "node": "ip-10-0-1-23",
+                                "pod": "checkout-7d9c8b5f4-x2k9p",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "payload": {
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Labels": [
+                                            "alertname = content_templating_metrics",
+                                            "container = checkout",
+                                            "namespace = production",
+                                            "node = ip-10-0-1-23",
+                                            "pod = checkout-7d9c8b5f4-x2k9p",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="slack_logs_default_templating",
+        rule_path="alertmanager/content_templating/logs_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="logs",
+                data_path="alertmanager/content_templating/logs_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="slack_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "content_templating_metrics",
+                                        "container": "checkout",
+                                        "namespace": "production",
+                                        "node": "ip-10-0-1-23",
+                                        "pod": "checkout-7d9c8b5f4-x2k9p",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "description": "Container checkout in pod checkout-7d9c8b5f4-x2k9p (production) exceeded memory threshold",
+                                        "summary": "High container memory in production/checkout-7d9c8b5f4-x2k9p",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "content_templating_metrics",
+                                "container": "checkout",
+                                "namespace": "production",
+                                "node": "ip-10-0-1-23",
+                                "pod": "checkout-7d9c8b5f4-x2k9p",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "content_templating_test_case",
+    CONTENT_TEMPLATING_TEST,
+    ids=lambda content_templating_test_case: content_templating_test_case.name,
+)
+def test_content_templating(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    create_alert_rule: Callable[[dict], str],
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    maildev: types.TestContainerDocker,
+    content_templating_test_case: types.AlertManagerNotificationTestCase,
+):
+    channel_name = str(uuid.uuid4())
+
+    channel_config = update_raw_channel_config(content_templating_test_case.channel_config, channel_name, notification_channel)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    webhook_validations = [v for v in content_templating_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    if any(v.destination_type == "email" for v in content_templating_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    time.sleep(12)
+
+    insert_alert_data(
+        content_templating_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=10),
+    )
+
+    rule_path = get_testdata_file_path(content_templating_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        content_templating_test_case.notification_expectation,
+    )

tests/integration/tests/alertmanager/conftest.py

@@ -0,0 +1,35 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.maildev import signoz_smtp_env
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev: types.TestContainerDocker,
+    notification_channel: types.TestContainerDocker,
+) -> types.SigNoz:
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_alertmanager",
+        env_overrides={
+            **signoz_smtp_env(maildev),
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_PAGERDUTY__URL": notification_channel.container_configs["8080"].get("/v2/enqueue"),
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_OPSGENIE__API__URL": notification_channel.container_configs["8080"].get("/"),
+        },
+    )

tests/integration/tests/alertmanagerrotation/01_smtp_rotation.py

@@ -0,0 +1,99 @@
+import time
+import uuid
+from collections.abc import Callable
+from http import HTTPStatus
+
+import docker
+import pytest
+import requests
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, token_getter
+from fixtures.logger import setup_logger
+from fixtures.maildev import (
+    NEW_PROVIDER_SMTP_PASS,
+    SMTP_TEST_FROM,
+    delete_all_mails,
+    get_all_mails,
+    signoz_smtp_env,
+    verify_email_received,
+)
+from fixtures.notification_channel import send_test_notification
+from fixtures.signoz import create_signoz
+
+logger = setup_logger(__name__)
+
+
+def wait_for_email(maildev: types.TestContainerDocker, filters: dict, wait_seconds: int = 30) -> None:
+    deadline = time.time() + wait_seconds
+    while time.time() < deadline:
+        if verify_email_received(maildev, filters):
+            return
+        time.sleep(1)
+    raise AssertionError(f"no email matching {filters} within {wait_seconds}s, inbox: {get_all_mails(maildev)}")
+
+
+def test_smtp_rotation_applies_to_existing_channels(  # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    maildev_old: types.TestContainerDocker,
+    maildev_new: types.TestContainerDocker,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    channel_name = f"rotation-{uuid.uuid4()}"
+    recipient = f"rotation-{uuid.uuid4()}@integration.test"
+
+    response = requests.post(
+        signoz.self.host_configs["8080"].get("/api/v1/channels"),
+        json={"name": channel_name, "email_configs": [{"to": recipient}]},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=10,
+    )
+    assert response.status_code == HTTPStatus.CREATED, response.text
+
+    delete_all_mails(maildev_old)
+    recipient_old_probe = f"probe-old-{uuid.uuid4()}@integration.test"
+    send_test_notification(signoz, token, {"name": f"probe-{uuid.uuid4()}", "email_configs": [{"to": recipient_old_probe}]})
+    wait_for_email(maildev_old, {"to": recipient_old_probe, "from": SMTP_TEST_FROM})
+    logger.info("Delivery through the old provider verified")
+
+    docker.from_env().containers.get(signoz.self.id).stop()
+    logger.info("Stopped signoz running against the old provider")
+
+    signoz_new = create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_smtp_rotation_new",
+        env_overrides=signoz_smtp_env(maildev_new, password=NEW_PROVIDER_SMTP_PASS),
+    )
+    token_new = token_getter(signoz_new)(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = requests.get(
+        signoz_new.self.host_configs["8080"].get("/api/v1/channels"),
+        headers={"Authorization": f"Bearer {token_new}"},
+        timeout=10,
+    )
+    assert response.status_code == HTTPStatus.OK, response.text
+    listed = {channel["name"]: channel for channel in response.json()["data"]}
+    assert channel_name in listed
+
+    delete_all_mails(maildev_new)
+    mails_at_old_provider = len(get_all_mails(maildev_old))
+    recipient_new_probe = f"probe-new-{uuid.uuid4()}@integration.test"
+    send_test_notification(signoz_new, token_new, {"name": f"probe-{uuid.uuid4()}", "email_configs": [{"to": recipient_new_probe}]})
+    wait_for_email(maildev_new, {"to": recipient_new_probe, "from": SMTP_TEST_FROM})
+    assert len(get_all_mails(maildev_old)) == mails_at_old_provider, "old provider must receive nothing after rotation"

tests/integration/tests/alertmanagerrotation/conftest.py

@@ -0,0 +1,40 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.maildev import NEW_PROVIDER_SMTP_PASS, OLD_PROVIDER_SMTP_PASS, create_maildev, signoz_smtp_env
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="maildev_old", scope="package")
+def maildev_old(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig, cache_key="maildev_smtp_old", incoming_pass=OLD_PROVIDER_SMTP_PASS)
+
+
+@pytest.fixture(name="maildev_new", scope="package")
+def maildev_new(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig, cache_key="maildev_smtp_new", incoming_pass=NEW_PROVIDER_SMTP_PASS)
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev_old: types.TestContainerDocker,
+) -> types.SigNoz:
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_smtp_rotation",
+        env_overrides=signoz_smtp_env(maildev_old, password=OLD_PROVIDER_SMTP_PASS),
+    )

tests/pyproject.toml

@@ -75,3 +75,7 @@ ignore = [
 
 [tool.ruff.format]
 # Defaults align with black (double quotes, 4-space indent).
+
+[tool.ruff.lint.per-file-ignores]
+"fixtures/notification_channel.py" = ["E501"]
+"integration/tests/alertmanager/*" = ["E501"]