fix(alertmanager): resolve email SMTP settings from env

.github/workflows/integrationci.yaml

@@ -39,6 +39,8 @@ jobs:
       matrix:
         suite:
           - alerts
+          - alertmanager
+          - alertmanagerrotation
           - basepath
           - callbackauthn
           - cloudintegrations

pkg/alertmanager/alertmanagerserver/server.go

@@ -241,9 +241,12 @@ func (server *Server) PutAlerts(ctx context.Context, postableAlerts alertmanager
 }
 
 func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertmanagertypes.Config) error {
-	config := alertmanagerConfig.AlertmanagerConfig()
+	resolved, err := alertmanagerConfig.Resolved()
+	if err != nil {
+		return err
+	}
+	config := resolved.AlertmanagerConfig()
 
-	var err error
 	// Load SigNoz's alertmanager notification templates from the configured
 	// globs. The upstream default templates (default.tmpl, email.tmpl) are
 	// always loaded from the embedded alertmanager assets inside FromGlobs, so
@@ -275,7 +278,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 			server.logger.InfoContext(ctx, "skipping creation of receiver not referenced by any route", slog.String("receiver", rcv.Name))
 			continue
 		}
-		extendedRcv, err := alertmanagerConfig.GetReceiver(rcv.Name)
+		extendedRcv, err := resolved.GetReceiver(rcv.Name)
 		if err != nil {
 			return err
 		}
@@ -350,7 +353,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 	go server.dispatcher.Run()
 	go server.inhibitor.Run()
 
-	server.alertmanagerConfig = alertmanagerConfig
+	server.alertmanagerConfig = resolved
 	return nil
 }
 

pkg/alertmanager/signozalertmanager/provider.go

@@ -167,6 +167,10 @@ func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgI
 		return err
 	}
 
+	if err := config.SetGlobalConfig(provider.config.Signoz.Global); err != nil {
+		return err
+	}
+
 	if err := config.UpdateReceiver(receiver); err != nil {
 		return err
 	}
@@ -217,6 +221,10 @@ func (provider *provider) CreateChannel(ctx context.Context, orgID string, recei
 		return nil, err
 	}
 
+	if err := config.SetGlobalConfig(provider.config.Signoz.Global); err != nil {
+		return nil, err
+	}
+
 	if err := config.CreateReceiver(receiver); err != nil {
 		return nil, err
 	}

pkg/alertmanager/signozalertmanager/provider_test.go

@@ -0,0 +1,162 @@
+package signozalertmanager
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagerserver"
+	"github.com/SigNoz/signoz/pkg/factory/factorytest"
+	"github.com/SigNoz/signoz/pkg/sqlstore"
+	"github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/prometheus/alertmanager/config"
+)
+
+func newTestSQLStore(t *testing.T) sqlstore.SQLStore {
+	t.Helper()
+
+	store, err := sqlitesqlstore.New(t.Context(), factorytest.NewSettings(), sqlstore.Config{
+		Provider: "sqlite",
+		Connection: sqlstore.ConnectionConfig{
+			MaxOpenConns:    1,
+			MaxConnLifetime: 0,
+		},
+		Sqlite: sqlstore.SqliteConfig{
+			Path:            filepath.Join(t.TempDir(), "test.db"),
+			Mode:            "wal",
+			BusyTimeout:     5 * time.Second,
+			TransactionMode: "deferred",
+		},
+	})
+	require.NoError(t, err)
+
+	_, err = store.BunDB().NewCreateTable().
+		Model((*alertmanagertypes.StoreableConfig)(nil)).
+		IfNotExists().
+		Exec(t.Context())
+	require.NoError(t, err)
+
+	_, err = store.BunDB().ExecContext(t.Context(), "CREATE UNIQUE INDEX IF NOT EXISTS idx_alertmanager_config_org_id ON alertmanager_config(org_id)")
+	require.NoError(t, err)
+
+	_, err = store.BunDB().NewCreateTable().
+		Model((*alertmanagertypes.Channel)(nil)).
+		IfNotExists().
+		Exec(t.Context())
+	require.NoError(t, err)
+
+	return store
+}
+
+func newTestProvider(t *testing.T) (*provider, sqlstore.SQLStore) {
+	t.Helper()
+
+	serverConfig := alertmanagerserver.NewConfig()
+	serverConfig.Global.SMTPFrom = "alerts@example.com"
+	serverConfig.Global.SMTPSmarthost = config.HostPort{Host: "smtp.sendgrid.net", Port: "587"}
+	serverConfig.Global.SMTPAuthUsername = "apikey"
+	serverConfig.Global.SMTPAuthPassword = "operator-secret"
+
+	sqlStore := newTestSQLStore(t)
+
+	p, err := New(
+		factorytest.NewSettings(),
+		alertmanager.Config{
+			Provider: "signoz",
+			Signoz: alertmanager.Signoz{
+				PollInterval: time.Minute,
+				Config:       serverConfig,
+			},
+		},
+		sqlStore,
+		nil,
+		nil,
+		nil,
+	)
+	require.NoError(t, err)
+
+	return p, sqlStore
+}
+
+func requireNoSMTPSettings(t *testing.T, payload string) {
+	t.Helper()
+
+	assert.NotContains(t, payload, "operator-secret")
+	assert.NotContains(t, payload, "smtp.sendgrid.net")
+	assert.NotContains(t, payload, "apikey")
+	assert.NotContains(t, payload, "auth_password")
+	assert.NotContains(t, payload, "smtp_auth_password")
+}
+
+func TestCreateEmailChannelWithStoredConfigWithoutSMTP(t *testing.T) {
+	p, _ := newTestProvider(t)
+	orgID := "test-org-1"
+
+	require.NoError(t, p.SetDefaultConfig(t.Context(), orgID))
+
+	seeded, err := p.GetConfig(t.Context(), orgID)
+	require.NoError(t, err)
+	requireNoSMTPSettings(t, seeded.StoreableConfig().Config)
+
+	receiver, err := alertmanagertypes.NewReceiver(`{"name":"email-receiver","email_configs":[{"to":"team@example.com"}]}`)
+	require.NoError(t, err)
+
+	channel, err := p.CreateChannel(t.Context(), orgID, receiver)
+	require.NoError(t, err)
+	requireNoSMTPSettings(t, channel.Data)
+
+	stored, err := p.GetChannelByID(t.Context(), orgID, channel.ID)
+	require.NoError(t, err)
+	assert.Contains(t, stored.Data, "team@example.com")
+	requireNoSMTPSettings(t, stored.Data)
+
+	cfg, err := p.GetConfig(t.Context(), orgID)
+	require.NoError(t, err)
+	requireNoSMTPSettings(t, cfg.StoreableConfig().Config)
+
+	require.NoError(t, cfg.SetGlobalConfig(p.config.Signoz.Global))
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+	resolvedReceiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Len(t, resolvedReceiver.EmailConfigs, 1)
+	assert.Equal(t, "smtp.sendgrid.net:587", resolvedReceiver.EmailConfigs[0].Smarthost.String())
+	assert.Equal(t, "operator-secret", string(resolvedReceiver.EmailConfigs[0].AuthPassword))
+}
+
+func TestUpdateStampedLegacyEmailChannel(t *testing.T) {
+	p, sqlStore := newTestProvider(t)
+	orgID := "test-org-2"
+
+	require.NoError(t, p.SetDefaultConfig(t.Context(), orgID))
+
+	receiver, err := alertmanagertypes.NewReceiver(`{"name":"email-receiver","email_configs":[{"to":"team@example.com"}]}`)
+	require.NoError(t, err)
+	channel, err := p.CreateChannel(t.Context(), orgID, receiver)
+	require.NoError(t, err)
+
+	stampedData := `{"name":"email-receiver","email_configs":[{"send_resolved":false,"to":"team@example.com","from":"old@example.com","hello":"localhost","smarthost":"email-smtp.us-east-1.amazonaws.com:587","auth_username":"AKIA000","auth_password":"old-ses-secret","require_tls":true}]}`
+	_, err = sqlStore.BunDB().ExecContext(t.Context(), "UPDATE notification_channel SET data = ? WHERE id = ?", stampedData, channel.ID.StringValue())
+	require.NoError(t, err)
+
+	updated, err := alertmanagertypes.NewReceiver(`{"name":"email-receiver","email_configs":[{"to":"new-team@example.com"}]}`)
+	require.NoError(t, err)
+	require.NoError(t, p.UpdateChannelByReceiverAndID(t.Context(), orgID, updated, channel.ID))
+
+	stored, err := p.GetChannelByID(t.Context(), orgID, channel.ID)
+	require.NoError(t, err)
+	assert.Contains(t, stored.Data, "new-team@example.com")
+	assert.NotContains(t, stored.Data, "old-ses-secret")
+	assert.NotContains(t, stored.Data, "amazonaws.com")
+	requireNoSMTPSettings(t, stored.Data)
+
+	cfg, err := p.GetConfig(t.Context(), orgID)
+	require.NoError(t, err)
+	assert.NotContains(t, cfg.StoreableConfig().Config, "old-ses-secret")
+	requireNoSMTPSettings(t, cfg.StoreableConfig().Config)
+}

pkg/signoz/provider.go

@@ -222,6 +222,7 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewAddTagUniqueIndexFactory(sqlstore, sqlschema),
 		sqlmigration.NewAddTelemetryTuplesFactory(sqlstore),
 		sqlmigration.NewAddTagRelationRankFactory(sqlstore, sqlschema),
+		sqlmigration.NewScrubEmailChannelTransportFactory(sqlstore),
 	)
 }
 

pkg/sqlmigration/103_scrub_email_channel_transport.go

@@ -0,0 +1,263 @@
+package sqlmigration
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/sqlstore"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/migrate"
+)
+
+type scrubEmailChannelTransport struct {
+	sqlstore sqlstore.SQLStore
+	logger   *slog.Logger
+}
+
+type alertmanagerConfigScrubRow struct {
+	bun.BaseModel `bun:"table:alertmanager_config"`
+
+	ID     string `bun:"id"`
+	Config string `bun:"config"`
+}
+
+type notificationChannelScrubRow struct {
+	bun.BaseModel `bun:"table:notification_channel"`
+
+	ID   string `bun:"id"`
+	Data string `bun:"data"`
+}
+
+var emailTransportKeys = []string{
+	"from",
+	"hello",
+	"smarthost",
+	"auth_username",
+	"auth_password",
+	"auth_password_file",
+	"auth_secret",
+	"auth_secret_file",
+	"auth_identity",
+	"require_tls",
+	"tls_config",
+	"force_implicit_tls",
+}
+
+var globalSMTPKeys = []string{
+	"smtp_from",
+	"smtp_hello",
+	"smtp_smarthost",
+	"smtp_auth_username",
+	"smtp_auth_password",
+	"smtp_auth_password_file",
+	"smtp_auth_secret",
+	"smtp_auth_secret_file",
+	"smtp_auth_identity",
+	"smtp_require_tls",
+	"smtp_tls_config",
+	"smtp_force_implicit_tls",
+}
+
+func NewScrubEmailChannelTransportFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
+	return factory.NewProviderFactory(
+		factory.MustNewName("scrub_email_channel_transport"),
+		func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
+			return &scrubEmailChannelTransport{sqlstore: sqlstore, logger: ps.Logger}, nil
+		},
+	)
+}
+
+func (migration *scrubEmailChannelTransport) Register(migrations *migrate.Migrations) error {
+	if err := migrations.Register(migration.Up, migration.Down); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (migration *scrubEmailChannelTransport) Up(ctx context.Context, db *bun.DB) error {
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = tx.Rollback()
+	}()
+
+	if err := migration.scrubConfigs(ctx, tx); err != nil {
+		return err
+	}
+
+	if err := migration.scrubChannels(ctx, tx); err != nil {
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func (migration *scrubEmailChannelTransport) scrubConfigs(ctx context.Context, tx bun.Tx) error {
+	rows := make([]*alertmanagerConfigScrubRow, 0)
+	if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		cfg := make(map[string]json.RawMessage)
+		if err := json.Unmarshal([]byte(row.Config), &cfg); err != nil {
+			migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable config", slog.String("config_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		changed := false
+
+		if globalRaw, ok := cfg["global"]; ok && string(globalRaw) != "null" {
+			global := make(map[string]json.RawMessage)
+			if err := json.Unmarshal(globalRaw, &global); err != nil {
+				migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable global", slog.String("config_id", row.ID), errors.Attr(err))
+				continue
+			}
+			if deleteKeys(global, globalSMTPKeys) {
+				newGlobal, err := json.Marshal(global)
+				if err != nil {
+					return err
+				}
+				cfg["global"] = newGlobal
+				changed = true
+			}
+		}
+
+		if receiversRaw, ok := cfg["receivers"]; ok && string(receiversRaw) != "null" {
+			receivers := make([]map[string]json.RawMessage, 0)
+			if err := json.Unmarshal(receiversRaw, &receivers); err != nil {
+				migration.logger.WarnContext(ctx, "skipping alertmanager config with unreadable receivers", slog.String("config_id", row.ID), errors.Attr(err))
+				continue
+			}
+
+			receiversChanged := false
+			for _, receiver := range receivers {
+				scrubbed, err := scrubEmailConfigs(receiver)
+				if err != nil {
+					return err
+				}
+				receiversChanged = receiversChanged || scrubbed
+			}
+
+			if receiversChanged {
+				newReceivers, err := json.Marshal(receivers)
+				if err != nil {
+					return err
+				}
+				cfg["receivers"] = newReceivers
+				changed = true
+			}
+		}
+
+		if !changed {
+			continue
+		}
+
+		newConfig, err := json.Marshal(cfg)
+		if err != nil {
+			return err
+		}
+
+		if _, err := tx.NewUpdate().
+			Model((*alertmanagerConfigScrubRow)(nil)).
+			Set("config = ?", string(newConfig)).
+			Set("hash = ?", fmt.Sprintf("%x", md5.Sum(newConfig))).
+			Where("id = ?", row.ID).
+			Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (migration *scrubEmailChannelTransport) scrubChannels(ctx context.Context, tx bun.Tx) error {
+	rows := make([]*notificationChannelScrubRow, 0)
+	if err := tx.NewSelect().Model(&rows).Where("type = ?", "email").Scan(ctx); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		receiver := make(map[string]json.RawMessage)
+		if err := json.Unmarshal([]byte(row.Data), &receiver); err != nil {
+			migration.logger.WarnContext(ctx, "skipping notification channel with unreadable data", slog.String("channel_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		scrubbed, err := scrubEmailConfigs(receiver)
+		if err != nil {
+			return err
+		}
+		if !scrubbed {
+			continue
+		}
+
+		newData, err := json.Marshal(receiver)
+		if err != nil {
+			return err
+		}
+
+		if _, err := tx.NewUpdate().
+			Model((*notificationChannelScrubRow)(nil)).
+			Set("data = ?", string(newData)).
+			Where("id = ?", row.ID).
+			Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func scrubEmailConfigs(receiver map[string]json.RawMessage) (bool, error) {
+	emailConfigsRaw, ok := receiver["email_configs"]
+	if !ok || string(emailConfigsRaw) == "null" {
+		return false, nil
+	}
+
+	emailConfigs := make([]map[string]json.RawMessage, 0)
+	if err := json.Unmarshal(emailConfigsRaw, &emailConfigs); err != nil {
+		return false, err
+	}
+
+	changed := false
+	for _, emailConfig := range emailConfigs {
+		changed = deleteKeys(emailConfig, emailTransportKeys) || changed
+	}
+
+	if !changed {
+		return false, nil
+	}
+
+	newEmailConfigs, err := json.Marshal(emailConfigs)
+	if err != nil {
+		return false, err
+	}
+	receiver["email_configs"] = newEmailConfigs
+
+	return true, nil
+}
+
+func deleteKeys(m map[string]json.RawMessage, keys []string) bool {
+	deleted := false
+	for _, key := range keys {
+		if _, ok := m[key]; ok {
+			delete(m, key)
+			deleted = true
+		}
+	}
+
+	return deleted
+}
+
+func (migration *scrubEmailChannelTransport) Down(context.Context, *bun.DB) error {
+	return nil
+}

pkg/types/alertmanagertypes/channel_test.go

@@ -35,12 +35,8 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"email_configs": []any{map[string]any{
 						"send_resolved": false,
 						"to":            "test@example.com",
-						"from":          "alerts@example.com",
-						"hello":         "localhost",
-						"smarthost":     "smtp.example.com:587",
-						"require_tls":   true,
+						"smarthost":     "",
 						"html":          "{{ template \"email.default.html\" . }}",
-						"tls_config":    map[string]any{"insecure_skip_verify": false},
 						"threading":     map[string]any{},
 					}},
 				},
@@ -63,7 +59,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"slack_configs": []any{map[string]any{
 						"send_resolved": true,
 						"api_url":       "https://slack.com/api/test",
-						"app_url":       "https://slack.com/api/chat.postMessage",
 						"channel":       "#alerts",
 						"callback_id":   "{{ template \"slack.default.callbackid\" . }}",
 						"color":         "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",
@@ -77,12 +72,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"title":         "{{ template \"slack.default.title\" . }}",
 						"title_link":    "{{ template \"slack.default.titlelink\" . }}",
 						"username":      "{{ template \"slack.default.username\" . }}",
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},
@@ -104,7 +93,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"pagerduty_configs": []any{map[string]any{
 						"send_resolved": false,
 						"service_key":   "test",
-						"url":           "https://events.pagerduty.com/v2/enqueue",
 						"client":        "{{ template \"pagerduty.default.client\" . }}",
 						"client_url":    "{{ template \"pagerduty.default.clientURL\" . }}",
 						"description":   "{{ template \"pagerduty.default.description\" .}}",
@@ -116,12 +104,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 							"num_resolved": "{{ .Alerts.Resolved | len }}",
 							"resolved":     "{{ .Alerts.Resolved | toJson }}",
 						},
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},
@@ -148,7 +130,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"pagerduty_configs": []any{map[string]any{
 						"send_resolved": false,
 						"service_key":   "test",
-						"url":           "https://events.pagerduty.com/v2/enqueue",
 						"client":        "{{ template \"pagerduty.default.client\" . }}",
 						"client_url":    "{{ template \"pagerduty.default.clientURL\" . }}",
 						"description":   "{{ template \"pagerduty.default.description\" .}}",
@@ -160,12 +141,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 							"num_resolved": "{{ .Alerts.Resolved | len }}",
 							"resolved":     "{{ .Alerts.Resolved | toJson }}",
 						},
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 				{
@@ -173,7 +148,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"slack_configs": []any{map[string]any{
 						"send_resolved": true,
 						"api_url":       "https://slack.com/api/test",
-						"app_url":       "https://slack.com/api/chat.postMessage",
 						"channel":       "#alerts",
 						"callback_id":   "{{ template \"slack.default.callbackid\" . }}",
 						"color":         "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",
@@ -187,12 +161,6 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"title":         "{{ template \"slack.default.title\" . }}",
 						"title_link":    "{{ template \"slack.default.titlelink\" . }}",
 						"username":      "{{ template \"slack.default.username\" . }}",
-						"http_config": map[string]any{
-							"tls_config":       map[string]any{"insecure_skip_verify": false},
-							"follow_redirects": true,
-							"enable_http2":     true,
-							"proxy_url":        nil,
-						},
 					}},
 				},
 			},

pkg/types/alertmanagertypes/config.go

@@ -174,7 +174,7 @@ func newConfigFromString(s string) (*config.Config, map[string]customReceiverCon
 	return amConfig, customConfigs, nil
 }
 
-func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverConfigs) []byte {
+func extendedReceivers(c *config.Config, customConfigs map[string]customReceiverConfigs) []*Receiver {
 	receivers := make([]*Receiver, len(c.Receivers))
 	for i := range c.Receivers {
 		base := c.Receivers[i]
@@ -185,7 +185,14 @@ func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverC
 		}
 	}
 
-	b, err := json.Marshal(storedConfig{Config: c, Receivers: receivers})
+	return receivers
+}
+
+func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverConfigs) []byte {
+	persistable := *c
+	persistable.Global = persistableGlobal(c.Global)
+
+	b, err := json.Marshal(storedConfig{Config: &persistable, Receivers: extendedReceivers(c, customConfigs)})
 	if err != nil {
 		// Taking inspiration from the upstream. This is never expected to happen.
 		return []byte(fmt.Sprintf("<error creating config string: %s>", err))
@@ -194,6 +201,28 @@ func newRawFromConfig(c *config.Config, customConfigs map[string]customReceiverC
 	return b
 }
 
+func persistableGlobal(g *config.GlobalConfig) *config.GlobalConfig {
+	if g == nil {
+		return nil
+	}
+
+	stripped := *g
+	stripped.SMTPFrom = ""
+	stripped.SMTPHello = ""
+	stripped.SMTPSmarthost = config.HostPort{}
+	stripped.SMTPAuthUsername = ""
+	stripped.SMTPAuthPassword = ""
+	stripped.SMTPAuthPasswordFile = ""
+	stripped.SMTPAuthSecret = ""
+	stripped.SMTPAuthSecretFile = ""
+	stripped.SMTPAuthIdentity = ""
+	stripped.SMTPRequireTLS = false
+	stripped.SMTPTLSConfig = nil
+	stripped.SMTPForceImplicitTLS = nil
+
+	return &stripped
+}
+
 func newConfigHash(s string) [16]byte {
 	return md5.Sum([]byte(s))
 }
@@ -206,6 +235,37 @@ func (c *Config) flush() {
 	c.storeableConfig.UpdatedAt = time.Now()
 }
 
+func (c *Config) Resolved() (*Config, error) {
+	raw, err := json.Marshal(storedConfig{Config: c.alertmanagerConfig, Receivers: extendedReceivers(c.alertmanagerConfig, c.customConfigs)})
+	if err != nil {
+		return nil, err
+	}
+
+	alertmanagerConfig, customConfigs, err := newConfigFromString(string(raw))
+	if err != nil {
+		return nil, err
+	}
+
+	if err := alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+		return nil, err
+	}
+
+	storeableConfig := *c.storeableConfig
+	resolved := &Config{
+		alertmanagerConfig: alertmanagerConfig,
+		customConfigs:      customConfigs,
+		storeableConfig:    &storeableConfig,
+	}
+	resolved.applyNativeDefaults()
+
+	return resolved, nil
+}
+
+func (c *Config) validate() error {
+	_, err := c.Resolved()
+	return err
+}
+
 func (c *Config) CopyWithReset() (*Config, error) {
 	newConfig, err := NewDefaultConfig(
 		*c.alertmanagerConfig.Global,
@@ -271,6 +331,15 @@ func (c *Config) StoreableConfig() *StoreableConfig {
 	return c.storeableConfig
 }
 
+func cloneReceiver(receiver *Receiver) (*Receiver, error) {
+	raw, err := json.Marshal(receiver)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewReceiver(string(raw))
+}
+
 func (c *Config) CreateReceiver(receiver *Receiver) error {
 	// check that receiver name is not already used
 	for _, existingReceiver := range c.alertmanagerConfig.Receivers {
@@ -279,16 +348,21 @@ func (c *Config) CreateReceiver(receiver *Receiver) error {
 		}
 	}
 
-	route, err := NewRouteFromReceiver(receiver)
+	owned, err := cloneReceiver(receiver)
+	if err != nil {
+		return err
+	}
+
+	route, err := NewRouteFromReceiver(owned)
 	if err != nil {
 		return err
 	}
 
 	c.alertmanagerConfig.Route.Routes = append(c.alertmanagerConfig.Route.Routes, route)
-	c.alertmanagerConfig.Receivers = append(c.alertmanagerConfig.Receivers, *receiver.Receiver)
-	c.setCustomConfigs(receiver)
+	c.alertmanagerConfig.Receivers = append(c.alertmanagerConfig.Receivers, *owned.Receiver)
+	c.setCustomConfigs(owned)
 
-	if err := c.alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+	if err := c.validate(); err != nil {
 		return err
 	}
 	c.applyNativeDefaults()
@@ -313,16 +387,21 @@ func (c *Config) GetReceiver(name string) (*Receiver, error) {
 }
 
 func (c *Config) UpdateReceiver(receiver *Receiver) error {
+	owned, err := cloneReceiver(receiver)
+	if err != nil {
+		return err
+	}
+
 	// find and update receiver
 	for i, existingReceiver := range c.alertmanagerConfig.Receivers {
-		if existingReceiver.Name == receiver.Name {
-			c.alertmanagerConfig.Receivers[i] = *receiver.Receiver
-			c.setCustomConfigs(receiver)
+		if existingReceiver.Name == owned.Name {
+			c.alertmanagerConfig.Receivers[i] = *owned.Receiver
+			c.setCustomConfigs(owned)
 			break
 		}
 	}
 
-	if err := c.alertmanagerConfig.UnmarshalYAML(func(i interface{}) error { return nil }); err != nil {
+	if err := c.validate(); err != nil {
 		return err
 	}
 	c.applyNativeDefaults()

pkg/types/alertmanagertypes/config_test.go

@@ -330,6 +330,120 @@ func TestSetGlobalConfigPreservesSMTPRequireTLS(t *testing.T) {
 	}
 }
 
+func newSMTPGlobalConfig() GlobalConfig {
+	return GlobalConfig{
+		SMTPFrom:         "alerts@example.com",
+		SMTPHello:        "example.com",
+		SMTPSmarthost:    config.HostPort{Host: "smtp.sendgrid.net", Port: "587"},
+		SMTPAuthUsername: "apikey",
+		SMTPAuthPassword: "operator-secret",
+		SMTPRequireTLS:   true,
+	}
+}
+
+func newEmailTestConfig(t *testing.T) *Config {
+	t.Helper()
+
+	cfg, err := NewDefaultConfig(
+		newSMTPGlobalConfig(),
+		RouteConfig{GroupInterval: time.Minute, GroupWait: time.Minute, RepeatInterval: time.Minute},
+		"1",
+	)
+	require.NoError(t, err)
+
+	receiver, err := NewReceiver(`{"name":"email-receiver","email_configs":[{"to":"team@example.com"}]}`)
+	require.NoError(t, err)
+	require.NoError(t, cfg.CreateReceiver(receiver))
+
+	return cfg
+}
+
+func TestStoreableConfigCarriesNoSMTPSettings(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	raw := cfg.StoreableConfig().Config
+	assert.NotContains(t, raw, "operator-secret")
+	assert.NotContains(t, raw, "smtp.sendgrid.net")
+	assert.NotContains(t, raw, "apikey")
+	assert.NotContains(t, raw, "alerts@example.com")
+
+	assert.Equal(t, "operator-secret", string(cfg.alertmanagerConfig.Global.SMTPAuthPassword))
+}
+
+func TestResolvedFillsEmailTransportFromGlobal(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Len(t, receiver.EmailConfigs, 1)
+
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "team@example.com", got.To)
+	assert.Equal(t, "smtp.sendgrid.net:587", got.Smarthost.String())
+	assert.Equal(t, "alerts@example.com", got.From)
+	assert.Equal(t, "apikey", got.AuthUsername)
+	assert.Equal(t, "operator-secret", string(got.AuthPassword))
+	require.NotNil(t, got.RequireTLS)
+	assert.True(t, *got.RequireTLS)
+
+	stored, err := cfg.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	assert.Empty(t, stored.EmailConfigs[0].Smarthost.String())
+	assert.NotContains(t, cfg.StoreableConfig().Config, "operator-secret")
+}
+
+func TestStaleStoredSMTPSettingsAreReplacedOnLoad(t *testing.T) {
+	stored := &StoreableConfig{
+		Config: `{"global":{"resolve_timeout":"5m","smtp_from":"old@example.com","smtp_hello":"localhost","smtp_smarthost":"email-smtp.us-east-1.amazonaws.com:587","smtp_auth_username":"old-user","smtp_auth_password":"old-secret","smtp_require_tls":true},"route":{"receiver":"default-receiver","group_by":["ruleId"],"routes":[{"receiver":"email-receiver","continue":true,"matchers":["ruleId=~\"-1\""]}],"group_wait":"30s","group_interval":"5m","repeat_interval":"4h"},"receivers":[{"name":"default-receiver"},{"name":"email-receiver","email_configs":[{"send_resolved":false,"to":"team@example.com","from":"old@example.com","hello":"localhost","smarthost":"email-smtp.us-east-1.amazonaws.com:587","auth_username":"old-user","auth_password":"old-secret","require_tls":true}]}]}`,
+		OrgID:  "1",
+	}
+
+	cfg, err := NewConfigFromStoreableConfig(stored)
+	require.NoError(t, err)
+
+	loaded, err := cfg.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Len(t, loaded.EmailConfigs, 1)
+	assert.Empty(t, loaded.EmailConfigs[0].Smarthost.String())
+	assert.Empty(t, string(loaded.EmailConfigs[0].AuthPassword))
+
+	require.NoError(t, cfg.SetGlobalConfig(newSMTPGlobalConfig()))
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "smtp.sendgrid.net:587", got.Smarthost.String())
+	assert.Equal(t, "operator-secret", string(got.AuthPassword))
+	assert.Equal(t, "alerts@example.com", got.From)
+
+	assert.NotContains(t, cfg.StoreableConfig().Config, "old-secret")
+	assert.NotContains(t, cfg.StoreableConfig().Config, "amazonaws.com")
+	assert.NotContains(t, cfg.StoreableConfig().Config, "operator-secret")
+}
+
+func TestCreateReceiverDoesNotMutateCaller(t *testing.T) {
+	cfg := newEmailTestConfig(t)
+
+	resolved, err := cfg.Resolved()
+	require.NoError(t, err)
+	receiver, err := resolved.GetReceiver("email-receiver")
+	require.NoError(t, err)
+	require.Equal(t, "smtp.sendgrid.net:587", receiver.EmailConfigs[0].Smarthost.String())
+
+	throwaway, err := cfg.CopyWithReset()
+	require.NoError(t, err)
+	require.NoError(t, throwaway.CreateReceiver(receiver))
+
+	assert.Equal(t, "smtp.sendgrid.net:587", receiver.EmailConfigs[0].Smarthost.String())
+	assert.Equal(t, "operator-secret", string(receiver.EmailConfigs[0].AuthPassword))
+}
+
 // Round-trip: create → serialize → reload → GetReceiver still has the configs.
 func TestConfigPreservesGoogleChatConfigs(t *testing.T) {
 	webhookURL, err := url.Parse("https://chat.googleapis.com/v1/spaces/test/messages")

pkg/types/alertmanagertypes/receiver.go

@@ -37,6 +37,7 @@ func NewReceiver(input string) (*Receiver, error) {
 	if err != nil {
 		return nil, err
 	}
+	stripEmailTransport(withDefaults)
 	receiver.Receiver = withDefaults
 
 	// Extend this block when adding another native notifier type.
@@ -51,6 +52,23 @@ func NewReceiver(input string) (*Receiver, error) {
 	return receiver, nil
 }
 
+func stripEmailTransport(base *config.Receiver) {
+	for _, ec := range base.EmailConfigs {
+		ec.From = ""
+		ec.Hello = ""
+		ec.Smarthost = config.HostPort{}
+		ec.AuthUsername = ""
+		ec.AuthPassword = ""
+		ec.AuthPasswordFile = ""
+		ec.AuthSecret = ""
+		ec.AuthSecretFile = ""
+		ec.AuthIdentity = ""
+		ec.RequireTLS = nil
+		ec.TLSConfig = nil
+		ec.ForceImplicitTLS = nil
+	}
+}
+
 func defaultedBaseReceiver(base *config.Receiver) (*config.Receiver, error) {
 	bytes, err := yaml.Marshal(base)
 	if err != nil {
@@ -100,7 +118,12 @@ func TestReceiver(ctx context.Context, receiver *Receiver, receiverIntegrationsF
 		return err
 	}
 
-	defaultedReceiver, err := testConfig.GetReceiver(receiver.Name)
+	resolvedConfig, err := testConfig.Resolved()
+	if err != nil {
+		return err
+	}
+
+	defaultedReceiver, err := resolvedConfig.GetReceiver(receiver.Name)
 	if err != nil {
 		return err
 	}

pkg/types/alertmanagertypes/receiver_test.go

@@ -46,6 +46,31 @@ func TestNewReceiver(t *testing.T) {
 	}
 }
 
+func TestNewReceiverStripsEmailTransport(t *testing.T) {
+	receiver, err := NewReceiver(`{"name":"email","email_configs":[{"to":"team@example.com","from":"attacker@example.com","hello":"example.com","smarthost":"smtp.example.com:587","auth_username":"user","auth_password":"supersecret","auth_secret":"alsosecret","auth_identity":"id","require_tls":false,"headers":{"Subject":"custom"}}]}`)
+	require.NoError(t, err)
+	require.Len(t, receiver.EmailConfigs, 1)
+
+	got := receiver.EmailConfigs[0]
+	assert.Equal(t, "team@example.com", got.To)
+	assert.Equal(t, map[string]string{"Subject": "custom"}, got.Headers)
+
+	assert.Empty(t, got.From)
+	assert.Empty(t, got.Hello)
+	assert.Empty(t, got.Smarthost.String())
+	assert.Empty(t, got.AuthUsername)
+	assert.Empty(t, string(got.AuthPassword))
+	assert.Empty(t, string(got.AuthSecret))
+	assert.Empty(t, got.AuthIdentity)
+	assert.Nil(t, got.RequireTLS)
+	assert.Nil(t, got.TLSConfig)
+
+	bytes, err := json.Marshal(receiver)
+	require.NoError(t, err)
+	assert.NotContains(t, string(bytes), "supersecret")
+	assert.NotContains(t, string(bytes), "smtp.example.com")
+}
+
 // Omitted fields fall back to DefaultGoogleChatReceiverConfig.
 func TestNewReceiverGoogleChatAppliesDefaults(t *testing.T) {
 	receiver, err := NewReceiver(`{"name":"googlechat","googlechat_configs":[{"webhook_url":"https://chat.googleapis.com/v1/spaces/test/messages"}]}`)

tests/conftest.py

@@ -22,6 +22,7 @@ pytest_plugins = [
     "fixtures.keycloak",
     "fixtures.idp",
     "fixtures.notification_channel",
+    "fixtures.maildev",
     "fixtures.alerts",
     "fixtures.cloudintegrations",
     "fixtures.jsontypes",

tests/fixtures/alerts.py

@@ -1,9 +1,11 @@
 import base64
 import json
+import re
 import time
 from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
+from urllib.parse import urlparse
 
 import pytest
 import requests
@@ -13,6 +15,7 @@ from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.fs import get_testdata_file_path
 from fixtures.logger import setup_logger
 from fixtures.logs import Logs
+from fixtures.maildev import get_all_mails, verify_email_received
 from fixtures.metrics import Metrics
 from fixtures.traces import Traces
 
@@ -218,3 +221,145 @@ def update_rule_channel_name(rule_data: dict, channel_name: str):
         # loop over all the sepcs and update the channels
         for spec in thresholds["spec"]:
             spec["channels"] = [channel_name]
+
+
+def _is_json_subset(subset, superset) -> bool:
+    """Check if subset is contained within superset recursively.
+    - For dicts: all keys in subset must exist in superset with matching values
+    - For lists: all items in subset must be present in superset
+    - For scalars: exact equality
+    """
+    if isinstance(subset, dict):
+        if not isinstance(superset, dict):
+            return False
+        return all(key in superset and _is_json_subset(value, superset[key]) for key, value in subset.items())
+    if isinstance(subset, list):
+        if not isinstance(superset, list):
+            return False
+        return all(any(_is_json_subset(sub_item, sup_item) for sup_item in superset) for sub_item in subset)
+    if isinstance(subset, re.Pattern):
+        return isinstance(superset, str) and subset.search(superset) is not None
+    return subset == superset
+
+
+def verify_webhook_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    validation_data: dict,
+) -> bool:
+    """Check if wiremock received a request at the given path
+    whose JSON body is a superset of the expected json_body."""
+    path = validation_data["path"]
+    json_body = validation_data["json_body"]
+
+    url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+    try:
+        res = requests.post(url, json={"method": "POST", "url": path}, timeout=10)
+    except requests.exceptions.RequestException:
+        return False
+    if res.status_code != HTTPStatus.OK:
+        return False
+
+    for req in res.json()["requests"]:
+        body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+        if _is_json_subset(json_body, body):
+            return True
+    return False
+
+
+def _check_notification_validation(
+    validation: types.NotificationValidation,
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+) -> bool:
+    """Dispatch a single validation check to the appropriate verifier."""
+    if validation.destination_type == "webhook":
+        return verify_webhook_notification_expectation(notification_channel, validation.validation_data)
+    if validation.destination_type == "email":
+        return verify_email_received(maildev, validation.validation_data)
+    raise ValueError(f"Invalid destination type: {validation.destination_type}")
+
+
+def verify_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    expected_notification: types.AMNotificationExpectation,
+) -> bool:
+    """Poll for expected notifications across webhook and email channels."""
+    time_to_wait = datetime.now() + timedelta(seconds=expected_notification.wait_time_seconds)
+
+    while datetime.now() < time_to_wait:
+        all_found = all(_check_notification_validation(v, notification_channel, maildev) for v in expected_notification.notification_validations)
+
+        if expected_notification.should_notify and all_found:
+            logger.info("All expected notifications found")
+            return True
+
+        time.sleep(1)
+
+    # Timeout reached
+    if not expected_notification.should_notify:
+        # Verify no notifications were received
+        for validation in expected_notification.notification_validations:
+            found = _check_notification_validation(validation, notification_channel, maildev)
+            assert not found, f"Expected no notification but found one for {validation.destination_type} with data {validation.validation_data}"
+        logger.info("No notifications found, as expected")
+        return True
+
+    missing = [v for v in expected_notification.notification_validations if not _check_notification_validation(v, notification_channel, maildev)]
+    assert len(missing) == 0, f"Expected all notifications to be found but missing: {missing}, received: {_received_notifications(notification_channel, maildev, missing)}"
+    return True
+
+
+def _received_notifications(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    missing: list[types.NotificationValidation],
+) -> dict:
+    received = {}
+    if any(v.destination_type == "webhook" for v in missing):
+        webhook_bodies = []
+        for validation in missing:
+            if validation.destination_type != "webhook":
+                continue
+            url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+            try:
+                res = requests.post(url, json={"method": "POST", "url": validation.validation_data["path"]}, timeout=10)
+                webhook_bodies.extend(json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8")) for req in res.json()["requests"])
+            except requests.exceptions.RequestException as exc:
+                webhook_bodies.append(f"<failed to fetch wiremock journal: {exc}>")
+        received["webhook"] = webhook_bodies
+    if any(v.destination_type == "email" for v in missing):
+        received["email"] = get_all_mails(maildev)
+    return received
+
+
+def update_raw_channel_config(
+    channel_config: dict,
+    channel_name: str,
+    notification_channel: types.TestContainerDocker,
+) -> dict:
+    """
+    Updates the channel config to point to the given wiremock
+    notification_channel container to receive notifications.
+    """
+    config = channel_config.copy()
+
+    config["name"] = channel_name
+
+    url_field_map = {
+        "slack_configs": "api_url",
+        "msteamsv2_configs": "webhook_url",
+        "webhook_configs": "url",
+        "pagerduty_configs": "url",
+        "opsgenie_configs": "api_url",
+    }
+
+    for config_key, url_field in url_field_map.items():
+        if config_key in config:
+            for entry in config[config_key]:
+                if url_field in entry:
+                    original_url = entry[url_field]
+                    path = urlparse(original_url).path
+                    entry[url_field] = notification_channel.container_configs["8080"].get(path)
+
+    return config

tests/fixtures/auth.py

@@ -77,7 +77,7 @@ def register_admin(
             timeout=5,
         )
 
-        assert response.status_code == HTTPStatus.OK
+        assert response.status_code == HTTPStatus.OK, f"failed to register admin: {response.status_code} {response.text}"
 
         return types.Operation(name="create_user_admin")
 

tests/fixtures/http.py

@@ -124,14 +124,20 @@ def gateway(
 
 
 @pytest.fixture(name="make_http_mocks", scope="function")
-def make_http_mocks() -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
+def make_http_mocks(
+    request: pytest.FixtureRequest,
+) -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
     def _make_http_mocks(container: types.TestContainerDocker, mappings: list[Mapping]) -> None:
         Config.base_url = container.host_configs["8080"].get("/__admin")
 
         for mapping in mappings:
             Mappings.create_mapping(mapping=mapping)
 
-    yield _make_http_mocks
+        def cleanup():
+            Config.base_url = container.host_configs["8080"].get("/__admin")
+            Mappings.delete_all_mappings()
+            Requests.reset_request_journal()
 
-    Mappings.delete_all_mappings()
-    Requests.reset_request_journal()
+        request.addfinalizer(cleanup)
+
+    return _make_http_mocks

tests/fixtures/maildev.py

@@ -0,0 +1,143 @@
+import re
+from http import HTTPStatus
+
+import docker
+import docker.errors
+import pytest
+import requests
+from testcontainers.core.container import DockerContainer, Network
+
+from fixtures import reuse, types
+from fixtures.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+MAILDEV_INCOMING_USER = "apikey"
+MAILDEV_INCOMING_PASS = "integration-smtp-secret"
+
+SMTP_TEST_FROM = "alertmanager@integration.test"
+
+OLD_PROVIDER_SMTP_PASS = "old-provider-smtp-secret"
+NEW_PROVIDER_SMTP_PASS = "new-provider-smtp-secret"
+
+
+def signoz_smtp_env(maildev: "types.TestContainerDocker", password: str = MAILDEV_INCOMING_PASS) -> dict:
+    return {
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__SMARTHOST": f"{maildev.container_configs['1025'].address}:{maildev.container_configs['1025'].port}",
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__FROM": SMTP_TEST_FROM,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__AUTH__USERNAME": MAILDEV_INCOMING_USER,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__AUTH__PASSWORD": password,
+        "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__REQUIRE__TLS": "false",
+    }
+
+
+def create_maildev(
+    network: Network,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    cache_key: str = "maildev",
+    incoming_user: str = MAILDEV_INCOMING_USER,
+    incoming_pass: str = MAILDEV_INCOMING_PASS,
+) -> types.TestContainerDocker:
+    def create() -> types.TestContainerDocker:
+        container = DockerContainer(image="maildev/maildev:2.2.1")
+        container.with_env("MAILDEV_INCOMING_USER", incoming_user)
+        container.with_env("MAILDEV_INCOMING_PASS", incoming_pass)
+        container.with_exposed_ports(1025, 1080)
+        container.with_network(network=network)
+        container.start()
+
+        return types.TestContainerDocker(
+            id=container.get_wrapped_container().id,
+            host_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1025),
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1080),
+                ),
+            },
+            container_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_wrapped_container().name,
+                    port=1025,
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_wrapped_container().name,
+                    port=1080,
+                ),
+            },
+        )
+
+    def delete(container: types.TestContainerDocker):
+        client = docker.from_env()
+        try:
+            client.containers.get(container_id=container.id).stop()
+            client.containers.get(container_id=container.id).remove(v=True)
+        except docker.errors.NotFound:
+            logger.info(
+                "Skipping removal of MailDev, MailDev(%s) not found. Maybe it was manually removed?",
+                {"id": container.id},
+            )
+
+    def restore(cache: dict) -> types.TestContainerDocker:
+        return types.TestContainerDocker.from_cache(cache)
+
+    return reuse.wrap(
+        request,
+        pytestconfig,
+        cache_key,
+        lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
+        create,
+        delete,
+        restore,
+    )
+
+
+@pytest.fixture(name="maildev", scope="package")
+def maildev(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig)
+
+
+def get_all_mails(_maildev: types.TestContainerDocker) -> list[dict]:
+    url = _maildev.host_configs["1080"].get("/email")
+    response = requests.get(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to fetch emails from MailDev, status code: {response.status_code}, response: {response.text}"
+
+    def addresses(entries: list[dict]) -> str:
+        return ",".join(sorted(entry.get("address", "") for entry in entries))
+
+    return [
+        {
+            "subject": email.get("subject", ""),
+            "html": email.get("html", ""),
+            "text": email.get("text", ""),
+            "from": addresses(email.get("from", [])),
+            "to": addresses(email.get("to", [])),
+        }
+        for email in response.json()
+    ]
+
+
+def verify_email_received(_maildev: types.TestContainerDocker, filters: dict) -> bool:
+    def matches(expected, actual: str) -> bool:
+        if isinstance(expected, re.Pattern):
+            return expected.search(actual) is not None
+        return expected == actual
+
+    for email in get_all_mails(_maildev):
+        if all(key in email and matches(filter_value, email[key]) for key, filter_value in filters.items()):
+            return True
+    return False
+
+
+def delete_all_mails(_maildev: types.TestContainerDocker) -> None:
+    url = _maildev.host_configs["1080"].get("/email/all")
+    response = requests.delete(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to delete emails from MailDev, status code: {response.status_code}, response: {response.text}"

tests/fixtures/notification_channel.py

@@ -1,3 +1,5 @@
+# pylint: disable=line-too-long
+import time
 from collections.abc import Callable
 from http import HTTPStatus
 
@@ -15,6 +17,99 @@ from fixtures.logger import setup_logger
 logger = setup_logger(__name__)
 
 
+EMAIL_TRANSPORT_KEYS = [
+    "from",
+    "hello",
+    "smarthost",
+    "auth_username",
+    "auth_password",
+    "auth_password_file",
+    "auth_secret",
+    "auth_secret_file",
+    "auth_identity",
+    "require_tls",
+    "tls_config",
+    "force_implicit_tls",
+]
+
+"""
+Default notification channel configs shared across alertmanager tests.
+"""
+slack_default_config = {
+    # channel name configured on runtime
+    "slack_configs": [
+        {
+            "api_url": "services/TEAM_ID/BOT_ID/TOKEN_ID",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+msteams_default_config = {
+    "msteamsv2_configs": [
+        {
+            "webhook_url": "msteams/webhook_url",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+pagerduty_default_config = {
+    "pagerduty_configs": [
+        {
+            "routing_key": "PagerDutyRoutingKey",
+            "url": "v2/enqueue",  # base_url configured on runtime
+            "client": "SigNoz Alert Manager",
+            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+            "description": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n\t{{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n\t {{" "}}(\n\t {{- with .CommonLabels.Remove .GroupLabels.Names }}\n\t\t{{- range $index, $label := .SortedPairs -}}\n\t\t {{ if $index }}, {{ end }}\n\t\t {{- $label.Name }}="{{ $label.Value -}}"\n\t\t{{- end }}\n\t {{- end -}}\n\t )\n\t{{- end }}',
+            "details": {
+                "firing": '{{ template "pagerduty.default.instances" .Alerts.Firing }}',
+                "num_firing": "{{ .Alerts.Firing | len }}",
+                "num_resolved": "{{ .Alerts.Resolved | len }}",
+                "resolved": '{{ template "pagerduty.default.instances" .Alerts.Resolved }}',
+            },
+            "source": "SigNoz Alert Manager",
+            "severity": "{{ (index .Alerts 0).Labels.severity }}",
+        }
+    ],
+}
+
+opsgenie_default_config = {
+    "opsgenie_configs": [
+        {
+            "api_key": "OpsGenieAPIKey",
+            "api_url": "/",  # base_url configured on runtime
+            "description": '{{ if gt (len .Alerts.Firing) 0 -}}\r\n\tAlerts Firing:\r\n\t{{ range .Alerts.Firing }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n\tAlerts Resolved:\r\n\t{{ range .Alerts.Resolved }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}',
+            "priority": '{{ if eq (index .Alerts 0).Labels.severity "critical" }}P1{{ else if eq (index .Alerts 0).Labels.severity "warning" }}P2{{ else if eq (index .Alerts 0).Labels.severity "info" }}P3{{ else }}P4{{ end }}',
+            "message": "{{ .CommonLabels.alertname }}",
+            "details": {},
+        }
+    ],
+}
+
+webhook_default_config = {
+    "webhook_configs": [
+        {
+            "url": "webhook/webhook_url",  # base_url configured on runtime
+        }
+    ],
+}
+
+email_default_config = {
+    "email_configs": [
+        {
+            "to": "test@example.com",
+            "html": '<html><body>{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}</body></html>',
+            "headers": {
+                "Subject": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}'
+            },
+        }
+    ],
+}
+
+
 @pytest.fixture(name="notification_channel", scope="package")
 def notification_channel(
     network: Network,
@@ -67,6 +162,40 @@ def notification_channel(
     )
 
 
+@pytest.fixture(name="create_notification_channel", scope="function")
+def create_notification_channel(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> Callable[[dict], str]:
+    admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    channel_ids = []
+
+    def _create_notification_channel(channel_config: dict) -> str:
+        response = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels"),
+            json=channel_config,
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        assert response.status_code == HTTPStatus.CREATED, f"Failed to create channel, Response: {response.text} Response status: {response.status_code}"
+        channel_id = response.json()["data"]["id"]
+        channel_ids.append(channel_id)
+        return channel_id
+
+    yield _create_notification_channel
+
+    for channel_id in channel_ids:
+        response = requests.delete(
+            signoz.self.host_configs["8080"].get(f"/api/v1/channels/{channel_id}"),
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        if response.status_code != HTTPStatus.NO_CONTENT:
+            logger.error("Failed to delete channel: %s", {"channel_id": channel_id, "status": response.status_code, "response": response.text})
+
+
 @pytest.fixture(name="create_webhook_notification_channel", scope="function")
 def create_webhook_notification_channel(
     signoz: types.SigNoz,
@@ -103,3 +232,19 @@ def create_webhook_notification_channel(
         return channel_id
 
     return _create_webhook_notification_channel
+
+
+def send_test_notification(signoz: types.SigNoz, token: str, receiver: dict, wait_seconds: int = 90) -> None:
+    deadline = time.time() + wait_seconds
+    last = None
+    while time.time() < deadline:
+        last = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels/test"),
+            json=receiver,
+            headers={"Authorization": f"Bearer {token}"},
+            timeout=30,
+        )
+        if last.status_code == HTTPStatus.NO_CONTENT:
+            return
+        time.sleep(2)
+    raise AssertionError(f"test notification did not succeed within {wait_seconds}s, last response: {last.status_code} {last.text}")

tests/fixtures/types.py

@@ -197,3 +197,40 @@ class AlertTestCase:
     alert_data: list[AlertData]
     # list of alert expectations for the test case
     alert_expectation: AlertExpectation
+
+
+@dataclass(frozen=True)
+class NotificationValidation:
+    # destination type of the notification, either webhook or email
+    # slack, msteams, pagerduty, opsgenie, webhook channels send notifications through webhook
+    # email channels send notifications through email
+    destination_type: Literal["webhook", "email"]
+    # validation data for validating the received notification payload
+    validation_data: dict[str, any]
+
+
+@dataclass(frozen=True)
+class AMNotificationExpectation:
+    # whether we expect any notifications to be fired or not, false when testing downtime scenarios
+    # or don't expect any notifications to be fired in given time period
+    should_notify: bool
+    # seconds to wait for the notifications to be fired, if no
+    # notifications are fired in the expected time, the test will fail
+    wait_time_seconds: int
+    # list of notifications to expect, as a single rule can trigger multiple notifications
+    # spanning across different notifiers
+    notification_validations: list[NotificationValidation]
+
+
+@dataclass(frozen=True)
+class AlertManagerNotificationTestCase:
+    # name of the test case
+    name: str
+    # path to the rule file in testdata directory
+    rule_path: str
+    # list of alert data that will be inserted into the database for the rule to be triggered
+    alert_data: list[AlertData]
+    # configuration for the notification channel
+    channel_config: dict[str, any]
+    # notification expectations for the test case
+    notification_expectation: AMNotificationExpectation

tests/integration/bootstrap/setup.py

@@ -39,5 +39,7 @@ def test_teardown(
     idp: types.TestContainerIDP,  # pylint: disable=unused-argument
     create_user_admin: types.Operation,  # pylint: disable=unused-argument
     migrator: types.Operation,  # pylint: disable=unused-argument
+    maildev: types.TestContainerDocker,  # pylint: disable=unused-argument
+    notification_channel: types.TestContainerDocker,  # pylint: disable=unused-argument
 ) -> None:
     pass

tests/integration/testdata/alertmanager/content_templating/logs_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "User login successful", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Database connection established", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "API request received", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Response sent to client", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }

tests/integration/testdata/alertmanager/content_templating/logs_rule.json

@@ -0,0 +1,69 @@
+{
+  "alert": "content_templating_logs",
+  "ruleType": "threshold_rule",
+  "alertType": "LOGS_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 0,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "logs",
+            "filter": {
+              "expression": "body CONTAINS 'payment failure'"
+            },
+            "aggregations": [
+              {
+                "expression": "count()"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Payment failure spike detected on $service_name",
+    "summary": "Payment failures elevated on $service_name"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/metrics_data.jsonl

@@ -0,0 +1,12 @@
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:01:00+00:00","value":80,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:02:00+00:00","value":95,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:03:00+00:00","value":110,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:04:00+00:00","value":120,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:05:00+00:00","value":125,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:06:00+00:00","value":130,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:07:00+00:00","value":135,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:08:00+00:00","value":140,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:09:00+00:00","value":145,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:10:00+00:00","value":150,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:11:00+00:00","value":155,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:12:00+00:00","value":160,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}

tests/integration/testdata/alertmanager/content_templating/metrics_rule.json

@@ -0,0 +1,72 @@
+{
+  "alert": "content_templating_metrics",
+  "ruleType": "threshold_rule",
+  "alertType": "METRIC_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 100,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "metrics",
+            "aggregations": [
+              {
+                "metricName": "container_memory_bytes_content_templating",
+                "timeAggregation": "avg",
+                "spaceAggregation": "max"
+              }
+            ],
+            "groupBy": [
+              {"name": "namespace", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "pod", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "container", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "node", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "severity", "fieldContext": "attribute", "fieldDataType": "string"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Container $container in pod $pod ($namespace) exceeded memory threshold",
+    "summary": "High container memory in $namespace/$pod"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/traces_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "duration": "PT1.2S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a1", "span_id": "c1b2c3d4e5f6a7b8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "duration": "PT1.4S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a2", "span_id": "c2b3c4d5e6f7a8b9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "duration": "PT1.6S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a3", "span_id": "c3b4c5d6e7f8a9b0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "duration": "PT1.8S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a4", "span_id": "c4b5c6d7e8f9a0b1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "duration": "PT2.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a5", "span_id": "c5b6c7d8e9f0a1b2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "duration": "PT2.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a6", "span_id": "c6b7c8d9e0f1a2b3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "duration": "PT2.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a7", "span_id": "c7b8c9d0e1f2a3b4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "duration": "PT2.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a8", "span_id": "c8b9c0d1e2f3a4b5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "duration": "PT2.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a9", "span_id": "c9b0c1d2e3f4a5b6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "duration": "PT3.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b1", "span_id": "d1c2d3e4f5a6b7c8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "duration": "PT3.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b2", "span_id": "d2c3d4e5f6a7b8c9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "duration": "PT3.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b3", "span_id": "d3c4d5e6f7a8b9c0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "duration": "PT3.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b4", "span_id": "d4c5d6e7f8a9b0c1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "duration": "PT3.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b5", "span_id": "d5c6d7e8f9a0b1c2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "duration": "PT4.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b6", "span_id": "d6c7d8e9f0a1b2c3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "duration": "PT4.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b7", "span_id": "d7c8d9e0f1a2b3c4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "duration": "PT4.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b8", "span_id": "d8c9d0e1f2a3b4c5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "duration": "PT4.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b9", "span_id": "d9c0d1e2f3a4b5c6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "duration": "PT4.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c1", "span_id": "e1d2e3f4a5b6c7d8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "duration": "PT5.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c2", "span_id": "e2d3e4f5a6b7c8d9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }

tests/integration/testdata/alertmanager/content_templating/traces_rule.json

@@ -0,0 +1,71 @@
+{
+  "alert": "content_templating_traces",
+  "ruleType": "threshold_rule",
+  "alertType": "TRACES_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 1,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ],
+          "targetUnit": "s"
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "unit": "ns",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "traces",
+            "filter": {
+              "expression": "http.request.path = '/checkout'"
+            },
+            "aggregations": [
+              {
+                "expression": "p90(duration_nano)"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "p90 latency high on $service_name",
+    "summary": "p90 latency exceeded threshold on $service_name"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/tests/alertmanager/01_channels.py

@@ -0,0 +1,291 @@
+import json
+import time
+import uuid
+from collections.abc import Callable
+from http import HTTPStatus
+
+import pytest
+import requests
+from sqlalchemy import text
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.logger import setup_logger
+from fixtures.maildev import (
+    MAILDEV_INCOMING_PASS,
+    SMTP_TEST_FROM,
+    delete_all_mails,
+    verify_email_received,
+)
+from fixtures.notification_channel import EMAIL_TRANSPORT_KEYS, send_test_notification
+
+logger = setup_logger(__name__)
+
+TIMEOUT = 10
+
+
+def channels_url(signoz: types.SigNoz, path: str = "") -> str:
+    return signoz.self.host_configs["8080"].get(f"/api/v1/channels{path}")
+
+
+def auth_headers(token: str) -> dict:
+    return {"Authorization": f"Bearer {token}"}
+
+
+CHANNEL_TYPE_CASES = [
+    (
+        "webhook",
+        lambda sink: {"webhook_configs": [{"url": sink.container_configs["8080"].get("/webhook/crud-original"), "send_resolved": True}]},
+        lambda sink: {"webhook_configs": [{"url": sink.container_configs["8080"].get("/webhook/crud-updated"), "send_resolved": True}]},
+        "crud-original",
+        "crud-updated",
+    ),
+    (
+        "slack",
+        lambda sink: {"slack_configs": [{"api_url": sink.container_configs["8080"].get("/services/T/B/X"), "channel": "#crud-original"}]},
+        lambda sink: {"slack_configs": [{"api_url": sink.container_configs["8080"].get("/services/T/B/X"), "channel": "#crud-updated"}]},
+        "#crud-original",
+        "#crud-updated",
+    ),
+    (
+        "pagerduty",
+        lambda sink: {"pagerduty_configs": [{"routing_key": "crud-original-routing-key"}]},
+        lambda sink: {"pagerduty_configs": [{"routing_key": "crud-updated-routing-key"}]},
+        "crud-original-routing-key",
+        "crud-updated-routing-key",
+    ),
+    (
+        "opsgenie",
+        lambda sink: {"opsgenie_configs": [{"api_key": "crud-original-api-key", "message": "{{ .CommonLabels.alertname }}"}]},
+        lambda sink: {"opsgenie_configs": [{"api_key": "crud-updated-api-key", "message": "{{ .CommonLabels.alertname }}"}]},
+        "crud-original-api-key",
+        "crud-updated-api-key",
+    ),
+    (
+        "msteamsv2",
+        lambda sink: {"msteamsv2_configs": [{"webhook_url": sink.container_configs["8080"].get("/msteams/crud-original")}]},
+        lambda sink: {"msteamsv2_configs": [{"webhook_url": sink.container_configs["8080"].get("/msteams/crud-updated")}]},
+        "crud-original",
+        "crud-updated",
+    ),
+    (
+        "email",
+        lambda sink: {"email_configs": [{"to": "crud-original@integration.test"}]},
+        lambda sink: {"email_configs": [{"to": "crud-updated@integration.test"}]},
+        "crud-original@integration.test",
+        "crud-updated@integration.test",
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "channel_type,make_config,make_updated_config,created_marker,updated_marker",
+    CHANNEL_TYPE_CASES,
+    ids=[case[0] for case in CHANNEL_TYPE_CASES],
+)
+def test_channel_crud(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    notification_channel: types.TestContainerDocker,
+    channel_type: str,
+    make_config: Callable[[types.TestContainerDocker], dict],
+    make_updated_config: Callable[[types.TestContainerDocker], dict],
+    created_marker: str,
+    updated_marker: str,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"crud-{channel_type}-{uuid.uuid4()}"
+
+    config = {"name": name, **make_config(notification_channel)}
+    response = requests.post(channels_url(signoz), json=config, headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.CREATED, response.text
+    created = response.json()["data"]
+    channel_id = created["id"]
+    assert created["name"] == name
+    assert created["type"] == channel_type
+
+    response = requests.get(channels_url(signoz), headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    listed = {channel["name"]: channel for channel in response.json()["data"]}
+    assert name in listed
+    assert listed[name]["type"] == channel_type
+
+    response = requests.get(channels_url(signoz, f"/{channel_id}"), headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert created_marker in response.json()["data"]["data"]
+
+    updated_config = {"name": name, **make_updated_config(notification_channel)}
+    response = requests.put(channels_url(signoz, f"/{channel_id}"), json=updated_config, headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NO_CONTENT, response.text
+
+    response = requests.get(channels_url(signoz, f"/{channel_id}"), headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    data = response.json()["data"]["data"]
+    assert updated_marker in data
+    assert created_marker not in data
+
+    response = requests.delete(channels_url(signoz, f"/{channel_id}"), headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NO_CONTENT, response.text
+
+    response = requests.get(channels_url(signoz, f"/{channel_id}"), headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.NOT_FOUND, response.text
+
+
+def test_create_rejects_duplicate_name(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    create_notification_channel: Callable[[dict], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"duplicate-{uuid.uuid4()}"
+
+    create_notification_channel({"name": name, "email_configs": [{"to": "first@integration.test"}]})
+
+    response = requests.post(
+        channels_url(signoz),
+        json={"name": name, "email_configs": [{"to": "second@integration.test"}]},
+        headers=auth_headers(token),
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "unique" in response.text
+
+
+def test_create_rejects_channel_without_configs(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = requests.post(
+        channels_url(signoz),
+        json={"name": f"empty-{uuid.uuid4()}"},
+        headers=auth_headers(token),
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "notification configuration" in response.text
+
+
+def test_update_rejects_name_change(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    create_notification_channel: Callable[[dict], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    name = f"rename-{uuid.uuid4()}"
+    channel_id = create_notification_channel({"name": name, "email_configs": [{"to": "rename@integration.test"}]})
+
+    response = requests.put(
+        channels_url(signoz, f"/{channel_id}"),
+        json={"name": f"{name}-renamed", "email_configs": [{"to": "rename@integration.test"}]},
+        headers=auth_headers(token),
+        timeout=TIMEOUT,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "cannot update channel name" in response.text
+
+
+def test_channels_require_authentication(signoz: types.SigNoz) -> None:
+    response = requests.get(channels_url(signoz), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.UNAUTHORIZED, response.text
+
+
+def assert_email_channel_payload_clean(payload: str) -> None:
+    receiver = json.loads(payload)
+    for email_config in receiver["email_configs"]:
+        transport_keys = set(email_config.keys()) & set(EMAIL_TRANSPORT_KEYS)
+        transport_keys -= {"smarthost"} if email_config.get("smarthost", "") == "" else set()
+        assert not transport_keys, f"email channel payload carries transport keys {transport_keys}: {payload}"
+
+    assert MAILDEV_INCOMING_PASS not in payload
+    assert SMTP_TEST_FROM not in payload
+
+
+def test_email_channel_never_stores_or_serves_smtp_settings(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    hostile_name = f"hostile-email-{uuid.uuid4()}"
+    hostile_config = {
+        "name": hostile_name,
+        "email_configs": [
+            {
+                "to": "hostile@integration.test",
+                "from": "spoofed@integration.test",
+                "hello": "attacker.test",
+                "smarthost": "smtp.attacker.test:2525",
+                "auth_username": "attacker",
+                "auth_password": "tenant-posted-secret",
+                "require_tls": False,
+                "headers": {"Subject": "hostile subject"},
+            }
+        ],
+    }
+
+    response = requests.post(channels_url(signoz), json=hostile_config, headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.CREATED, response.text
+    created = response.json()["data"]
+    assert_email_channel_payload_clean(created["data"])
+
+    response = requests.get(channels_url(signoz, f"/{created['id']}"), headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    served = response.json()["data"]["data"]
+    assert_email_channel_payload_clean(served)
+    assert "hostile@integration.test" in served
+    assert "hostile subject" in served
+    assert "smtp.attacker.test" not in served
+    assert "tenant-posted-secret" not in served
+
+    response = requests.get(channels_url(signoz), headers=auth_headers(token), timeout=TIMEOUT)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert "tenant-posted-secret" not in response.text
+    assert MAILDEV_INCOMING_PASS not in response.text
+
+    with signoz.sqlstore.conn.connect() as conn:
+        stored = conn.execute(
+            text("SELECT data FROM notification_channel WHERE name = :name"),
+            {"name": hostile_name},
+        ).fetchone()
+        assert stored is not None
+        assert_email_channel_payload_clean(stored[0])
+        assert "tenant-posted-secret" not in stored[0]
+
+        configs = conn.execute(text("SELECT config FROM alertmanager_config")).fetchall()
+        assert len(configs) > 0
+        for (config_raw,) in configs:
+            assert MAILDEV_INCOMING_PASS not in config_raw
+            assert "tenant-posted-secret" not in config_raw
+            assert '"smtp_auth_password"' not in config_raw
+            assert '"auth_password"' not in config_raw
+
+
+def test_email_test_channel_delivers_via_env_transport(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    maildev: types.TestContainerDocker,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    delete_all_mails(maildev)
+
+    recipient = f"delivery-{uuid.uuid4()}@integration.test"
+    send_test_notification(
+        signoz,
+        token,
+        {"name": f"delivery-{uuid.uuid4()}", "email_configs": [{"to": recipient}]},
+    )
+
+    deadline = time.time() + 30
+    while time.time() < deadline:
+        if verify_email_received(maildev, {"to": recipient, "from": SMTP_TEST_FROM}):
+            return
+        time.sleep(1)
+    raise AssertionError(f"no email delivered to {recipient} from {SMTP_TEST_FROM}")

tests/integration/tests/alertmanager/02_notifiers.py

@@ -0,0 +1,360 @@
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    email_default_config,
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+logger = setup_logger(__name__)
+
+NOTIFIERS_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="slack_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            # extra wait for alertmanager server setup
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="msteams_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                                        "type": "AdaptiveCard",
+                                        "version": "1.2",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Alerts",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                                "wrap": True,
+                                                "color": "Attention",
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Labels",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "threshold.name",
+                                                        "value": "critical",
+                                                    }
+                                                ],
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Annotations",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "description",
+                                                        "value": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                                    },
+                                                ],
+                                            },
+                                        ],
+                                        "msteams": {"width": "full"},
+                                        "actions": [
+                                            {
+                                                "type": "Action.OpenUrl",
+                                                "title": "View Alert",
+                                            }
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "event_action": "trigger",
+                            "payload": {
+                                "source": "SigNoz Alert Manager",
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Annotations": [
+                                            {"description = This alert is fired when the defined metric (current value": "15) crosses the threshold (10)"},
+                                        ],
+                                        "Labels": [
+                                            "alertname = threshold_above_at_least_once",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "threshold_above_at_least_once",
+                            "details": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "threshold_above_at_least_once",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                        "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "commonAnnotations": {
+                                "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="email_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=email_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="email",
+                    validation_data={
+                        "subject": re.compile(r'\[FIRING:1\] threshold_above_at_least_once for  \(alertname="threshold_above_at_least_once", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'),
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "notifier_test_case",
+    NOTIFIERS_TEST,
+    ids=lambda notifier_test_case: notifier_test_case.name,
+)
+def test_notifier_templating(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    create_alert_rule: Callable[[dict], str],
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    maildev: types.TestContainerDocker,
+    notifier_test_case: types.AlertManagerNotificationTestCase,
+):
+    channel_name = str(uuid.uuid4())
+
+    channel_config = update_raw_channel_config(notifier_test_case.channel_config, channel_name, notification_channel)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    webhook_validations = [v for v in notifier_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    if any(v.destination_type == "email" for v in notifier_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    time.sleep(12)
+
+    insert_alert_data(
+        notifier_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=5),
+    )
+
+    rule_path = get_testdata_file_path(notifier_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        notifier_test_case.notification_expectation,
+    )

tests/integration/tests/alertmanager/03_content_templating.py

@@ -0,0 +1,332 @@
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+logger = setup_logger(__name__)
+
+
+CONTENT_TEMPLATING_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="msteams_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "type": "AdaptiveCard",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": re.compile(
+                                                    r'\[FIRING:1\] content_templating_metrics for  \(alertname="content_templating_metrics", container="checkout", namespace="production", node="ip-10-0-1-23", pod="checkout-7d9c8b5f4-x2k9p", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'
+                                                ),
+                                            },
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "content_templating_metrics",
+                            "details": {
+                                "alertname": "content_templating_metrics",
+                                "container": "checkout",
+                                "namespace": "production",
+                                "node": "ip-10-0-1-23",
+                                "pod": "checkout-7d9c8b5f4-x2k9p",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "payload": {
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Labels": [
+                                            "alertname = content_templating_metrics",
+                                            "container = checkout",
+                                            "namespace = production",
+                                            "node = ip-10-0-1-23",
+                                            "pod = checkout-7d9c8b5f4-x2k9p",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="slack_logs_default_templating",
+        rule_path="alertmanager/content_templating/logs_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="logs",
+                data_path="alertmanager/content_templating/logs_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="slack_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "content_templating_metrics",
+                                        "container": "checkout",
+                                        "namespace": "production",
+                                        "node": "ip-10-0-1-23",
+                                        "pod": "checkout-7d9c8b5f4-x2k9p",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "description": "Container checkout in pod checkout-7d9c8b5f4-x2k9p (production) exceeded memory threshold",
+                                        "summary": "High container memory in production/checkout-7d9c8b5f4-x2k9p",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "content_templating_metrics",
+                                "container": "checkout",
+                                "namespace": "production",
+                                "node": "ip-10-0-1-23",
+                                "pod": "checkout-7d9c8b5f4-x2k9p",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "content_templating_test_case",
+    CONTENT_TEMPLATING_TEST,
+    ids=lambda content_templating_test_case: content_templating_test_case.name,
+)
+def test_content_templating(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    create_alert_rule: Callable[[dict], str],
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    maildev: types.TestContainerDocker,
+    content_templating_test_case: types.AlertManagerNotificationTestCase,
+):
+    channel_name = str(uuid.uuid4())
+
+    channel_config = update_raw_channel_config(content_templating_test_case.channel_config, channel_name, notification_channel)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    webhook_validations = [v for v in content_templating_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    if any(v.destination_type == "email" for v in content_templating_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    time.sleep(12)
+
+    insert_alert_data(
+        content_templating_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=10),
+    )
+
+    rule_path = get_testdata_file_path(content_templating_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        content_templating_test_case.notification_expectation,
+    )

tests/integration/tests/alertmanager/conftest.py

@@ -0,0 +1,35 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.maildev import signoz_smtp_env
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev: types.TestContainerDocker,
+    notification_channel: types.TestContainerDocker,
+) -> types.SigNoz:
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_alertmanager",
+        env_overrides={
+            **signoz_smtp_env(maildev),
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_PAGERDUTY__URL": notification_channel.container_configs["8080"].get("/v2/enqueue"),
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_OPSGENIE__API__URL": notification_channel.container_configs["8080"].get("/"),
+        },
+    )

tests/integration/tests/alertmanagerrotation/01_smtp_rotation.py

@@ -0,0 +1,176 @@
+import json
+import time
+import uuid
+from collections.abc import Callable
+from http import HTTPStatus
+
+import docker
+import pytest
+import requests
+from sqlalchemy import text
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, token_getter
+from fixtures.logger import setup_logger
+from fixtures.maildev import (
+    NEW_PROVIDER_SMTP_PASS,
+    SMTP_TEST_FROM,
+    delete_all_mails,
+    get_all_mails,
+    signoz_smtp_env,
+    verify_email_received,
+)
+from fixtures.notification_channel import EMAIL_TRANSPORT_KEYS, send_test_notification
+from fixtures.signoz import create_signoz
+
+logger = setup_logger(__name__)
+
+LEGACY_SMTP_SECRET = "legacy-smtp-secret"
+LEGACY_SMARTHOST = "email-smtp.us-east-1.amazonaws.com:587"
+
+
+def wait_for_email(maildev: types.TestContainerDocker, filters: dict, wait_seconds: int = 30) -> None:
+    deadline = time.time() + wait_seconds
+    while time.time() < deadline:
+        if verify_email_received(maildev, filters):
+            return
+        time.sleep(1)
+    raise AssertionError(f"no email matching {filters} within {wait_seconds}s, inbox: {get_all_mails(maildev)}")
+
+
+def stamp_legacy_transport(sqlstore: types.TestContainerSQL, channel_name: str) -> None:
+    legacy_transport = {
+        "from": "legacy@integration.test",
+        "hello": "localhost",
+        "smarthost": LEGACY_SMARTHOST,
+        "auth_username": "AKIA000LEGACY",
+        "auth_password": LEGACY_SMTP_SECRET,
+        "require_tls": True,
+    }
+
+    with sqlstore.conn.connect() as conn:
+        data = conn.execute(text("SELECT data FROM notification_channel WHERE name = :name"), {"name": channel_name}).fetchone()[0]
+        receiver = json.loads(data)
+        receiver["email_configs"][0].update(legacy_transport)
+        conn.execute(
+            text("UPDATE notification_channel SET data = :data WHERE name = :name"),
+            {"data": json.dumps(receiver), "name": channel_name},
+        )
+
+        rows = conn.execute(text("SELECT id, config FROM alertmanager_config")).fetchall()
+        assert len(rows) == 1
+        config_id, config_raw = rows[0]
+        config = json.loads(config_raw)
+        config.setdefault("global", {}).update(
+            {
+                "smtp_from": "legacy@integration.test",
+                "smtp_hello": "localhost",
+                "smtp_smarthost": LEGACY_SMARTHOST,
+                "smtp_auth_username": "AKIA000LEGACY",
+                "smtp_auth_password": LEGACY_SMTP_SECRET,
+                "smtp_require_tls": True,
+            }
+        )
+        for rcv in config.get("receivers", []):
+            if rcv.get("name") == channel_name:
+                rcv["email_configs"][0].update(legacy_transport)
+        conn.execute(
+            text("UPDATE alertmanager_config SET config = :config WHERE id = :id"),
+            {"config": json.dumps(config), "id": config_id},
+        )
+
+        deleted = conn.execute(text("DELETE FROM migration WHERE name IN ('103', '103_scrub_email_channel_transport')"))
+        assert deleted.rowcount == 1, "expected exactly one scrub migration record"
+
+        conn.commit()
+
+
+def assert_stored_channel_clean(sqlstore: types.TestContainerSQL, channel_name: str, recipient: str) -> None:
+    with sqlstore.conn.connect() as conn:
+        data = conn.execute(text("SELECT data FROM notification_channel WHERE name = :name"), {"name": channel_name}).fetchone()[0]
+        receiver = json.loads(data)
+        email_config = receiver["email_configs"][0]
+        assert email_config["to"] == recipient
+        transport_keys = set(email_config.keys()) & set(EMAIL_TRANSPORT_KEYS)
+        assert not transport_keys, f"stored channel carries transport keys {transport_keys}: {data}"
+        assert LEGACY_SMTP_SECRET not in data
+
+        config_raw = conn.execute(text("SELECT config FROM alertmanager_config")).fetchone()[0]
+        assert LEGACY_SMTP_SECRET not in config_raw
+        assert LEGACY_SMARTHOST not in config_raw
+        assert NEW_PROVIDER_SMTP_PASS not in config_raw
+
+
+def test_smtp_rotation_applies_to_existing_channels(  # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    maildev_old: types.TestContainerDocker,
+    maildev_new: types.TestContainerDocker,
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    channel_name = f"rotation-{uuid.uuid4()}"
+    recipient = f"rotation-{uuid.uuid4()}@integration.test"
+
+    response = requests.post(
+        signoz.self.host_configs["8080"].get("/api/v1/channels"),
+        json={"name": channel_name, "email_configs": [{"to": recipient}]},
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=10,
+    )
+    assert response.status_code == HTTPStatus.CREATED, response.text
+
+    delete_all_mails(maildev_old)
+    recipient_old_probe = f"probe-old-{uuid.uuid4()}@integration.test"
+    send_test_notification(signoz, token, {"name": f"probe-{uuid.uuid4()}", "email_configs": [{"to": recipient_old_probe}]})
+    wait_for_email(maildev_old, {"to": recipient_old_probe, "from": SMTP_TEST_FROM})
+    logger.info("Delivery through the old provider verified")
+
+    docker.from_env().containers.get(signoz.self.id).stop()
+    logger.info("Stopped signoz running against the old provider")
+
+    stamp_legacy_transport(sqlstore, channel_name)
+
+    signoz_new = create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_smtp_rotation_new",
+        env_overrides=signoz_smtp_env(maildev_new, password=NEW_PROVIDER_SMTP_PASS),
+    )
+    token_new = token_getter(signoz_new)(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    assert_stored_channel_clean(sqlstore, channel_name, recipient)
+    with sqlstore.conn.connect() as conn:
+        replayed = conn.execute(text("SELECT count(*) FROM migration WHERE name IN ('103', '103_scrub_email_channel_transport')")).fetchone()[0]
+        assert replayed == 1
+
+    response = requests.get(
+        signoz_new.self.host_configs["8080"].get("/api/v1/channels"),
+        headers={"Authorization": f"Bearer {token_new}"},
+        timeout=10,
+    )
+    assert response.status_code == HTTPStatus.OK, response.text
+    listed = {channel["name"]: channel for channel in response.json()["data"]}
+    assert channel_name in listed
+    assert LEGACY_SMTP_SECRET not in listed[channel_name]["data"]
+    assert LEGACY_SMARTHOST not in listed[channel_name]["data"]
+
+    delete_all_mails(maildev_new)
+    mails_at_old_provider = len(get_all_mails(maildev_old))
+    recipient_new_probe = f"probe-new-{uuid.uuid4()}@integration.test"
+    send_test_notification(signoz_new, token_new, {"name": f"probe-{uuid.uuid4()}", "email_configs": [{"to": recipient_new_probe}]})
+    wait_for_email(maildev_new, {"to": recipient_new_probe, "from": SMTP_TEST_FROM})
+    assert len(get_all_mails(maildev_old)) == mails_at_old_provider, "old provider must receive nothing after rotation"

tests/integration/tests/alertmanagerrotation/conftest.py

@@ -0,0 +1,40 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.maildev import NEW_PROVIDER_SMTP_PASS, OLD_PROVIDER_SMTP_PASS, create_maildev, signoz_smtp_env
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="maildev_old", scope="package")
+def maildev_old(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig, cache_key="maildev_smtp_old", incoming_pass=OLD_PROVIDER_SMTP_PASS)
+
+
+@pytest.fixture(name="maildev_new", scope="package")
+def maildev_new(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    return create_maildev(network, request, pytestconfig, cache_key="maildev_smtp_new", incoming_pass=NEW_PROVIDER_SMTP_PASS)
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev_old: types.TestContainerDocker,
+) -> types.SigNoz:
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_smtp_rotation",
+        env_overrides=signoz_smtp_env(maildev_old, password=OLD_PROVIDER_SMTP_PASS),
+    )

tests/pyproject.toml

@@ -75,3 +75,7 @@ ignore = [
 
 [tool.ruff.format]
 # Defaults align with black (double quotes, 4-space indent).
+
+[tool.ruff.lint.per-file-ignores]
+"fixtures/notification_channel.py" = ["E501"]
+"integration/tests/alertmanager/*" = ["E501"]