feat: added basic html markdown templater

* feat(authn): move identn to factory and config

* feat(authn): add support for enabling identNs

* feat(authn): add support for enabling identNs

.github/CODEOWNERS

@@ -105,6 +105,10 @@ go.mod @therealpandey
 /pkg/modules/authdomain/ @vikrantgupta25
 /pkg/modules/role/ @vikrantgupta25
 
+# IdentN Owners 
+/pkg/identn/ @vikrantgupta25
+/pkg/http/middleware/identn.go @vikrantgupta25
+
 # Integration tests
 
 /tests/integration/ @vikrantgupta25

conf/example.yaml

@@ -321,3 +321,19 @@ user:
     org:
       name: default
       id: 00000000-0000-0000-0000-000000000000
+
+##################### IdentN #####################
+identn:
+  tokenizer:
+    # toggle the identN resolver
+    enabled: true
+    # headers to use for tokenizer identN resolver
+    headers:
+      - Authorization
+      - Sec-WebSocket-Protocol
+  apikey:
+    # toggle the identN resolver
+    enabled: true
+    # headers to use for apikey identN resolver
+    headers:
+      - SIGNOZ-API-KEY

ee/sqlschema/postgressqlschema/provider.go

@@ -223,7 +223,8 @@ SELECT
     i.indisunique AS unique,
     i.indisprimary AS primary,
     a.attname AS column_name,
-    array_position(i.indkey, a.attnum) AS column_position
+    array_position(i.indkey, a.attnum) AS column_position,
+    pg_get_expr(i.indpred, i.indrelid) AS predicate
 FROM
     pg_index i
     LEFT JOIN pg_class ct ON ct.oid = i.indrelid
@@ -246,7 +247,12 @@ ORDER BY index_name, column_position`, string(name))
 		}
 	}()
 
-	uniqueIndicesMap := make(map[string]*sqlschema.UniqueIndex)
+	type indexEntry struct {
+		columns   []sqlschema.ColumnName
+		predicate *string
+	}
+
+	uniqueIndicesMap := make(map[string]*indexEntry)
 	for rows.Next() {
 		var (
 			tableName  string
@@ -256,30 +262,50 @@ ORDER BY index_name, column_position`, string(name))
 			columnName string
 			// starts from 0 and is unused in this function, this is to ensure that the column names are in the correct order
 			columnPosition int
+			predicate      *string
 		)
 
-		if err := rows.Scan(&tableName, &indexName, &unique, &primary, &columnName, &columnPosition); err != nil {
+		if err := rows.Scan(&tableName, &indexName, &unique, &primary, &columnName, &columnPosition, &predicate); err != nil {
 			return nil, err
 		}
 
 		if unique {
 			if _, ok := uniqueIndicesMap[indexName]; !ok {
-				uniqueIndicesMap[indexName] = &sqlschema.UniqueIndex{
-					TableName:   name,
-					ColumnNames: []sqlschema.ColumnName{sqlschema.ColumnName(columnName)},
+				uniqueIndicesMap[indexName] = &indexEntry{
+					columns:   []sqlschema.ColumnName{sqlschema.ColumnName(columnName)},
+					predicate: predicate,
 				}
 			} else {
-				uniqueIndicesMap[indexName].ColumnNames = append(uniqueIndicesMap[indexName].ColumnNames, sqlschema.ColumnName(columnName))
+				uniqueIndicesMap[indexName].columns = append(uniqueIndicesMap[indexName].columns, sqlschema.ColumnName(columnName))
 			}
 		}
 	}
 
 	indices := make([]sqlschema.Index, 0)
-	for indexName, index := range uniqueIndicesMap {
-		if index.Name() == indexName {
-			indices = append(indices, index)
+	for indexName, entry := range uniqueIndicesMap {
+		if entry.predicate != nil {
+			index := &sqlschema.PartialUniqueIndex{
+				TableName:   name,
+				ColumnNames: entry.columns,
+				Where:       *entry.predicate,
+			}
+
+			if index.Name() == indexName {
+				indices = append(indices, index)
+			} else {
+				indices = append(indices, index.Named(indexName))
+			}
 		} else {
-			indices = append(indices, index.Named(indexName))
+			index := &sqlschema.UniqueIndex{
+				TableName:   name,
+				ColumnNames: entry.columns,
+			}
+
+			if index.Name() == indexName {
+				indices = append(indices, index)
+			} else {
+				indices = append(indices, index.Named(indexName))
+			}
 		}
 	}
 

frontend/vite.config.ts

@@ -1,9 +1,8 @@
 import { sentryVitePlugin } from '@sentry/vite-plugin';
 import react from '@vitejs/plugin-react';
-import { readFileSync } from 'fs';
 import { resolve } from 'path';
 import { visualizer } from 'rollup-plugin-visualizer';
-import type { Plugin, UserConfig } from 'vite';
+import type { Plugin, TransformResult, UserConfig } from 'vite';
 import { defineConfig, loadEnv } from 'vite';
 import vitePluginChecker from 'vite-plugin-checker';
 import viteCompression from 'vite-plugin-compression';
@@ -14,15 +13,14 @@ import tsconfigPaths from 'vite-tsconfig-paths';
 function rawMarkdownPlugin(): Plugin {
 	return {
 		name: 'raw-markdown',
-		transform(_, id): any {
-			if (id.endsWith('.md')) {
-				const content = readFileSync(id, 'utf-8');
-				return {
-					code: `export default ${JSON.stringify(content)};`,
-					map: null,
-				};
+		transform(code, id): TransformResult | undefined {
+			if (!id.endsWith('.md')) {
+				return undefined;
 			}
-			return undefined;
+			return {
+				code: `export default ${JSON.stringify(code)};`,
+				map: null,
+			};
 		},
 	};
 }
@@ -71,7 +69,7 @@ export default defineConfig(
 			);
 		}
 
-		if (env.NODE_ENV === 'production') {
+		if (mode === 'production') {
 			plugins.push(
 				ViteImageOptimizer({
 					jpeg: { quality: 80 },
@@ -102,22 +100,25 @@ export default defineConfig(
 			},
 			define: {
 				// TODO: Remove this in favor of import.meta.env
-				'process.env': JSON.stringify({
-					NODE_ENV: mode,
-					FRONTEND_API_ENDPOINT: env.VITE_FRONTEND_API_ENDPOINT,
-					WEBSOCKET_API_ENDPOINT: env.VITE_WEBSOCKET_API_ENDPOINT,
-					PYLON_APP_ID: env.VITE_PYLON_APP_ID,
-					PYLON_IDENTITY_SECRET: env.VITE_PYLON_IDENTITY_SECRET,
-					APPCUES_APP_ID: env.VITE_APPCUES_APP_ID,
-					POSTHOG_KEY: env.VITE_POSTHOG_KEY,
-					SENTRY_AUTH_TOKEN: env.VITE_SENTRY_AUTH_TOKEN,
-					SENTRY_ORG: env.VITE_SENTRY_ORG,
-					SENTRY_PROJECT_ID: env.VITE_SENTRY_PROJECT_ID,
-					SENTRY_DSN: env.VITE_SENTRY_DSN,
-					TUNNEL_URL: env.VITE_TUNNEL_URL,
-					TUNNEL_DOMAIN: env.VITE_TUNNEL_DOMAIN,
-					DOCS_BASE_URL: env.VITE_DOCS_BASE_URL,
-				}),
+				'process.env.NODE_ENV': JSON.stringify(mode),
+				'process.env.FRONTEND_API_ENDPOINT': JSON.stringify(
+					env.VITE_FRONTEND_API_ENDPOINT,
+				),
+				'process.env.WEBSOCKET_API_ENDPOINT': JSON.stringify(
+					env.VITE_WEBSOCKET_API_ENDPOINT,
+				),
+				'process.env.PYLON_APP_ID': JSON.stringify(env.VITE_PYLON_APP_ID),
+				'process.env.PYLON_IDENTITY_SECRET': JSON.stringify(
+					env.VITE_PYLON_IDENTITY_SECRET,
+				),
+				'process.env.APPCUES_APP_ID': JSON.stringify(env.VITE_APPCUES_APP_ID),
+				'process.env.POSTHOG_KEY': JSON.stringify(env.VITE_POSTHOG_KEY),
+				'process.env.SENTRY_ORG': JSON.stringify(env.VITE_SENTRY_ORG),
+				'process.env.SENTRY_PROJECT_ID': JSON.stringify(env.VITE_SENTRY_PROJECT_ID),
+				'process.env.SENTRY_DSN': JSON.stringify(env.VITE_SENTRY_DSN),
+				'process.env.TUNNEL_URL': JSON.stringify(env.VITE_TUNNEL_URL),
+				'process.env.TUNNEL_DOMAIN': JSON.stringify(env.VITE_TUNNEL_DOMAIN),
+				'process.env.DOCS_BASE_URL': JSON.stringify(env.VITE_DOCS_BASE_URL),
 			},
 			build: {
 				sourcemap: true,

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/coreos/go-oidc/v3 v3.17.0
 	github.com/dgraph-io/ristretto/v2 v2.3.0
 	github.com/dustin/go-humanize v1.0.1
+	github.com/emersion/go-smtp v0.24.0
 	github.com/gin-gonic/gin v1.11.0
 	github.com/go-co-op/gocron v1.30.1
 	github.com/go-openapi/runtime v0.29.2
@@ -64,6 +65,7 @@ require (
 	github.com/uptrace/bun/dialect/pgdialect v1.2.9
 	github.com/uptrace/bun/dialect/sqlitedialect v1.2.9
 	github.com/uptrace/bun/extra/bunotel v1.2.9
+	github.com/yuin/goldmark v1.7.16
 	go.opentelemetry.io/collector/confmap v1.51.0
 	go.opentelemetry.io/collector/otelcol v0.144.0
 	go.opentelemetry.io/collector/pdata v1.51.0
@@ -108,6 +110,7 @@ require (
 	github.com/bytedance/gopkg v0.1.3 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
+	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/go-openapi/swag/cmdutils v0.25.4 // indirect
 	github.com/go-openapi/swag/conv v0.25.4 // indirect
@@ -161,7 +164,7 @@ require (
 	github.com/ClickHouse/ch-go v0.67.0 // indirect
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Yiling-J/theine-go v0.6.2 // indirect
-	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
+	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beevik/etree v1.1.0 // indirect

go.sum

@@ -1142,6 +1142,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE=
+github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A=

pkg/alertmanager/alertmanagernotify/email/email.go

@@ -0,0 +1,430 @@
+package email
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"log/slog"
+	"math/rand"
+	"mime"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net"
+	"net/mail"
+	"net/smtp"
+	"net/textproto"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "email"
+)
+
+// Email implements a Notifier for email notifications.
+type Email struct {
+	conf     *config.EmailConfig
+	tmpl     *template.Template
+	logger   *slog.Logger
+	hostname string
+}
+
+var errNoAuthUserNameConfigured = errors.NewInternalf(errors.CodeInternal, "no auth username configured")
+
+// New returns a new Email notifier.
+func New(c *config.EmailConfig, t *template.Template, l *slog.Logger) *Email {
+	if _, ok := c.Headers["Subject"]; !ok {
+		c.Headers["Subject"] = config.DefaultEmailSubject
+	}
+	if _, ok := c.Headers["To"]; !ok {
+		c.Headers["To"] = c.To
+	}
+	if _, ok := c.Headers["From"]; !ok {
+		c.Headers["From"] = c.From
+	}
+
+	h, err := os.Hostname()
+	// If we can't get the hostname, we'll use localhost
+	if err != nil {
+		h = "localhost.localdomain"
+	}
+	return &Email{conf: c, tmpl: t, logger: l, hostname: h}
+}
+
+// auth resolves a string of authentication mechanisms.
+func (n *Email) auth(mechs string) (smtp.Auth, error) {
+	username := n.conf.AuthUsername
+
+	// If no username is set, return custom error which can be ignored if needed.
+	if n.conf.AuthUsername == "" {
+		return nil, errNoAuthUserNameConfigured
+	}
+
+	err := &types.MultiError{}
+	for mech := range strings.SplitSeq(mechs, " ") {
+		switch mech {
+		case "CRAM-MD5":
+			secret, secretErr := n.getAuthSecret()
+			if secretErr != nil {
+				err.Add(secretErr)
+				continue
+			}
+			if secret == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing secret for CRAM-MD5 auth mechanism"))
+				continue
+			}
+			return smtp.CRAMMD5Auth(username, secret), nil
+
+		case "PLAIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				err.Add(passwordErr)
+				continue
+			}
+			if password == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing password for PLAIN auth mechanism"))
+				continue
+			}
+			identity := n.conf.AuthIdentity
+
+			return smtp.PlainAuth(identity, username, password, n.conf.Smarthost.Host), nil
+		case "LOGIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				err.Add(passwordErr)
+				continue
+			}
+			if password == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing password for LOGIN auth mechanism"))
+				continue
+			}
+			return LoginAuth(username, password), nil
+		}
+	}
+	if err.Len() == 0 {
+		err.Add(errors.NewInternalf(errors.CodeInternal, "unknown auth mechanism: %s", mechs))
+	}
+	return nil, err
+}
+
+// Notify implements the Notifier interface.
+func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var (
+		c       *smtp.Client
+		conn    net.Conn
+		err     error
+		success = false
+	)
+	// Determine whether to use Implicit TLS
+	var useImplicitTLS bool
+	if n.conf.ForceImplicitTLS != nil {
+		useImplicitTLS = *n.conf.ForceImplicitTLS
+	} else {
+		// Default logic: port 465 uses implicit TLS (backward compatibility)
+		useImplicitTLS = n.conf.Smarthost.Port == "465"
+	}
+
+	if useImplicitTLS {
+		tlsConfig, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "parse TLS configuration")
+		}
+		if tlsConfig.ServerName == "" {
+			tlsConfig.ServerName = n.conf.Smarthost.Host
+		}
+
+		conn, err = tls.Dial("tcp", n.conf.Smarthost.String(), tlsConfig)
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "establish TLS connection to server")
+		}
+	} else {
+		var (
+			d   = net.Dialer{}
+			err error
+		)
+		conn, err = d.DialContext(ctx, "tcp", n.conf.Smarthost.String())
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "establish connection to server")
+		}
+	}
+	c, err = smtp.NewClient(conn, n.conf.Smarthost.Host)
+	if err != nil {
+		conn.Close()
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "create SMTP client")
+	}
+	defer func() {
+		// Try to clean up after ourselves but don't log anything if something has failed.
+		if err := c.Quit(); success && err != nil {
+			n.logger.WarnContext(ctx, "failed to close SMTP connection", "err", err)
+		}
+	}()
+
+	if n.conf.Hello != "" {
+		err = c.Hello(n.conf.Hello)
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send EHLO command")
+		}
+	}
+
+	// Global Config guarantees RequireTLS is not nil.
+	if *n.conf.RequireTLS && !useImplicitTLS {
+		if ok, _ := c.Extension("STARTTLS"); !ok {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "'require_tls' is true (default) but %q does not advertise the STARTTLS extension", n.conf.Smarthost)
+		}
+
+		tlsConf, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "parse TLS configuration")
+		}
+		if tlsConf.ServerName == "" {
+			tlsConf.ServerName = n.conf.Smarthost.Host
+		}
+
+		if err := c.StartTLS(tlsConf); err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send STARTTLS command")
+		}
+	}
+
+	if ok, mech := c.Extension("AUTH"); ok {
+		auth, err := n.auth(mech)
+		if err != nil && err != errNoAuthUserNameConfigured {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "find auth mechanism")
+		} else if err == errNoAuthUserNameConfigured {
+			n.logger.DebugContext(ctx, "no auth username configured. Attempting to send email without authenticating")
+		}
+		if auth != nil {
+			if err := c.Auth(auth); err != nil {
+				return true, errors.WrapInternalf(err, errors.CodeInternal, "%T auth", auth)
+			}
+		}
+	}
+
+	var (
+		tmplErr error
+		data    = notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
+		tmpl    = notify.TmplText(n.tmpl, data, &tmplErr)
+	)
+	from := tmpl(n.conf.From)
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "execute 'from' template")
+	}
+	to := tmpl(n.conf.To)
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "execute 'to' template")
+	}
+
+	addrs, err := mail.ParseAddressList(from)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "parse 'from' addresses")
+	}
+	if len(addrs) != 1 {
+		return false, errors.NewInternalf(errors.CodeInternal, "must be exactly one 'from' address (got: %d)", len(addrs))
+	}
+	if err = c.Mail(addrs[0].Address); err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "send MAIL command")
+	}
+	addrs, err = mail.ParseAddressList(to)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "parse 'to' addresses")
+	}
+	for _, addr := range addrs {
+		if err = c.Rcpt(addr.Address); err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send RCPT command")
+		}
+	}
+
+	// Send the email headers and body.
+	message, err := c.Data()
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "send DATA command")
+	}
+	closeOnce := sync.OnceValue(func() error {
+		return message.Close()
+	})
+	// Close the message when this method exits in order to not leak resources. Even though we're calling this explicitly
+	// further down, the method may exit before then.
+	defer func() {
+		// If we try close an already-closed writer, it'll send a subsequent request to the server which is invalid.
+		_ = closeOnce()
+	}()
+
+	buffer := &bytes.Buffer{}
+	for header, t := range n.conf.Headers {
+		value, err := n.tmpl.ExecuteTextString(t, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute %q header template", header)
+		}
+		fmt.Fprintf(buffer, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", value))
+	}
+
+	if _, ok := n.conf.Headers["Message-Id"]; !ok {
+		fmt.Fprintf(buffer, "Message-Id: %s\r\n", fmt.Sprintf("<%d.%d@%s>", time.Now().UnixNano(), rand.Uint64(), n.hostname))
+	}
+
+	if n.conf.Threading.Enabled {
+		key, err := notify.ExtractGroupKey(ctx)
+		if err != nil {
+			return false, err
+		}
+		// Add threading headers. All notifications for the same alert group
+		// (identified by key hash) are threaded together.
+		threadBy := ""
+		if n.conf.Threading.ThreadByDate != "none" {
+			// ThreadByDate is 'daily':
+			// Use current date so all mails for this alert today thread together.
+			threadBy = time.Now().Format("2006-01-02")
+		}
+		keyHash := key.Hash()
+		if len(keyHash) > 16 {
+			keyHash = keyHash[:16]
+		}
+		// The thread root ID is a Message-ID that doesn't correspond to
+		// any actual email. Email clients following the (commonly used) JWZ
+		// algorithm will create a dummy container to group these messages.
+		threadRootID := fmt.Sprintf("<alert-%s-%s@alertmanager>", keyHash, threadBy)
+		fmt.Fprintf(buffer, "References: %s\r\n", threadRootID)
+		fmt.Fprintf(buffer, "In-Reply-To: %s\r\n", threadRootID)
+	}
+
+	multipartBuffer := &bytes.Buffer{}
+	multipartWriter := multipart.NewWriter(multipartBuffer)
+
+	fmt.Fprintf(buffer, "Date: %s\r\n", time.Now().Format(time.RFC1123Z))
+	fmt.Fprintf(buffer, "Content-Type: multipart/alternative;  boundary=%s\r\n", multipartWriter.Boundary())
+	fmt.Fprintf(buffer, "MIME-Version: 1.0\r\n\r\n")
+
+	// TODO: Add some useful headers here, such as URL of the alertmanager
+	// and active/resolved.
+	_, err = message.Write(buffer.Bytes())
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "write headers")
+	}
+
+	if len(n.conf.Text) > 0 {
+		// Text template
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/plain; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for text template")
+		}
+		body, err := n.tmpl.ExecuteTextString(n.conf.Text, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute text template")
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "write text part")
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "close text part")
+		}
+	}
+
+	if len(n.conf.HTML) > 0 {
+		// Html template
+		// Preferred alternative placed last per section 5.1.4 of RFC 2046
+		// https://www.ietf.org/rfc/rfc2046.txt
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/html; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for html template")
+		}
+		body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute html template")
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "write HTML part")
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "close HTML part")
+		}
+	}
+
+	err = multipartWriter.Close()
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "close multipartWriter")
+	}
+
+	_, err = message.Write(multipartBuffer.Bytes())
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "write body buffer")
+	}
+
+	// Complete the message and await response.
+	if err = closeOnce(); err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "delivery failure")
+	}
+
+	success = true
+	return false, nil
+}
+
+type loginAuth struct {
+	username, password string
+}
+
+func LoginAuth(username, password string) smtp.Auth {
+	return &loginAuth{username, password}
+}
+
+func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
+	return "LOGIN", []byte{}, nil
+}
+
+// Used for AUTH LOGIN. (Maybe password should be encrypted).
+func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
+	if more {
+		switch strings.ToLower(string(fromServer)) {
+		case "username:":
+			return []byte(a.username), nil
+		case "password:":
+			return []byte(a.password), nil
+		default:
+			return nil, errors.NewInternalf(errors.CodeInternal, "unexpected server challenge")
+		}
+	}
+	return nil, nil
+}
+
+func (n *Email) getPassword() (string, error) {
+	if len(n.conf.AuthPasswordFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthPasswordFile)
+		if err != nil {
+			return "", errors.NewInternalf(errors.CodeInternal, "could not read %s: %v", n.conf.AuthPasswordFile, err)
+		}
+		return strings.TrimSpace(string(content)), nil
+	}
+	return string(n.conf.AuthPassword), nil
+}
+
+func (n *Email) getAuthSecret() (string, error) {
+	if len(n.conf.AuthSecretFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthSecretFile)
+		if err != nil {
+			return "", errors.NewInternalf(errors.CodeInternal, "could not read %s: %v", n.conf.AuthSecretFile, err)
+		}
+		return string(content), nil
+	}
+	return string(n.conf.AuthSecret), nil
+}

pkg/alertmanager/alertmanagernotify/email/testdata/auth-local.yml

@@ -0,0 +1,4 @@
+smarthost: 127.0.0.1:1026
+server: http://127.0.0.1:1081/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/auth.yml

@@ -0,0 +1,4 @@
+smarthost: maildev-auth:1025
+server: http://maildev-auth:1080/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/noauth-local.yml

@@ -0,0 +1,2 @@
+smarthost: 127.0.0.1:1025
+server: http://127.0.0.1:1080/

pkg/alertmanager/alertmanagernotify/email/testdata/noauth.yml

@@ -0,0 +1,2 @@
+smarthost: maildev-noauth:1025
+server: http://maildev-noauth:1080/

pkg/alertmanager/alertmanagernotify/msteamsv2/msteamsv2.go

@@ -27,6 +27,10 @@ const (
 	colorGrey  = "Warning"
 )
 
+const (
+	Integration = "msteamsv2"
+)
+
 type Notifier struct {
 	conf         *config.MSTeamsV2Config
 	titleLink    string
@@ -87,7 +91,7 @@ type teamsMessage struct {
 
 // New returns a new notifier that uses the Microsoft Teams Power Platform connector.
 func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
-	client, err := commoncfg.NewClientFromConfig(*c.HTTPConfig, "msteamsv2", httpOpts...)
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
 	}

pkg/alertmanager/alertmanagernotify/opsgenie/api_key_file

@@ -0,0 +1,2 @@
+my_secret_api_key
+

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie.go

@@ -0,0 +1,290 @@
+package opsgenie
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"maps"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "opsgenie"
+)
+
+// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
+const maxMessageLenRunes = 130
+
+// Notifier implements a Notifier for OpsGenie notifications.
+type Notifier struct {
+	conf    *config.OpsGenieConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new OpsGenie notifier.
+func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:    c,
+		tmpl:    t,
+		logger:  l,
+		client:  client,
+		retrier: &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
+	}, nil
+}
+
+type opsGenieCreateMessage struct {
+	Alias       string                           `json:"alias"`
+	Message     string                           `json:"message"`
+	Description string                           `json:"description,omitempty"`
+	Details     map[string]string                `json:"details"`
+	Source      string                           `json:"source"`
+	Responders  []opsGenieCreateMessageResponder `json:"responders,omitempty"`
+	Tags        []string                         `json:"tags,omitempty"`
+	Note        string                           `json:"note,omitempty"`
+	Priority    string                           `json:"priority,omitempty"`
+	Entity      string                           `json:"entity,omitempty"`
+	Actions     []string                         `json:"actions,omitempty"`
+}
+
+type opsGenieCreateMessageResponder struct {
+	ID       string `json:"id,omitempty"`
+	Name     string `json:"name,omitempty"`
+	Username string `json:"username,omitempty"`
+	Type     string `json:"type"` // team, user, escalation, schedule etc.
+}
+
+type opsGenieCloseMessage struct {
+	Source string `json:"source"`
+}
+
+type opsGenieUpdateMessageMessage struct {
+	Message string `json:"message,omitempty"`
+}
+
+type opsGenieUpdateDescriptionMessage struct {
+	Description string `json:"description,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	requests, retry, err := n.createRequests(ctx, as...)
+	if err != nil {
+		return retry, err
+	}
+
+	for _, req := range requests {
+		req.Header.Set("User-Agent", notify.UserAgentHeader)
+		resp, err := n.client.Do(req) //nolint:bodyclose
+		if err != nil {
+			return true, err
+		}
+		shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+		notify.Drain(resp)
+		if err != nil {
+			return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+		}
+	}
+	return true, nil
+}
+
+// Like Split but filter out empty strings.
+func safeSplit(s, sep string) []string {
+	a := strings.Split(strings.TrimSpace(s), sep)
+	b := a[:0]
+	for _, x := range a {
+		if x != "" {
+			b = append(b, x)
+		}
+	}
+	return b
+}
+
+// Create requests for a list of alerts.
+func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+	logger := n.logger.With("group_key", key)
+	logger.DebugContext(ctx, "extracted group key")
+
+	data := notify.GetTemplateData(ctx, n.tmpl, as, logger)
+
+	tmpl := notify.TmplText(n.tmpl, data, &err)
+
+	details := make(map[string]string)
+
+	maps.Copy(details, data.CommonLabels)
+
+	for k, v := range n.conf.Details {
+		details[k] = tmpl(v)
+	}
+
+	requests := []*http.Request{}
+
+	var (
+		alias  = key.Hash()
+		alerts = types.Alerts(as...)
+	)
+	switch alerts.Status() {
+	case model.AlertResolved:
+		resolvedEndpointURL := n.conf.APIURL.Copy()
+		resolvedEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
+		q := resolvedEndpointURL.Query()
+		q.Set("identifierType", "alias")
+		resolvedEndpointURL.RawQuery = q.Encode()
+		msg := &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", resolvedEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+	default:
+		message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
+		if truncated {
+			logger.WarnContext(ctx, "Truncated message", "alert", key, "max_runes", maxMessageLenRunes)
+		}
+
+		createEndpointURL := n.conf.APIURL.Copy()
+		createEndpointURL.Path += "v2/alerts"
+
+		var responders []opsGenieCreateMessageResponder
+		for _, r := range n.conf.Responders {
+			responder := opsGenieCreateMessageResponder{
+				ID:       tmpl(r.ID),
+				Name:     tmpl(r.Name),
+				Username: tmpl(r.Username),
+				Type:     tmpl(r.Type),
+			}
+
+			if responder == (opsGenieCreateMessageResponder{}) {
+				// Filter out empty responders. This is useful if you want to fill
+				// responders dynamically from alert's common labels.
+				continue
+			}
+
+			if responder.Type == "teams" {
+				teams := safeSplit(responder.Name, ",")
+				for _, team := range teams {
+					newResponder := opsGenieCreateMessageResponder{
+						Name: tmpl(team),
+						Type: tmpl("team"),
+					}
+					responders = append(responders, newResponder)
+				}
+				continue
+			}
+
+			responders = append(responders, responder)
+		}
+
+		msg := &opsGenieCreateMessage{
+			Alias:       alias,
+			Message:     message,
+			Description: tmpl(n.conf.Description),
+			Details:     details,
+			Source:      tmpl(n.conf.Source),
+			Responders:  responders,
+			Tags:        safeSplit(tmpl(n.conf.Tags), ","),
+			Note:        tmpl(n.conf.Note),
+			Priority:    tmpl(n.conf.Priority),
+			Entity:      tmpl(n.conf.Entity),
+			Actions:     safeSplit(tmpl(n.conf.Actions), ","),
+		}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", createEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+
+		if n.conf.UpdateAlerts {
+			updateMessageEndpointURL := n.conf.APIURL.Copy()
+			updateMessageEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/message", alias)
+			q := updateMessageEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateMessageEndpointURL.RawQuery = q.Encode()
+			updateMsgMsg := &opsGenieUpdateMessageMessage{
+				Message: msg.Message,
+			}
+			var updateMessageBuf bytes.Buffer
+			if err := json.NewEncoder(&updateMessageBuf).Encode(updateMsgMsg); err != nil {
+				return nil, false, err
+			}
+			req, err := http.NewRequest("PUT", updateMessageEndpointURL.String(), &updateMessageBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req)
+
+			updateDescriptionEndpointURL := n.conf.APIURL.Copy()
+			updateDescriptionEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/description", alias)
+			q = updateDescriptionEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateDescriptionEndpointURL.RawQuery = q.Encode()
+			updateDescMsg := &opsGenieUpdateDescriptionMessage{
+				Description: msg.Description,
+			}
+
+			var updateDescriptionBuf bytes.Buffer
+			if err := json.NewEncoder(&updateDescriptionBuf).Encode(updateDescMsg); err != nil {
+				return nil, false, err
+			}
+			req, err = http.NewRequest("PUT", updateDescriptionEndpointURL.String(), &updateDescriptionBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req.WithContext(ctx))
+		}
+	}
+
+	var apiKey string
+	if n.conf.APIKey != "" {
+		apiKey = tmpl(string(n.conf.APIKey))
+	} else {
+		content, err := os.ReadFile(n.conf.APIKeyFile)
+		if err != nil {
+			return nil, false, errors.WrapInternalf(err, errors.CodeInternal, "read key_file error")
+		}
+		apiKey = tmpl(string(content))
+		apiKey = strings.TrimSpace(string(apiKey))
+	}
+
+	if err != nil {
+		return nil, false, errors.WrapInternalf(err, errors.CodeInternal, "templating error")
+	}
+
+	for _, req := range requests {
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
+	}
+
+	return requests, true, nil
+}

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie_test.go

@@ -0,0 +1,333 @@
+package opsgenie
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestOpsGenieRetry(t *testing.T) {
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestOpsGenieRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKey:     config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestGettingOpsGegineApikeyFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+
+	f, err := os.CreateTemp(t.TempDir(), "opsgenie_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKeyFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestOpsGenie(t *testing.T) {
+	u, err := url.Parse("https://opsgenie/api")
+	if err != nil {
+		t.Fatalf("failed to parse URL: %v", err)
+	}
+	logger := promslog.NewNopLogger()
+	tmpl := test.CreateTmpl(t)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.OpsGenieConfig
+
+		expectedEmptyAlertBody string
+		expectedBody           string
+	}{
+		{
+			title: "config without details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with multiple teams",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName3 }}`,
+						Type: `{{ .CommonLabels.ResponderType3 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"TeamB","type":"team"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1"}
+`,
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			notifier, err := New(tc.cfg, tmpl, logger)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			expectedURL, _ := url.Parse("https://opsgenie/apiv2/alerts")
+
+			// Empty alert.
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+
+			req, retry, err := notifier.createRequests(ctx, alert1)
+			require.NoError(t, err)
+			require.Len(t, req, 1)
+			require.True(t, retry)
+			require.Equal(t, expectedURL, req[0].URL)
+			require.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization"))
+			require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0]))
+
+			// Fully defined alert.
+			alert2 := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"Message":        "message",
+						"Description":    "description",
+						"Source":         "http://prometheus",
+						"ResponderName1": "TeamA",
+						"ResponderType1": "team",
+						"ResponderName2": "EscalationA",
+						"ResponderType2": "escalation",
+						"ResponderName3": "TeamA,TeamB",
+						"ResponderType3": "teams",
+						"Tags":           "tag1,tag2",
+						"Note":           "this is a note",
+						"Priority":       "P1",
+						"Entity":         "test-domain",
+						"Actions":        "doThis,doThat",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			req, retry, err = notifier.createRequests(ctx, alert2)
+			require.NoError(t, err)
+			require.True(t, retry)
+			require.Len(t, req, 1)
+			require.Equal(t, tc.expectedBody, readBody(t, req[0]))
+
+			// Broken API Key Template.
+			tc.cfg.APIKey = "{{ kaput "
+			_, _, err = notifier.createRequests(ctx, alert2)
+			require.Error(t, err)
+			require.Equal(t, "template: :1: function \"kaput\" not defined", err.Error())
+		})
+	}
+}
+
+func TestOpsGenieWithUpdate(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		Message:      `{{ .CommonLabels.Message }}`,
+		Description:  `{{ .CommonLabels.Description }}`,
+		UpdateAlerts: true,
+		APIKey:       "test-api-key",
+		APIURL:       &config.URL{URL: u},
+		HTTPConfig:   &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+	alert := &types.Alert{
+		Alert: model.Alert{
+			StartsAt: time.Now(),
+			EndsAt:   time.Now().Add(time.Hour),
+			Labels: model.LabelSet{
+				"Message":     "new message",
+				"Description": "new description",
+			},
+		},
+	}
+	require.NoError(t, err)
+	requests, retry, err := notifierWithUpdate.createRequests(ctx, alert)
+	require.NoError(t, err)
+	require.True(t, retry)
+	require.Len(t, requests, 3)
+
+	body0 := readBody(t, requests[0])
+	body1 := readBody(t, requests[1])
+	body2 := readBody(t, requests[2])
+	key, _ := notify.ExtractGroupKey(ctx)
+	alias := key.Hash()
+
+	require.Equal(t, "https://test-opsgenie-url/v2/alerts", requests[0].URL.String())
+	require.NotEmpty(t, body0)
+
+	require.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias))
+	require.JSONEq(t, `{"message":"new message"}`, body1)
+	require.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias))
+	require.JSONEq(t, `{"description":"new description"}`, body2)
+}
+
+func TestOpsGenieApiKeyFile(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		APIKeyFile: `./api_key_file`,
+		APIURL:     &config.URL{URL: u},
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+
+	require.NoError(t, err)
+	requests, _, err := notifierWithUpdate.createRequests(ctx)
+	require.NoError(t, err)
+	require.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
+}
+
+func readBody(t *testing.T, r *http.Request) string {
+	t.Helper()
+	body, err := io.ReadAll(r.Body)
+	require.NoError(t, err)
+	return string(body)
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty.go

@@ -0,0 +1,374 @@
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/alecthomas/units"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "pagerduty"
+)
+
+const (
+	maxEventSize int = 512000
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTc4-send-a-v1-event - 1024 characters or runes.
+	maxV1DescriptionLenRunes = 1024
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1024 characters or runes.
+	maxV2SummaryLenRunes = 1024
+)
+
+// Notifier implements a Notifier for PagerDuty notifications.
+type Notifier struct {
+	conf    *config.PagerdutyConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	apiV1   string // for tests.
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new PagerDuty notifier.
+func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	n := &Notifier{conf: c, tmpl: t, logger: l, client: client}
+	if c.ServiceKey != "" || c.ServiceKeyFile != "" {
+		n.apiV1 = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+		// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v1-overview#api-response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusForbidden}, CustomDetailsFunc: errDetails}
+	} else {
+		// Retrying can solve the issue on 429 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v2-overview#response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}, CustomDetailsFunc: errDetails}
+	}
+	return n, nil
+}
+
+const (
+	pagerDutyEventTrigger = "trigger"
+	pagerDutyEventResolve = "resolve"
+)
+
+type pagerDutyMessage struct {
+	RoutingKey  string            `json:"routing_key,omitempty"`
+	ServiceKey  string            `json:"service_key,omitempty"`
+	DedupKey    string            `json:"dedup_key,omitempty"`
+	IncidentKey string            `json:"incident_key,omitempty"`
+	EventType   string            `json:"event_type,omitempty"`
+	Description string            `json:"description,omitempty"`
+	EventAction string            `json:"event_action"`
+	Payload     *pagerDutyPayload `json:"payload"`
+	Client      string            `json:"client,omitempty"`
+	ClientURL   string            `json:"client_url,omitempty"`
+	Details     map[string]any    `json:"details,omitempty"`
+	Images      []pagerDutyImage  `json:"images,omitempty"`
+	Links       []pagerDutyLink   `json:"links,omitempty"`
+}
+
+type pagerDutyLink struct {
+	HRef string `json:"href"`
+	Text string `json:"text"`
+}
+
+type pagerDutyImage struct {
+	Src  string `json:"src"`
+	Alt  string `json:"alt"`
+	Href string `json:"href"`
+}
+
+type pagerDutyPayload struct {
+	Summary       string         `json:"summary"`
+	Source        string         `json:"source"`
+	Severity      string         `json:"severity"`
+	Timestamp     string         `json:"timestamp,omitempty"`
+	Class         string         `json:"class,omitempty"`
+	Component     string         `json:"component,omitempty"`
+	Group         string         `json:"group,omitempty"`
+	CustomDetails map[string]any `json:"custom_details,omitempty"`
+}
+
+func (n *Notifier) encodeMessage(ctx context.Context, msg *pagerDutyMessage) (bytes.Buffer, error) {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return buf, errors.WrapInternalf(err, errors.CodeInternal, "failed to encode PagerDuty message")
+	}
+
+	if buf.Len() > maxEventSize {
+		truncatedMsg := fmt.Sprintf("Custom details have been removed because the original event exceeds the maximum size of %s", units.MetricBytes(maxEventSize).String())
+
+		if n.apiV1 != "" {
+			msg.Details = map[string]any{"error": truncatedMsg}
+		} else {
+			msg.Payload.CustomDetails = map[string]any{"error": truncatedMsg}
+		}
+
+		n.logger.WarnContext(ctx, "Truncated Details because message of size exceeds limit", "message_size", units.MetricBytes(buf.Len()).String(), "max_size", units.MetricBytes(maxEventSize).String())
+
+		buf.Reset()
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return buf, errors.WrapInternalf(err, errors.CodeInternal, "failed to encode PagerDuty message")
+		}
+	}
+
+	return buf, nil
+}
+
+func (n *Notifier) notifyV1(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated description", "key", key, "max_runes", maxV1DescriptionLenRunes)
+	}
+
+	serviceKey := string(n.conf.ServiceKey)
+	if serviceKey == "" {
+		content, fileErr := os.ReadFile(n.conf.ServiceKeyFile)
+		if fileErr != nil {
+			return false, errors.WrapInternalf(fileErr, errors.CodeInternal, "failed to read service key from file")
+		}
+		serviceKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		ServiceKey:  tmpl(serviceKey),
+		EventType:   eventType,
+		IncidentKey: key.Hash(),
+		Description: description,
+		Details:     details,
+	}
+
+	if eventType == pagerDutyEventTrigger {
+		msg.Client = tmpl(n.conf.Client)
+		msg.ClientURL = tmpl(n.conf.ClientURL)
+	}
+
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "failed to template PagerDuty v1 message")
+	}
+
+	// Ensure that the service key isn't empty after templating.
+	if msg.ServiceKey == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "service key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(ctx, msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &encodedMsg) //nolint:bodyclose
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "failed to post message to PagerDuty v1")
+	}
+	defer notify.Drain(resp)
+
+	return n.retrier.Check(resp.StatusCode, resp.Body)
+}
+
+func (n *Notifier) notifyV2(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.Severity == "" {
+		n.conf.Severity = "error"
+	}
+
+	summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV2SummaryLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated summary", "key", key, "max_runes", maxV2SummaryLenRunes)
+	}
+
+	routingKey := string(n.conf.RoutingKey)
+	if routingKey == "" {
+		content, fileErr := os.ReadFile(n.conf.RoutingKeyFile)
+		if fileErr != nil {
+			return false, errors.WrapInternalf(fileErr, errors.CodeInternal, "failed to read routing key from file")
+		}
+		routingKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		Client:      tmpl(n.conf.Client),
+		ClientURL:   tmpl(n.conf.ClientURL),
+		RoutingKey:  tmpl(routingKey),
+		EventAction: eventType,
+		DedupKey:    key.Hash(),
+		Images:      make([]pagerDutyImage, 0, len(n.conf.Images)),
+		Links:       make([]pagerDutyLink, 0, len(n.conf.Links)),
+		Payload: &pagerDutyPayload{
+			Summary:       summary,
+			Source:        tmpl(n.conf.Source),
+			Severity:      tmpl(n.conf.Severity),
+			CustomDetails: details,
+			Class:         tmpl(n.conf.Class),
+			Component:     tmpl(n.conf.Component),
+			Group:         tmpl(n.conf.Group),
+		},
+	}
+
+	for _, item := range n.conf.Images {
+		image := pagerDutyImage{
+			Src:  tmpl(item.Src),
+			Alt:  tmpl(item.Alt),
+			Href: tmpl(item.Href),
+		}
+
+		if image.Src != "" {
+			msg.Images = append(msg.Images, image)
+		}
+	}
+
+	for _, item := range n.conf.Links {
+		link := pagerDutyLink{
+			HRef: tmpl(item.Href),
+			Text: tmpl(item.Text),
+		}
+
+		if link.HRef != "" {
+			msg.Links = append(msg.Links, link)
+		}
+	}
+
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "failed to template PagerDuty v2 message")
+	}
+
+	// Ensure that the routing key isn't empty after templating.
+	if msg.RoutingKey == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "routing key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(ctx, msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &encodedMsg) //nolint:bodyclose
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "failed to post message to PagerDuty")
+	}
+	defer notify.Drain(resp)
+
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return retry, err
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With("group_key", key)
+
+	var (
+		alerts    = types.Alerts(as...)
+		data      = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		eventType = pagerDutyEventTrigger
+	)
+
+	if alerts.Status() == model.AlertResolved {
+		eventType = pagerDutyEventResolve
+	}
+
+	logger.DebugContext(ctx, "extracted group key", "event_type", eventType)
+
+	details, err := n.renderDetails(data)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "failed to render details: %v", err)
+	}
+
+	if n.conf.Timeout > 0 {
+		nfCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeTimeout, "configured pagerduty timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = nfCtx
+	}
+
+	nf := n.notifyV2
+	if n.apiV1 != "" {
+		nf = n.notifyV1
+	}
+	retry, err := nf(ctx, eventType, key, data, details)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.WrapInternalf(err, errors.CodeInternal, "failed to notify PagerDuty: %v", context.Cause(ctx))
+		}
+		return retry, err
+	}
+	return retry, nil
+}
+
+func errDetails(status int, body io.Reader) string {
+	// See https://v2.developer.pagerduty.com/docs/trigger-events for the v1 events API.
+	// See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 for the v2 events API.
+	if status != http.StatusBadRequest || body == nil {
+		return ""
+	}
+	var pgr struct {
+		Status  string   `json:"status"`
+		Message string   `json:"message"`
+		Errors  []string `json:"errors"`
+	}
+	if err := json.NewDecoder(body).Decode(&pgr); err != nil {
+		return ""
+	}
+	return fmt.Sprintf("%s: %s", pgr.Message, strings.Join(pgr.Errors, ","))
+}
+
+func (n *Notifier) renderDetails(
+	data *template.Data,
+) (map[string]any, error) {
+	var (
+		tmplTextErr  error
+		tmplText     = notify.TmplText(n.tmpl, data, &tmplTextErr)
+		tmplTextFunc = func(tmpl string) (string, error) {
+			return tmplText(tmpl), tmplTextErr
+		}
+	)
+	var err error
+	rendered := make(map[string]any, len(n.conf.Details))
+	for k, v := range n.conf.Details {
+		rendered[k], err = template.DeepCopyWithTemplate(v, tmplTextFunc)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return rendered, nil
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty_test.go

@@ -0,0 +1,879 @@
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestPagerDutyRetryV1(t *testing.T) {
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusForbidden)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "retryv1 - error on status %d", statusCode)
+	}
+}
+
+func TestPagerDutyRetryV2(t *testing.T) {
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "retryv2 - error on status %d", statusCode)
+	}
+}
+
+func TestPagerDutyRedactedURLV1(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "01234567890123456789012345678901"
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+	notifier.apiV1 = u.String()
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyRedactedURLV2(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "01234567890123456789012345678901"
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			URL:        &config.URL{URL: u},
+			RoutingKey: config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyV1ServiceKeyFromFile(t *testing.T) {
+	key := "01234567890123456789012345678901"
+	f, err := os.CreateTemp(t.TempDir(), "pagerduty_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKeyFile: f.Name(),
+			HTTPConfig:     &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+	notifier.apiV1 = u.String()
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyV2RoutingKeyFromFile(t *testing.T) {
+	key := "01234567890123456789012345678901"
+	f, err := os.CreateTemp(t.TempDir(), "pagerduty_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			URL:            &config.URL{URL: u},
+			RoutingKeyFile: f.Name(),
+			HTTPConfig:     &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyTemplating(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		dec := json.NewDecoder(r.Body)
+		out := make(map[string]any)
+		err := dec.Decode(&out)
+		if err != nil {
+			panic(err)
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.PagerdutyConfig
+
+		retry  bool
+		errMsg string
+	}{
+		{
+			title: "full-blown legacy message",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Images: []config.PagerdutyImage{
+					{
+						Src:  "{{ .Status }}",
+						Alt:  "{{ .Status }}",
+						Href: "{{ .Status }}",
+					},
+				},
+				Links: []config.PagerdutyLink{
+					{
+						Href: "{{ .Status }}",
+						Text: "{{ .Status }}",
+					},
+				},
+				Details: map[string]any{
+					"firing":       `{{ .Alerts.Firing | toJson }}`,
+					"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+		},
+		{
+			title: "full-blown legacy message",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Images: []config.PagerdutyImage{
+					{
+						Src:  "{{ .Status }}",
+						Alt:  "{{ .Status }}",
+						Href: "{{ .Status }}",
+					},
+				},
+				Links: []config.PagerdutyLink{
+					{
+						Href: "{{ .Status }}",
+						Text: "{{ .Status }}",
+					},
+				},
+				Details: map[string]any{
+					"firing":       `{{ template "pagerduty.default.instances" .Alerts.Firing }}`,
+					"resolved":     `{{ template "pagerduty.default.instances" .Alerts.Resolved }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+		},
+		{
+			title: "nested details",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": map[string]any{
+								"firing":       `{{ .Alerts.Firing | toJson }}`,
+								"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+								"num_firing":   `{{ .Alerts.Firing | len }}`,
+								"num_resolved": `{{ .Alerts.Resolved | len }}`,
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			title: "nested details with template error",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": map[string]any{
+								"firing": `{{ template "pagerduty.default.instances" .Alerts.Firing`,
+							},
+						},
+					},
+				},
+			},
+			errMsg: "failed to render details: template: :1: unclosed action",
+		},
+		{
+			title: "details with templating errors",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"firing":       `{{ .Alerts.Firing | toJson`,
+					"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+			errMsg: "failed to render details: template: :1: unclosed action",
+		},
+		{
+			title: "v2 message with templating errors",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Severity:   "{{ ",
+			},
+			errMsg: "failed to template",
+		},
+		{
+			title: "v1 message with templating errors",
+			cfg: &config.PagerdutyConfig{
+				ServiceKey: config.Secret("01234567890123456789012345678901"),
+				Client:     "{{ ",
+			},
+			errMsg: "failed to template",
+		},
+		{
+			title: "routing key cannot be empty",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret(`{{ "" }}`),
+			},
+			errMsg: "routing key cannot be empty",
+		},
+		{
+			title: "service_key cannot be empty",
+			cfg: &config.PagerdutyConfig{
+				ServiceKey: config.Secret(`{{ "" }}`),
+			},
+			errMsg: "service key cannot be empty",
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			tc.cfg.URL = &config.URL{URL: u}
+			tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
+			pd, err := New(tc.cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+			if pd.apiV1 != "" {
+				pd.apiV1 = u.String()
+			}
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			ok, err := pd.Notify(ctx, []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels: model.LabelSet{
+							"lbl1": "val1",
+						},
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}...)
+			if tc.errMsg == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				if errors.Asc(err, errors.CodeInternal) {
+					_, _, errMsg, _, _, _ := errors.Unwrapb(err)
+					require.Contains(t, errMsg, tc.errMsg)
+				} else {
+					require.Contains(t, err.Error(), tc.errMsg)
+				}
+			}
+			require.Equal(t, tc.retry, ok)
+		})
+	}
+}
+
+func TestErrDetails(t *testing.T) {
+	for _, tc := range []struct {
+		status int
+		body   io.Reader
+
+		exp string
+	}{
+		{
+			status: http.StatusBadRequest,
+			body: bytes.NewBuffer([]byte(
+				`{"status":"invalid event","message":"Event object is invalid","errors":["Length of 'routing_key' is incorrect (should be 32 characters)"]}`,
+			)),
+
+			exp: "Length of 'routing_key' is incorrect",
+		},
+		{
+			status: http.StatusBadRequest,
+			body:   bytes.NewBuffer([]byte(`{"status"}`)),
+
+			exp: "",
+		},
+		{
+			status: http.StatusBadRequest,
+
+			exp: "",
+		},
+		{
+			status: http.StatusTooManyRequests,
+
+			exp: "",
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			err := errDetails(tc.status, tc.body)
+			require.Contains(t, err, tc.exp)
+		})
+	}
+}
+
+func TestEventSizeEnforcement(t *testing.T) {
+	bigDetailsV1 := map[string]any{
+		"firing": strings.Repeat("a", 513000),
+	}
+	bigDetailsV2 := map[string]any{
+		"firing": strings.Repeat("a", 513000),
+	}
+
+	// V1 Messages
+	msgV1 := &pagerDutyMessage{
+		ServiceKey: "01234567890123456789012345678901",
+		EventType:  "trigger",
+		Details:    bigDetailsV1,
+	}
+
+	notifierV1, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV1, err := notifierV1.encodeMessage(context.Background(), msgV1)
+	require.NoError(t, err)
+	require.Contains(t, encodedV1.String(), `"details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+
+	// V2 Messages
+	msgV2 := &pagerDutyMessage{
+		RoutingKey:  "01234567890123456789012345678901",
+		EventAction: "trigger",
+		Payload: &pagerDutyPayload{
+			CustomDetails: bigDetailsV2,
+		},
+	}
+
+	notifierV2, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV2, err := notifierV2.encodeMessage(context.Background(), msgV2)
+	require.NoError(t, err)
+	require.Contains(t, encodedV2.String(), `"custom_details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+}
+
+func TestPagerDutyEmptySrcHref(t *testing.T) {
+	type pagerDutyEvent struct {
+		RoutingKey  string           `json:"routing_key"`
+		EventAction string           `json:"event_action"`
+		DedupKey    string           `json:"dedup_key"`
+		Payload     pagerDutyPayload `json:"payload"`
+		Images      []pagerDutyImage
+		Links       []pagerDutyLink
+	}
+
+	images := []config.PagerdutyImage{
+		{
+			Src:  "",
+			Alt:  "Empty src",
+			Href: "https://example.com/",
+		},
+		{
+			Src:  "https://example.com/cat.jpg",
+			Alt:  "Empty href",
+			Href: "",
+		},
+		{
+			Src:  "https://example.com/cat.jpg",
+			Alt:  "",
+			Href: "https://example.com/",
+		},
+	}
+
+	links := []config.PagerdutyLink{
+		{
+			Href: "",
+			Text: "Empty href",
+		},
+		{
+			Href: "https://example.com/",
+			Text: "",
+		},
+	}
+
+	expectedImages := make([]pagerDutyImage, 0, len(images))
+	for _, image := range images {
+		if image.Src == "" {
+			continue
+		}
+		expectedImages = append(expectedImages, pagerDutyImage{
+			Src:  image.Src,
+			Alt:  image.Alt,
+			Href: image.Href,
+		})
+	}
+
+	expectedLinks := make([]pagerDutyLink, 0, len(links))
+	for _, link := range links {
+		if link.Href == "" {
+			continue
+		}
+		expectedLinks = append(expectedLinks, pagerDutyLink{
+			HRef: link.Href,
+			Text: link.Text,
+		})
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			decoder := json.NewDecoder(r.Body)
+			var event pagerDutyEvent
+			if err := decoder.Decode(&event); err != nil {
+				panic(err)
+			}
+
+			if event.RoutingKey == "" || event.EventAction == "" {
+				http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+				return
+			}
+
+			for _, image := range event.Images {
+				if image.Src == "" {
+					http.Error(w, "Event object is invalid: 'image src' is missing or blank", http.StatusBadRequest)
+					return
+				}
+			}
+
+			for _, link := range event.Links {
+				if link.HRef == "" {
+					http.Error(w, "Event object is invalid: 'link href' is missing or blank", http.StatusBadRequest)
+					return
+				}
+			}
+
+			require.Equal(t, expectedImages, event.Images)
+			require.Equal(t, expectedLinks, event.Links)
+		},
+	))
+	defer server.Close()
+
+	url, err := url.Parse(server.URL)
+	require.NoError(t, err)
+
+	pagerDutyConfig := config.PagerdutyConfig{
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+		RoutingKey: config.Secret("01234567890123456789012345678901"),
+		URL:        &config.URL{URL: url},
+		Images:     images,
+		Links:      links,
+	}
+
+	pagerDuty, err := New(&pagerDutyConfig, test.CreateTmpl(t), promslog.NewNopLogger())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+
+	_, err = pagerDuty.Notify(ctx, []*types.Alert{
+		{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"lbl1": "val1",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		},
+	}...)
+	require.NoError(t, err)
+}
+
+func TestPagerDutyTimeout(t *testing.T) {
+	type pagerDutyEvent struct {
+		RoutingKey  string           `json:"routing_key"`
+		EventAction string           `json:"event_action"`
+		DedupKey    string           `json:"dedup_key"`
+		Payload     pagerDutyPayload `json:"payload"`
+		Images      []pagerDutyImage
+		Links       []pagerDutyLink
+	}
+
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(
+				func(w http.ResponseWriter, r *http.Request) {
+					decoder := json.NewDecoder(r.Body)
+					var event pagerDutyEvent
+					if err := decoder.Decode(&event); err != nil {
+						panic(err)
+					}
+
+					if event.RoutingKey == "" || event.EventAction == "" {
+						http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+						return
+					}
+					time.Sleep(tt.latency)
+				},
+			))
+			defer srv.Close()
+			u, err := url.Parse(srv.URL)
+			require.NoError(t, err)
+
+			cfg := config.PagerdutyConfig{
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				URL:        &config.URL{URL: u},
+				Timeout:    tt.timeout,
+			}
+
+			pd, err := New(&cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+			alert := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"lbl1": "val1",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = pd.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+func TestRenderDetails(t *testing.T) {
+	type args struct {
+		details map[string]any
+		data    *template.Data
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    map[string]any
+		wantErr bool
+	}{
+		{
+			name: "flat",
+			args: args{
+				details: map[string]any{
+					"a": "{{ .Status }}",
+					"b": "String",
+				},
+				data: &template.Data{
+					Status: "Flat",
+				},
+			},
+			want: map[string]any{
+				"a": "Flat",
+				"b": "String",
+			},
+			wantErr: false,
+		},
+		{
+			name: "flat error",
+			args: args{
+				details: map[string]any{
+					"a": "{{ .Status",
+				},
+				data: &template.Data{
+					Status: "Error",
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "nested",
+			args: args{
+				details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": "{{ .Status }}",
+							"d": "String",
+						},
+					},
+				},
+				data: &template.Data{
+					Status: "Nested",
+				},
+			},
+			want: map[string]any{
+				"a": map[string]any{
+					"b": map[string]any{
+						"c": "Nested",
+						"d": "String",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "nested error",
+			args: args{
+				details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": "{{ .Status",
+						},
+					},
+				},
+				data: &template.Data{
+					Status: "Error",
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "alerts",
+			args: args{
+				details: map[string]any{
+					"alerts": map[string]any{
+						"firing":       "{{ .Alerts.Firing | toJson }}",
+						"resolved":     "{{ .Alerts.Resolved | toJson }}",
+						"num_firing":   "{{ len .Alerts.Firing }}",
+						"num_resolved": "{{ len .Alerts.Resolved }}",
+					},
+				},
+				data: &template.Data{
+					Alerts: template.Alerts{
+						{
+							Status: "firing",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Firing1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint1",
+							GeneratorURL: "http://generator1",
+							StartsAt:     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "firing",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Firing2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint2",
+							GeneratorURL: "http://generator2",
+							StartsAt:     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "resolved",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Resolved1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint3",
+							GeneratorURL: "http://generator3",
+							StartsAt:     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "resolved",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Resolved2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint4",
+							GeneratorURL: "http://generator4",
+							StartsAt:     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+					},
+				},
+			},
+			want: map[string]any{
+				"alerts": map[string]any{
+					"firing": []any{
+						map[string]any{
+							"status": "firing",
+							"labels": map[string]any{
+								"alertname": "Firing1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint1",
+							"generatorURL": "http://generator1",
+						},
+						map[string]any{
+							"status": "firing",
+							"labels": map[string]any{
+								"alertname": "Firing2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint2",
+							"generatorURL": "http://generator2",
+						},
+					},
+					"resolved": []any{
+						map[string]any{
+							"status": "resolved",
+							"labels": map[string]any{
+								"alertname": "Resolved1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint3",
+							"generatorURL": "http://generator3",
+						},
+						map[string]any{
+							"status": "resolved",
+							"labels": map[string]any{
+								"alertname": "Resolved2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint4",
+							"generatorURL": "http://generator4",
+						},
+					},
+					"num_firing":   2,
+					"num_resolved": 2,
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			n := &Notifier{
+				conf: &config.PagerdutyConfig{
+					Details: tt.args.details,
+				},
+				tmpl: test.CreateTmpl(t),
+			}
+			got, err := n.renderDetails(tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("renderDetails() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			require.Equal(t, tt.want, got)
+		})
+	}
+}

pkg/alertmanager/alertmanagernotify/receiver.go

@@ -2,8 +2,14 @@ package alertmanagernotify
 
 import (
 	"log/slog"
+	"slices"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/email"
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/msteamsv2"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/opsgenie"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/pagerduty"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/slack"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/webhook"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/prometheus/alertmanager/config/receiver"
 	"github.com/prometheus/alertmanager/notify"
@@ -11,6 +17,15 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+var customNotifierIntegrations = []string{
+	webhook.Integration,
+	email.Integration,
+	pagerduty.Integration,
+	opsgenie.Integration,
+	slack.Integration,
+	msteamsv2.Integration,
+}
+
 func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error) {
 	upstreamIntegrations, err := receiver.BuildReceiverIntegrations(nc, tmpl, logger)
 	if err != nil {
@@ -31,14 +46,29 @@ func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Templ
 	)
 
 	for _, integration := range upstreamIntegrations {
-		// skip upstream msteamsv2 integration
-		if integration.Name() != "msteamsv2" {
+		// skip upstream integration if we support custom integration for it
+		if !slices.Contains(customNotifierIntegrations, integration.Name()) {
 			integrations = append(integrations, integration)
 		}
 	}
 
+	for i, c := range nc.WebhookConfigs {
+		add(webhook.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) })
+	}
+	for i, c := range nc.EmailConfigs {
+		add(email.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil })
+	}
+	for i, c := range nc.PagerdutyConfigs {
+		add(pagerduty.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) })
+	}
+	for i, c := range nc.OpsGenieConfigs {
+		add(opsgenie.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) })
+	}
+	for i, c := range nc.SlackConfigs {
+		add(slack.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) })
+	}
 	for i, c := range nc.MSTeamsV2Configs {
-		add("msteamsv2", i, c, func(l *slog.Logger) (notify.Notifier, error) {
+		add(msteamsv2.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
 			return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l)
 		})
 	}

pkg/alertmanager/alertmanagernotify/slack/slack.go

@@ -0,0 +1,278 @@
+package slack
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "slack"
+)
+
+// https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
+const maxTitleLenRunes = 1024
+
+// Notifier implements a Notifier for Slack notifications.
+type Notifier struct {
+	conf    *config.SlackConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+
+	postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
+}
+
+// New returns a new Slack notification handler.
+func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Notifier{
+		conf:         c,
+		tmpl:         t,
+		logger:       l,
+		client:       client,
+		retrier:      &notify.Retrier{},
+		postJSONFunc: notify.PostJSON,
+	}, nil
+}
+
+// request is the request for sending a slack notification.
+type request struct {
+	Channel     string       `json:"channel,omitempty"`
+	Username    string       `json:"username,omitempty"`
+	IconEmoji   string       `json:"icon_emoji,omitempty"`
+	IconURL     string       `json:"icon_url,omitempty"`
+	LinkNames   bool         `json:"link_names,omitempty"`
+	Text        string       `json:"text,omitempty"`
+	Attachments []attachment `json:"attachments"`
+}
+
+// attachment is used to display a richly-formatted message block.
+type attachment struct {
+	Title      string               `json:"title,omitempty"`
+	TitleLink  string               `json:"title_link,omitempty"`
+	Pretext    string               `json:"pretext,omitempty"`
+	Text       string               `json:"text"`
+	Fallback   string               `json:"fallback"`
+	CallbackID string               `json:"callback_id"`
+	Fields     []config.SlackField  `json:"fields,omitempty"`
+	Actions    []config.SlackAction `json:"actions,omitempty"`
+	ImageURL   string               `json:"image_url,omitempty"`
+	ThumbURL   string               `json:"thumb_url,omitempty"`
+	Footer     string               `json:"footer"`
+	Color      string               `json:"color,omitempty"`
+	MrkdwnIn   []string             `json:"mrkdwn_in,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var err error
+
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With("group_key", key)
+	logger.DebugContext(ctx, "extracted group key")
+
+	var (
+		data     = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		tmplText = notify.TmplText(n.tmpl, data, &err)
+	)
+	var markdownIn []string
+
+	if len(n.conf.MrkdwnIn) == 0 {
+		markdownIn = []string{"fallback", "pretext", "text"}
+	} else {
+		markdownIn = n.conf.MrkdwnIn
+	}
+
+	title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), maxTitleLenRunes)
+	if truncated {
+		logger.WarnContext(ctx, "Truncated title", "max_runes", maxTitleLenRunes)
+	}
+	att := &attachment{
+		Title:      title,
+		TitleLink:  tmplText(n.conf.TitleLink),
+		Pretext:    tmplText(n.conf.Pretext),
+		Text:       tmplText(n.conf.Text),
+		Fallback:   tmplText(n.conf.Fallback),
+		CallbackID: tmplText(n.conf.CallbackID),
+		ImageURL:   tmplText(n.conf.ImageURL),
+		ThumbURL:   tmplText(n.conf.ThumbURL),
+		Footer:     tmplText(n.conf.Footer),
+		Color:      tmplText(n.conf.Color),
+		MrkdwnIn:   markdownIn,
+	}
+
+	numFields := len(n.conf.Fields)
+	if numFields > 0 {
+		fields := make([]config.SlackField, numFields)
+		for index, field := range n.conf.Fields {
+			// Check if short was defined for the field otherwise fallback to the global setting
+			var short bool
+			if field.Short != nil {
+				short = *field.Short
+			} else {
+				short = n.conf.ShortFields
+			}
+
+			// Rebuild the field by executing any templates and setting the new value for short
+			fields[index] = config.SlackField{
+				Title: tmplText(field.Title),
+				Value: tmplText(field.Value),
+				Short: &short,
+			}
+		}
+		att.Fields = fields
+	}
+
+	numActions := len(n.conf.Actions)
+	if numActions > 0 {
+		actions := make([]config.SlackAction, numActions)
+		for index, action := range n.conf.Actions {
+			slackAction := config.SlackAction{
+				Type:  tmplText(action.Type),
+				Text:  tmplText(action.Text),
+				URL:   tmplText(action.URL),
+				Style: tmplText(action.Style),
+				Name:  tmplText(action.Name),
+				Value: tmplText(action.Value),
+			}
+
+			if action.ConfirmField != nil {
+				slackAction.ConfirmField = &config.SlackConfirmationField{
+					Title:       tmplText(action.ConfirmField.Title),
+					Text:        tmplText(action.ConfirmField.Text),
+					OkText:      tmplText(action.ConfirmField.OkText),
+					DismissText: tmplText(action.ConfirmField.DismissText),
+				}
+			}
+
+			actions[index] = slackAction
+		}
+		att.Actions = actions
+	}
+
+	req := &request{
+		Channel:     tmplText(n.conf.Channel),
+		Username:    tmplText(n.conf.Username),
+		IconEmoji:   tmplText(n.conf.IconEmoji),
+		IconURL:     tmplText(n.conf.IconURL),
+		LinkNames:   n.conf.LinkNames,
+		Text:        tmplText(n.conf.MessageText),
+		Attachments: []attachment{*att},
+	}
+	if err != nil {
+		return false, err
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(req); err != nil {
+		return false, err
+	}
+
+	var u string
+	if n.conf.APIURL != nil {
+		u = n.conf.APIURL.String()
+	} else {
+		content, err := os.ReadFile(n.conf.APIURLFile)
+		if err != nil {
+			return false, err
+		}
+		u = strings.TrimSpace(string(content))
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeTimeout, "configured slack timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := n.postJSONFunc(ctx, n.client, u, &buf) //nolint:bodyclose
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.NewInternalf(errors.CodeInternal, "failed to post JSON to slack: %v", context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	// Use a retrier to generate an error message for non-200 responses and
+	// classify them as retriable or not.
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		err = errors.NewInternalf(errors.CodeInternal, "channel %q: %v", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+
+	// Slack web API might return errors with a 200 response code.
+	// https://slack.dev/node-slack-sdk/web-api#handle-errors
+	retry, err = checkResponseError(resp)
+	if err != nil {
+		err = errors.NewInternalf(errors.CodeInternal, "channel %q: %v", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.ClientErrorReason, err)
+	}
+
+	return retry, nil
+}
+
+// checkResponseError parses out the error message from Slack API response.
+func checkResponseError(resp *http.Response) (bool, error) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "could not read response body")
+	}
+
+	if strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") {
+		return checkJSONResponseError(body)
+	}
+	return checkTextResponseError(body)
+}
+
+// checkTextResponseError classifies plaintext responses from Slack.
+// A plaintext (non-JSON) response is successful if it's a string "ok".
+// This is typically a response for an Incoming Webhook
+// (https://api.slack.com/messaging/webhooks#handling_errors)
+func checkTextResponseError(body []byte) (bool, error) {
+	if !bytes.Equal(body, []byte("ok")) {
+		return false, errors.NewInternalf(errors.CodeInternal, "received an error response from Slack: %s", string(body))
+	}
+	return false, nil
+}
+
+// checkJSONResponseError classifies JSON responses from Slack.
+func checkJSONResponseError(body []byte) (bool, error) {
+	// response is for parsing out errors from the JSON response.
+	type response struct {
+		OK    bool   `json:"ok"`
+		Error string `json:"error"`
+	}
+
+	var data response
+	if err := json.Unmarshal(body, &data); err != nil {
+		return true, errors.NewInternalf(errors.CodeInternal, "could not unmarshal JSON response %q: %v", string(body), err)
+	}
+	if !data.OK {
+		return false, errors.NewInternalf(errors.CodeInternal, "error response from Slack: %s", data.Error)
+	}
+	return false, nil
+}

pkg/alertmanager/alertmanagernotify/slack/slack_test.go

@@ -0,0 +1,339 @@
+package slack
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestSlackRetry(t *testing.T) {
+	notifier, err := New(
+		&config.SlackConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestSlackRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURL:     &config.SecretURL{URL: u},
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestGettingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String())
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestTrimmingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test_newline")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestNotifier_Notify_WithReason(t *testing.T) {
+	tests := []struct {
+		name           string
+		statusCode     int
+		responseBody   string
+		expectedReason notify.Reason
+		expectedErr    string
+		expectedRetry  bool
+		noError        bool
+	}{
+		{
+			name:           "with a 4xx status code",
+			statusCode:     http.StatusUnauthorized,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 401",
+		},
+		{
+			name:           "with a 5xx status code",
+			statusCode:     http.StatusInternalServerError,
+			expectedReason: notify.ServerErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "unexpected status code 500",
+		},
+		{
+			name:           "with a 3xx status code",
+			statusCode:     http.StatusTemporaryRedirect,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 307",
+		},
+		{
+			name:           "with a 1xx status code",
+			statusCode:     http.StatusSwitchingProtocols,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 101",
+		},
+		{
+			name:           "2xx response with invalid JSON",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"not valid json"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "could not unmarshal",
+		},
+		{
+			name:           "2xx response with a JSON error",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"ok":false,"error":"error_message"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: error_message",
+		},
+		{
+			name:           "2xx response with a plaintext error",
+			statusCode:     http.StatusOK,
+			responseBody:   "no_channel",
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: no_channel",
+		},
+		{
+			name:         "successful JSON response",
+			statusCode:   http.StatusOK,
+			responseBody: `{"ok":true}`,
+			noError:      true,
+		},
+		{
+			name:         "successful plaintext response",
+			statusCode:   http.StatusOK,
+			responseBody: "ok",
+			noError:      true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			apiurl, _ := url.Parse("https://slack.com/post.Message")
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: apiurl},
+					Channel:        "channelname",
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				resp := httptest.NewRecorder()
+				if strings.HasPrefix(tt.responseBody, "{") {
+					resp.Header().Add("Content-Type", "application/json; charset=utf-8")
+				}
+				resp.WriteHeader(tt.statusCode)
+				_, _ = resp.WriteString(tt.responseBody)
+				return resp.Result(), nil
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			retry, err := notifier.Notify(ctx, alert1)
+			require.Equal(t, tt.expectedRetry, retry)
+			if tt.noError {
+				require.NoError(t, err)
+			} else {
+				var reasonError *notify.ErrorWithReason
+				require.ErrorAs(t, err, &reasonError)
+				require.Equal(t, tt.expectedReason, reasonError.Reason)
+				require.Contains(t, err.Error(), tt.expectedErr)
+				require.Contains(t, err.Error(), "channelname")
+			}
+		})
+	}
+}
+
+func TestSlackTimeout(t *testing.T) {
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			u, _ := url.Parse("https://slack.com/post.Message")
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: u},
+					Channel:        "channelname",
+					Timeout:        tt.timeout,
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				select {
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				case <-time.After(tt.latency):
+					resp := httptest.NewRecorder()
+					resp.Header().Set("Content-Type", "application/json; charset=utf-8")
+					resp.WriteHeader(http.StatusOK)
+					_, _ = resp.WriteString(`{"ok":true}`)
+
+					return resp.Result(), nil
+				}
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = notifier.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+func TestSlackMessageField(t *testing.T) {
+	// 1. Setup a fake Slack server
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatal(err)
+		}
+
+		// 2. VERIFY: Top-level text exists
+		if body["text"] != "My Top Level Message" {
+			t.Errorf("Expected top-level 'text' to be 'My Top Level Message', got %v", body["text"])
+		}
+
+		// 3. VERIFY: Old attachments still exist
+		attachments, ok := body["attachments"].([]any)
+		if !ok || len(attachments) == 0 {
+			t.Errorf("Expected attachments to exist")
+		} else {
+			first := attachments[0].(map[string]any)
+			if first["title"] != "Old Attachment Title" {
+				t.Errorf("Expected attachment title 'Old Attachment Title', got %v", first["title"])
+			}
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"ok": true}`))
+	}))
+	defer server.Close()
+
+	// 4. Configure Notifier with BOTH new and old fields
+	u, _ := url.Parse(server.URL)
+	conf := &config.SlackConfig{
+		APIURL:      &config.SecretURL{URL: u},
+		MessageText: "My Top Level Message", // Your NEW field
+		Title:       "Old Attachment Title", // An OLD field
+		Channel:     "#test-channel",
+		HTTPConfig:  &commoncfg.HTTPClientConfig{},
+	}
+
+	tmpl, err := template.FromGlobs([]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpl.ExternalURL = u
+
+	logger := slog.New(slog.DiscardHandler)
+	notifier, err := New(conf, tmpl, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group-key")
+
+	if _, err := notifier.Notify(ctx); err != nil {
+		t.Fatal("Notify failed:", err)
+	}
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook.go

@@ -0,0 +1,136 @@
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "webhook"
+)
+
+// Notifier implements a Notifier for generic webhooks.
+type Notifier struct {
+	conf    *config.WebhookConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new Webhook.
+func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*conf.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:   conf,
+		tmpl:   t,
+		logger: l,
+		client: client,
+		// Webhooks are assumed to respond with 2xx response codes on a successful
+		// request and 5xx response codes are assumed to be recoverable.
+		retrier: &notify.Retrier{},
+	}, nil
+}
+
+// Message defines the JSON object send to webhook endpoints.
+type Message struct {
+	*template.Data
+
+	// The protocol version.
+	Version         string `json:"version"`
+	GroupKey        string `json:"groupKey"`
+	TruncatedAlerts uint64 `json:"truncatedAlerts"`
+}
+
+func truncateAlerts(maxAlerts uint64, alerts []*types.Alert) ([]*types.Alert, uint64) {
+	if maxAlerts != 0 && uint64(len(alerts)) > maxAlerts {
+		return alerts[:maxAlerts], uint64(len(alerts)) - maxAlerts
+	}
+
+	return alerts, 0
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
+	alerts, numTruncated := truncateAlerts(n.conf.MaxAlerts, alerts)
+	data := notify.GetTemplateData(ctx, n.tmpl, alerts, n.logger)
+
+	groupKey, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	logger := n.logger.With("group_key", groupKey)
+	logger.DebugContext(ctx, "extracted group key")
+
+	msg := &Message{
+		Version:         "4",
+		Data:            data,
+		GroupKey:        groupKey.String(),
+		TruncatedAlerts: numTruncated,
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return false, err
+	}
+
+	var url string
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.URL != "" {
+		url = tmpl(string(n.conf.URL))
+	} else {
+		content, err := os.ReadFile(n.conf.URLFile)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "read url_file")
+		}
+		url = tmpl(strings.TrimSpace(string(content)))
+	}
+
+	if tmplErr != nil {
+		return false, errors.NewInternalf(errors.CodeInternal, "failed to template webhook URL: %v", tmplErr)
+	}
+
+	if url == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "webhook URL is empty after templating")
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeTimeout, "configured webhook timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, url, &buf) //nolint:bodyclose
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.NewInternalf(errors.CodeInternal, "failed to post JSON to webhook: %v", context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return shouldRetry, err
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook_test.go

@@ -0,0 +1,214 @@
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestWebhookRetry(t *testing.T) {
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL("http://example.com"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	if err != nil {
+		require.NoError(t, err)
+	}
+
+	t.Run("test retry status code", func(t *testing.T) {
+		for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+			actual, _ := notifier.retrier.Check(statusCode, nil)
+			require.Equal(t, expected, actual, "error on status %d", statusCode)
+		}
+	})
+
+	t.Run("test retry error details", func(t *testing.T) {
+		for _, tc := range []struct {
+			status int
+			body   io.Reader
+
+			exp string
+		}{
+			{
+				status: http.StatusBadRequest,
+				body: bytes.NewBuffer([]byte(
+					`{"status":"invalid event"}`,
+				)),
+
+				exp: fmt.Sprintf(`unexpected status code %d: {"status":"invalid event"}`, http.StatusBadRequest),
+			},
+			{
+				status: http.StatusBadRequest,
+
+				exp: fmt.Sprintf(`unexpected status code %d`, http.StatusBadRequest),
+			},
+		} {
+			t.Run("", func(t *testing.T) {
+				_, err = notifier.retrier.Check(tc.status, tc.body)
+				require.Equal(t, tc.exp, err.Error())
+			})
+		}
+	})
+}
+
+func TestWebhookTruncateAlerts(t *testing.T) {
+	alerts := make([]*types.Alert, 10)
+
+	truncatedAlerts, numTruncated := truncateAlerts(0, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(4, alerts)
+	require.Len(t, truncatedAlerts, 4)
+	require.EqualValues(t, 6, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(100, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+}
+
+func TestWebhookRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	secret := "secret"
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL(u.String()),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, secret)
+}
+
+func TestWebhookReadingURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "webhook_url")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.WebhookConfig{
+			URLFile:    f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestWebhookURLTemplating(t *testing.T) {
+	var calledURL string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calledURL = r.URL.Path
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	tests := []struct {
+		name           string
+		url            string
+		groupLabels    model.LabelSet
+		alertLabels    model.LabelSet
+		expectError    bool
+		expectedErrMsg string
+		expectedPath   string
+	}{
+		{
+			name:         "templating with alert labels",
+			url:          srv.URL + "/{{ .GroupLabels.alertname }}/{{ .CommonLabels.severity }}",
+			groupLabels:  model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:  model.LabelSet{"alertname": "TestAlert", "severity": "critical"},
+			expectError:  false,
+			expectedPath: "/TestAlert/critical",
+		},
+		{
+			name:           "invalid template field",
+			url:            srv.URL + "/{{ .InvalidField }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "failed to template webhook URL",
+		},
+		{
+			name:           "template renders to empty string",
+			url:            "{{ if .CommonLabels.nonexistent }}http://example.com{{ end }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "webhook URL is empty after templating",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			calledURL = "" // Reset for each test
+
+			notifier, err := New(
+				&config.WebhookConfig{
+					URL:        config.SecretTemplateURL(tc.url),
+					HTTPConfig: &commoncfg.HTTPClientConfig{},
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "test-group")
+			if tc.groupLabels != nil {
+				ctx = notify.WithGroupLabels(ctx, tc.groupLabels)
+			}
+
+			alerts := []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels:   tc.alertLabels,
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}
+
+			_, err = notifier.Notify(ctx, alerts...)
+
+			if tc.expectError {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expectedErrMsg)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedPath, calledURL)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate.go

@@ -0,0 +1,254 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+)
+
+// AlertManagerTemplater processes alert notification templates.
+type AlertManagerTemplater interface {
+	// ProcessTemplates expands the title and body templates from input
+	// against the provided alerts and returns the expanded templates.
+	ProcessTemplates(ctx context.Context, input TemplateInput, alerts []*types.Alert) (*ExpandedTemplates, error)
+}
+
+type alertManagerTemplater struct {
+	tmpl   *template.Template
+	logger *slog.Logger
+}
+
+func New(tmpl *template.Template, logger *slog.Logger) AlertManagerTemplater {
+	return &alertManagerTemplater{tmpl: tmpl, logger: logger}
+}
+
+// ProcessTemplates expands the title and body templates from input
+// against the provided alerts and returns the expanded templates.
+func (at *alertManagerTemplater) ProcessTemplates(
+	ctx context.Context,
+	input TemplateInput,
+	alerts []*types.Alert,
+) (*ExpandedTemplates, error) {
+	ntd := at.buildNotificationTemplateData(ctx, alerts)
+
+	title, titleMissingVars, err := at.expandTitle(ctx, input, alerts, ntd)
+	if err != nil {
+		return nil, err
+	}
+
+	body, bodyMissingVars, err := at.expandBody(ctx, input, alerts, ntd)
+	if err != nil {
+		return nil, err
+	}
+
+	missingVars := make(map[string]bool)
+	for k := range titleMissingVars {
+		missingVars[k] = true
+	}
+	for k := range bodyMissingVars {
+		missingVars[k] = true
+	}
+
+	return &ExpandedTemplates{Title: title, Body: body, MissingVars: missingVars}, nil
+}
+
+// expandTitle expands the title template. Falls back to the default if the custom template
+// result in empty string.
+func (at *alertManagerTemplater) expandTitle(
+	ctx context.Context,
+	input TemplateInput,
+	alerts []*types.Alert,
+	ntd *NotificationTemplateData,
+) (string, map[string]bool, error) {
+	if input.TitleTemplate != "" {
+		processRes, err := PreProcessTemplateAndData(input.TitleTemplate, ntd)
+		if err != nil {
+			return "", nil, err
+		}
+		result, err := at.tmpl.ExecuteTextString(processRes.Template, processRes.Data)
+		if err != nil {
+			return "", nil, errors.NewInternalf(errors.CodeInvalidInput, "failed to execute template: %s", err.Error())
+		}
+		if strings.TrimSpace(result) != "" {
+			return result, processRes.UnknownVars, nil
+		}
+	}
+
+	if input.DefaultTitleTemplate == "" {
+		return "", nil, nil
+	}
+	// Fall back to the default title template if present in the input
+	data := notify.GetTemplateData(ctx, at.tmpl, alerts, at.logger)
+	result, err := at.tmpl.ExecuteTextString(input.DefaultTitleTemplate, data)
+	return result, nil, err
+}
+
+// expandBody expands the body template once per alert and concatenates the results to return resulting body template
+// it falls back to the default templates if body template is empty or result in empty string.
+func (at *alertManagerTemplater) expandBody(
+	ctx context.Context,
+	input TemplateInput,
+	alerts []*types.Alert,
+	ntd *NotificationTemplateData,
+) (string, map[string]bool, error) {
+	if input.BodyTemplate != "" {
+		var sb strings.Builder
+		missingVars := make(map[string]bool)
+		for i := range ntd.Alerts {
+			processRes, err := PreProcessTemplateAndData(input.BodyTemplate, &ntd.Alerts[i])
+			if err != nil {
+				return "", nil, err
+			}
+			for k := range processRes.UnknownVars {
+				missingVars[k] = true
+			}
+			part, err := at.tmpl.ExecuteTextString(processRes.Template, processRes.Data)
+			if err != nil {
+				return "", nil, errors.NewInternalf(errors.CodeInvalidInput, "failed to execute template: %s", err.Error())
+			}
+			sb.WriteString(part)
+			// Add separator if not last alert
+			if i < len(ntd.Alerts)-1 {
+				sb.WriteString("<br><br>")
+			}
+		}
+		result := sb.String()
+		if strings.TrimSpace(result) != "" {
+			return result, missingVars, nil
+		}
+	}
+
+	if input.DefaultBodyTemplate == "" {
+		return "", nil, nil
+	}
+	// Fall back to the default body template if present in the input
+	data := notify.GetTemplateData(ctx, at.tmpl, alerts, at.logger)
+	result, err := at.tmpl.ExecuteTextString(input.DefaultBodyTemplate, data)
+	return result, nil, err
+}
+
+// buildNotificationTemplateData creates the NotificationTemplateData using
+// info from context and the raw alerts.
+func (at *alertManagerTemplater) buildNotificationTemplateData(
+	ctx context.Context,
+	alerts []*types.Alert,
+) *NotificationTemplateData {
+	// extract the required data from the context
+	receiver, ok := notify.ReceiverName(ctx)
+	if !ok {
+		at.logger.WarnContext(ctx, "missing receiver name in context")
+	}
+
+	groupLabels, ok := notify.GroupLabels(ctx)
+	if !ok {
+		at.logger.WarnContext(ctx, "missing group labels in context")
+	}
+
+	// extract the external URL from the template
+	externalURL := ""
+	if at.tmpl.ExternalURL != nil {
+		externalURL = at.tmpl.ExternalURL.String()
+	}
+
+	commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+	commonLabels := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
+
+	// aggregate labels and annotations from all alerts
+	labels := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
+	annotations := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+
+	// build the alert data slice
+	alertDataSlice := make([]AlertData, 0, len(alerts))
+	for _, a := range alerts {
+		ad := buildAlertData(a, receiver)
+		alertDataSlice = append(alertDataSlice, ad)
+	}
+
+	// count the number of firing and resolved alerts
+	var firing, resolved int
+	for _, ad := range alertDataSlice {
+		if ad.IsFiring {
+			firing++
+		} else if ad.IsResolved {
+			resolved++
+		}
+	}
+
+	// extract the rule-level convenience fields from common labels
+	alertName := commonLabels[ruletypes.LabelAlertName]
+	ruleID := commonLabels[ruletypes.LabelRuleId]
+	ruleLink := commonLabels[ruletypes.LabelRuleSource]
+
+	// build the group labels
+	gl := make(template.KV, len(groupLabels))
+	for k, v := range groupLabels {
+		gl[string(k)] = string(v)
+	}
+
+	// build the notification template data
+	return &NotificationTemplateData{
+		Receiver:          receiver,
+		Status:            string(types.Alerts(alerts...).Status()),
+		AlertName:         alertName,
+		RuleID:            ruleID,
+		RuleLink:          ruleLink,
+		TotalFiring:       firing,
+		TotalResolved:     resolved,
+		Alerts:            alertDataSlice,
+		GroupLabels:       gl,
+		CommonLabels:      commonLabels,
+		CommonAnnotations: commonAnnotations,
+		ExternalURL:       externalURL,
+		Labels:            labels,
+		Annotations:       annotations,
+	}
+}
+
+// buildAlertData converts a single *types.Alert into an AlertData.
+func buildAlertData(a *types.Alert, receiver string) AlertData {
+	labels := make(template.KV, len(a.Labels))
+	for k, v := range a.Labels {
+		labels[string(k)] = string(v)
+	}
+
+	annotations := make(template.KV, len(a.Annotations))
+	for k, v := range a.Annotations {
+		annotations[string(k)] = string(v)
+	}
+
+	status := string(a.Status())
+	isFiring := a.Status() == model.AlertFiring
+	isResolved := a.Status() == model.AlertResolved
+	isMissingData := labels[ruletypes.LabelNoData] == "true"
+
+	return AlertData{
+		Receiver:      receiver,
+		Status:        status,
+		Labels:        labels,
+		Annotations:   annotations,
+		StartsAt:      a.StartsAt,
+		EndsAt:        a.EndsAt,
+		GeneratorURL:  a.GeneratorURL,
+		Fingerprint:   a.Fingerprint().String(),
+		AlertName:     labels[ruletypes.LabelAlertName],
+		RuleID:        labels[ruletypes.LabelRuleId],
+		RuleLink:      labels[ruletypes.LabelRuleSource],
+		Severity:      labels[ruletypes.LabelSeverityName],
+		LogLink:       annotations[ruletypes.AnnotationRelatedLogs],
+		TraceLink:     annotations[ruletypes.AnnotationRelatedTraces],
+		Value:         annotations[ruletypes.AnnotationValue],
+		Threshold:     annotations[ruletypes.AnnotationThreshold],
+		CompareOp:     annotations[ruletypes.AnnotationCompareOp],
+		MatchType:     annotations[ruletypes.AnnotationMatchType],
+		IsFiring:      isFiring,
+		IsResolved:    isResolved,
+		IsMissingData: isMissingData,
+	}
+}

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate_test.go

@@ -0,0 +1,272 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"testing"
+	"time"
+
+	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// testSetup returns an AlertTemplater and a context pre-populated with group key,
+// receiver name, and group labels for use in tests.
+func testSetup(t *testing.T) (AlertManagerTemplater, context.Context) {
+	t.Helper()
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "slack")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": "TestAlert", "severity": "critical"})
+	logger := slog.New(slog.DiscardHandler)
+	return New(tmpl, logger), ctx
+}
+
+func createAlert(labels, annotations map[string]string, isFiring bool) *types.Alert {
+	ls := model.LabelSet{}
+	for k, v := range labels {
+		ls[model.LabelName(k)] = model.LabelValue(v)
+	}
+	ann := model.LabelSet{}
+	for k, v := range annotations {
+		ann[model.LabelName(k)] = model.LabelValue(v)
+	}
+	startsAt := time.Now()
+	var endsAt time.Time
+	if isFiring {
+		endsAt = startsAt.Add(time.Hour)
+	} else {
+		startsAt = startsAt.Add(-2 * time.Hour)
+		endsAt = startsAt.Add(-time.Hour)
+	}
+	return &types.Alert{Alert: model.Alert{Labels: ls, Annotations: ann, StartsAt: startsAt, EndsAt: endsAt}}
+}
+
+func TestExpandTemplates(t *testing.T) {
+	at, ctx := testSetup(t)
+
+	tests := []struct {
+		name            string
+		alerts          []*types.Alert
+		input           TemplateInput
+		wantTitle       string
+		wantBody        string
+		wantMissingVars []string
+		errorContains   string
+	}{
+		{
+			// High request throughput on a service — service is a custom label.
+			// $labels.service extracts the label value; $annotations.description pulls the annotation.
+			name: "new template: high request throughput for a service",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "HighRequestThroughput",
+						ruletypes.LabelSeverityName: "warning",
+						"service":                   "payment-service",
+					},
+					map[string]string{"description": "Request rate exceeded 10k/s"},
+					true,
+				),
+			},
+			input: TemplateInput{
+				TitleTemplate: "High request throughput for $service",
+				BodyTemplate: `The service $service is getting high request. Please investigate.
+Severity: $severity
+Status: $status
+Service: $service
+Description: $description`,
+			},
+			wantTitle: "High request throughput for payment-service",
+			wantBody: `The service payment-service is getting high request. Please investigate.
+Severity: warning
+Status: firing
+Service: payment-service
+Description: Request rate exceeded 10k/s`,
+		},
+		{
+			// Disk usage alert using old Go template syntax throughout.
+			// No custom templates — both title and body use the default fallback path.
+			name: "old template: disk usage high on database host",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "DiskUsageHigh",
+						ruletypes.LabelSeverityName: "critical",
+						"instance":                  "db-primary-01",
+					},
+					map[string]string{
+						"summary":        "Disk usage high on database host",
+						"description":    "Disk usage is high on the database host",
+						"related_logs":   "https://logs.example.com/search?q=DiskUsageHigh",
+						"related_traces": "https://traces.example.com/search?q=DiskUsageHigh",
+					},
+					true,
+				),
+			},
+			input: TemplateInput{
+				DefaultTitleTemplate: `[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
+     {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
+       {{" "}}(
+       {{- with .CommonLabels.Remove .GroupLabels.Names }}
+         {{- range $index, $label := .SortedPairs -}}
+           {{ if $index }}, {{ end }}
+           {{- $label.Name }}="{{ $label.Value -}}"
+         {{- end }}
+       {{- end -}}
+       )
+     {{- end }}`,
+				DefaultBodyTemplate: `{{ range .Alerts -}}
+     *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}
+
+     *Summary:* {{ .Annotations.summary }}
+     *Description:* {{ .Annotations.description }}
+     *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}
+     *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}
+
+     *Details:*
+       {{ range .Labels.SortedPairs }} • *{{ .Name }}:* {{ .Value }}
+       {{ end }}
+     {{ end }}`,
+			},
+			wantTitle: "[FIRING:1] DiskUsageHigh for  (instance=\"db-primary-01\")",
+			wantBody: `*Alert:* DiskUsageHigh - critical
+
+     *Summary:* Disk usage high on database host
+     *Description:* Disk usage is high on the database host
+     *RelatedLogs:* View in <https://logs.example.com/search?q=DiskUsageHigh|logs explorer>
+     *RelatedTraces:* View in <https://traces.example.com/search?q=DiskUsageHigh|traces explorer>
+
+     *Details:*
+        • *alertname:* DiskUsageHigh
+        • *instance:* db-primary-01
+        • *severity:* critical
+       
+     `,
+		},
+		{
+			// Pod crash loop on multiple pods — body is expanded once per alert
+			// and joined with "<br><br>", with the pod name pulled from labels.
+			name: "new template: pod crash loop on multiple pods, body per-alert",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-1"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-2"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-3"}, nil, true),
+			},
+			input: TemplateInput{
+				TitleTemplate: "$rule_name: $total_firing pods affected",
+				BodyTemplate:  "$labels.pod is crash looping",
+			},
+			wantTitle: "PodCrashLoop: 3 pods affected",
+			wantBody:  "api-worker-1 is crash looping<br><br>api-worker-2 is crash looping<br><br>api-worker-3 is crash looping",
+		},
+		{
+			// Incident partially resolved — one service still down, one recovered.
+			// Title shows the aggregate counts; body shows per-service status.
+			name: "new template: service degradation with mixed firing and resolved alerts",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "ServiceDown", "service": "auth-service"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "ServiceDown", "service": "payment-service"}, nil, false),
+			},
+			input: TemplateInput{
+				TitleTemplate: "$total_firing firing, $total_resolved resolved",
+				BodyTemplate:  "$labels.service ($status)",
+			},
+			wantTitle: "1 firing, 1 resolved",
+			wantBody:  "auth-service (firing)<br><br>payment-service (resolved)",
+		},
+		{
+			// $environment is not a known AlertData or NotificationTemplateData field,
+			// so it lands in MissingVars and renders as "<no value>" in the output.
+			name: "missing vars: unknown $environment variable in title",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: TemplateInput{
+				TitleTemplate: "[$environment] $rule_name",
+			},
+			wantTitle:       "[<no value>] HighCPU",
+			wantMissingVars: []string{"environment"},
+		},
+		{
+			// $runbook_url is not a known field — someone tried to embed a runbook link
+			// directly as a variable instead of via annotations.
+			name: "missing vars: unknown $runbook_url variable in body",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodOOMKilled", ruletypes.LabelSeverityName: "warning"}, nil, true),
+			},
+			input: TemplateInput{
+				BodyTemplate: "$rule_name: see runbook at $runbook_url",
+			},
+			wantBody:        "PodOOMKilled: see runbook at <no value>",
+			wantMissingVars: []string{"runbook_url"},
+		},
+		{
+			// Both title and body use unknown variables; MissingVars is the union of both.
+			name: "missing vars: unknown variables in both title and body",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighMemory", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: TemplateInput{
+				TitleTemplate: "[$environment] $rule_name and [{{ $service }}]",
+				BodyTemplate:  "$rule_name: see runbook at $runbook_url",
+			},
+			wantTitle:       "[<no value>] HighMemory and [<no value>]",
+			wantBody:        "HighMemory: see runbook at <no value>",
+			wantMissingVars: []string{"environment", "runbook_url", "service"},
+		},
+		{
+			// Custom title template that expands to only whitespace triggers the fallback,
+			// so the default title template is used instead.
+			name: "fallback: whitespace-only custom title falls back to default",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, false),
+			},
+			input: TemplateInput{
+				TitleTemplate:        "   ",
+				DefaultTitleTemplate: "{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})",
+				BodyTemplate:         "$rule_name ($severity) for $alertname",
+			},
+			wantTitle: "HighCPU (RESOLVED)",
+			wantBody:  "HighCPU (critical) for HighCPU",
+		},
+		{
+			name: "using non-existing function in template",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: TemplateInput{
+				TitleTemplate: "$rule_name ({{$severity | toUpperAndTrim}}) for $alertname",
+			},
+			errorContains: "function \"toUpperAndTrim\" not defined",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := at.ProcessTemplates(ctx, tc.input, tc.alerts)
+			if tc.errorContains != "" {
+				require.ErrorContains(t, err, tc.errorContains)
+				return
+			}
+			require.NoError(t, err)
+
+			if tc.wantTitle != "" {
+				require.Equal(t, tc.wantTitle, got.Title)
+			}
+			if tc.wantBody != "" {
+				require.Equal(t, tc.wantBody, got.Body)
+			}
+
+			require.Len(t, got.MissingVars, len(tc.wantMissingVars))
+			for _, v := range tc.wantMissingVars {
+				require.True(t, got.MissingVars[v], "expected %q in MissingVars", v)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/preprocessor.go

@@ -0,0 +1,242 @@
+package alertmanagertemplate
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/go-viper/mapstructure/v2"
+)
+
+// fieldMapping represents a mapping from a JSON tag name to its struct field name.
+type fieldMapping struct {
+	VarName   string // JSON tag name (e.g., "receiver", "rule_name")
+	FieldName string // Struct field name (e.g., "Receiver", "AlertName")
+}
+
+// extractFieldMappings uses reflection to extract field mappings from a struct.
+func extractFieldMappings(data any) []fieldMapping {
+	val := reflect.ValueOf(data)
+	// Handle pointer types
+	if val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return nil
+		}
+		val = val.Elem()
+	}
+	// return nil if the given data is not a struct
+	if val.Kind() != reflect.Struct {
+		return nil
+	}
+	typ := val.Type()
+
+	var mappings []fieldMapping
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		// Skip unexported fields
+		if !field.IsExported() {
+			continue
+		}
+		// Get JSON tag name
+		jsonTag := field.Tag.Get("json")
+		if jsonTag == "" || jsonTag == "-" {
+			continue
+		}
+		// Extract the name part (before any comma options like omitempty)
+		varName := strings.Split(jsonTag, ",")[0]
+		if varName == "" {
+			continue
+		}
+		varFieldName := field.Tag.Get("mapstructure")
+		if varFieldName == "" {
+			varFieldName = field.Name
+		}
+		// Skip complex types: slices and interfaces
+		kind := field.Type.Kind()
+		if kind == reflect.Slice || kind == reflect.Interface {
+			continue
+		}
+		// For struct types, we skip all but with few exceptions like time.Time
+		if kind == reflect.Struct {
+			// Allow time.Time which is commonly used
+			if field.Type.String() != "time.Time" {
+				continue
+			}
+		}
+
+		mappings = append(mappings, fieldMapping{
+			VarName:   varName,
+			FieldName: varFieldName,
+		})
+	}
+
+	return mappings
+}
+
+// extractNestedFieldsDefinitions adds the labels and annotations keys from the data struct to the template variable definitions
+// it takes the known data struct and extracts the labels and annotations maps and adds their keys to template variable definitions to be used in the template
+func extractNestedFieldsDefinitions(data any) map[string]string {
+	variables := make(map[string]string)
+
+	addLabelsAndAnnotations := func(labels, annotations map[string]string) {
+		for k := range annotations {
+			variables[k] = fmt.Sprintf("index .annotations \"%s\"", k)
+		}
+		for k := range labels {
+			variables[k] = fmt.Sprintf("index .labels \"%s\"", k)
+		}
+	}
+
+	switch data := data.(type) {
+	case *NotificationTemplateData:
+		addLabelsAndAnnotations(data.Labels, data.Annotations)
+	case *AlertData:
+		addLabelsAndAnnotations(data.Labels, data.Annotations)
+	default:
+		return variables
+	}
+
+	return variables
+}
+
+// prepareDataForTemplating prepares the data for templating by adding the labels and annotations values to the resulting map
+// so they can be accessed directly from root level, the predefined values takes precedence over the labels and annotations values
+// for example, if labels has a value called rule_name, which collides with the rule_name field in the data struct, the value from the data struct will take precedence
+func prepareDataForTemplating(data any) (map[string]interface{}, error) {
+	var result map[string]interface{}
+	if err := mapstructure.Decode(data, &result); err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to prepare data for templating")
+	}
+
+	addLabelsAndAnnotationsValues := func(labels, annotations map[string]string) {
+		for k, v := range labels {
+			if _, ok := result[k]; !ok {
+				result[k] = v
+			}
+		}
+		for k, v := range annotations {
+			if _, ok := result[k]; !ok {
+				result[k] = v
+			}
+		}
+	}
+
+	switch data := data.(type) {
+	case *NotificationTemplateData:
+		addLabelsAndAnnotationsValues(data.Labels, data.Annotations)
+	case *AlertData:
+		addLabelsAndAnnotationsValues(data.Labels, data.Annotations)
+	default:
+		return result, nil
+	}
+
+	return result, nil
+}
+
+// generateVariableDefinitions creates `{{ $varname := "" }}` declarations for each variable name.
+func generateVariableDefinitions(varNames map[string]string) string {
+	if len(varNames) == 0 {
+		return ""
+	}
+	var sb strings.Builder
+	for name := range varNames {
+		fmt.Fprintf(&sb, `{{ $%s := %s }}`, name, varNames[name])
+	}
+	return sb.String()
+}
+
+// buildVariableDefinitions constructs the full variable definition preamble for a template.
+// containing all known and unknown variables, the reason to include unknown variables is to
+// populate them with "<no value>" in template so go-text-template don't throw errors
+// when these variables are used in the template.
+func buildVariableDefinitions(tmpl string, data any) (string, map[string]bool, error) {
+	// Extract the initial fields from the data struct and add to the definitions
+	mappings := extractFieldMappings(data)
+
+	// Add variables from struct root level fields to the definitions
+	variables := make(map[string]string)
+	for _, m := range mappings {
+		variables[m.VarName] = fmt.Sprintf(".%s", m.FieldName)
+	}
+
+	// Extract the nested fields definitions from the data struct, like labels, annotations, etc.
+	// once extracted we add them to the variables map along with the field address
+	nestedVariables := extractNestedFieldsDefinitions(data)
+	for k, v := range nestedVariables {
+		variables[k] = v
+	}
+
+	// variables that are used throughout the template
+	usedVars, err := ExtractUsedVariables(tmpl)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// Compute unknown variables: used in template but not covered by a field mapping
+	probableUnknownVars := make(map[string]bool)
+	for name := range usedVars {
+		_, ok := variables[name]
+		if !ok {
+			probableUnknownVars[name] = true
+		}
+	}
+
+	// Add missing variables to the definitions with "<no value>"
+	// missingkey=zero is used to replace the missing value with "<no value>"
+	// but it only works when getting map values like {{ .keyfrommap }} from map and in struct this breaks
+	// with missing variable errors, we add missing variables in map so when directly variables
+	// are accessed directly in template block like {{ $variable }} it's handled and doesn't throw errors.
+	for name := range probableUnknownVars {
+		variables[name] = `"<no value>"`
+	}
+
+	return generateVariableDefinitions(variables), probableUnknownVars, nil
+}
+
+type ProcessingResult struct {
+	Template string
+	Data     map[string]interface{}
+	// UnknownVars is the set of possible unknown variables exptracted using regex
+	UnknownVars map[string]bool
+}
+
+// PreProcessTemplateAndData prepares a template string and struct data for Go template execution.
+//
+//	Input:  "$receiver has $rule_name in $status state"
+//	Output: "{{ $receiver := .Receiver }}...{{ $receiver }} has {{ $rule_name }} in {{ $status }} state"
+func PreProcessTemplateAndData(tmpl string, data any) (*ProcessingResult, error) {
+	// Handle empty template
+	unknownVars := make(map[string]bool)
+	if tmpl == "" {
+		result, err := prepareDataForTemplating(data)
+		if err != nil {
+			return nil, err
+		}
+		return &ProcessingResult{Data: result, UnknownVars: unknownVars}, nil
+	}
+
+	// Build variable definitions: known struct fields + fallback empty-string declarations
+	definitions, unknownVars, err := buildVariableDefinitions(tmpl, data)
+	if err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to build template definitions")
+	}
+
+	// Attach definitions prefix so WrapDollarVariables can parse the AST without "undefined variable" errors.
+	finalTmpl := definitions + tmpl
+
+	// Call WrapDollarVariables to transform bare $variable references to go-text-template format
+	// with {{ $variable }} syntax from $variable syntax
+	wrappedTmpl, err := WrapDollarVariables(finalTmpl)
+	if err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to prepare template for templating")
+	}
+
+	// Convert struct to map using mapstructure to be used for template execution
+	result, err := prepareDataForTemplating(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ProcessingResult{Template: wrappedTmpl, Data: result, UnknownVars: unknownVars}, nil
+}

pkg/alertmanager/alertmanagertemplate/preprocessor_test.go

@@ -0,0 +1,316 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/alertmanager/template"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExtractFieldMappings(t *testing.T) {
+	// Struct with various field types to test extraction logic
+	type TestStruct struct {
+		Name            string    `json:"name"`
+		Status          string    `json:"status"`
+		ActiveUserCount int       `json:"user_count" mapstructure:"active_user_count"`
+		IsActive        bool      `json:"is_active"`
+		CreatedAt       time.Time `json:"created_at"` // time.Time allowed
+		Items           []string  `json:"items"`      // slice skipped
+		unexported      string    // unexported skipped (no tag needed)
+		NoTag           string    // no json tag skipped
+		SkippedTag      string    `json:"-"` // json:"-" skipped
+	}
+
+	testCases := []struct {
+		name     string
+		data     any
+		expected []fieldMapping
+	}{
+		{
+			name: "struct with mixed field types",
+			data: TestStruct{Name: "test", ActiveUserCount: 5, unexported: ""},
+			expected: []fieldMapping{
+				{VarName: "name", FieldName: "Name"},
+				{VarName: "status", FieldName: "Status"},
+				{VarName: "user_count", FieldName: "active_user_count"},
+				{VarName: "is_active", FieldName: "IsActive"},
+				{VarName: "created_at", FieldName: "CreatedAt"},
+			},
+		},
+		{
+			name:     "nil data",
+			data:     nil,
+			expected: nil,
+		},
+		{
+			name:     "non-struct type",
+			data:     "string",
+			expected: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := extractFieldMappings(tc.data)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestBuildVariableDefinitions(t *testing.T) {
+	testCases := []struct {
+		name         string
+		tmpl         string
+		data         any
+		expectedVars []string // substrings that must appear in result
+		expectError  bool
+	}{
+		{
+			name: "empty template still returns struct field definitions",
+			tmpl: "",
+			data: &NotificationTemplateData{Receiver: "test"},
+			expectedVars: []string{
+				"{{ $receiver := .receiver }}",
+				"{{ $status := .status }}",
+			},
+		},
+		{
+			name: "mix of known and unknown vars",
+			tmpl: "$rule_name: $custom_label",
+			data: &AlertData{AlertName: "test", Status: "ok", Severity: "critical"},
+			expectedVars: []string{
+				"{{ $rule_name := .rule_name }}",
+				"{{ $status := .status }}",
+				"{{ $severity := .severity }}",
+				`{{ $custom_label := "<no value>" }}`,
+			},
+		},
+		{
+			name: "nested fields definitions coming from NotificationTemplateData",
+			tmpl: "$severity for $service",
+			data: &NotificationTemplateData{Labels: template.KV{
+				"severity": "critical",
+				"service":  "test",
+			}},
+			expectedVars: []string{
+				"{{ $severity := index .labels \"severity\" }}",
+				"{{ $service := index .labels \"service\" }}",
+			},
+		},
+		{
+			name: "nested fields definitions coming from AlertData",
+			tmpl: "$severity for $service",
+			data: &AlertData{Labels: template.KV{
+				"severity": "critical",
+				"service":  "test",
+			}},
+			expectedVars: []string{
+				"{{ $severity := index .labels \"severity\" }}",
+				"{{ $service := index .labels \"service\" }}",
+			},
+		},
+		{
+			name:        "invalid template syntax returns error",
+			tmpl:        "{{invalid",
+			data:        &NotificationTemplateData{},
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, _, err := buildVariableDefinitions(tc.tmpl, tc.data)
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			if len(tc.expectedVars) == 0 {
+				require.Empty(t, result)
+				return
+			}
+			for _, expected := range tc.expectedVars {
+				require.Contains(t, result, expected)
+			}
+		})
+	}
+}
+
+func TestPreProcessTemplateAndData(t *testing.T) {
+	testCases := []struct {
+		name                     string
+		tmpl                     string
+		data                     any
+		expectedTemplateContains []string
+		expectedData             map[string]any
+		expectedUnknownVars      map[string]bool
+		expectError              bool
+	}{
+		{
+			name: "NotificationTemplateData with dollar variables",
+			tmpl: "[$status] $rule_name (ID: $rule_id) - Firing: $total_firing, Resolved: $total_resolved, Severity: $severity",
+			data: &NotificationTemplateData{
+				Receiver:  "pagerduty",
+				Status:    "firing",
+				AlertName: "HighMemory",
+				RuleID:    "rule-123",
+				Labels: template.KV{
+					"severity": "critical",
+				},
+				TotalFiring:   3,
+				TotalResolved: 1,
+			},
+			expectedTemplateContains: []string{
+				"{{$status := .status}}",
+				"{{$rule_name := .rule_name}}",
+				"{{$rule_id := .rule_id}}",
+				"{{$total_firing := .total_firing}}",
+				"{{$total_resolved := .total_resolved}}",
+				"{{$severity := index .labels \"severity\"}}",
+				"[{{ .status }}] {{ .rule_name }} (ID: {{ .rule_id }}) - Firing: {{ .total_firing }}, Resolved: {{ .total_resolved }}",
+			},
+			expectedData: map[string]any{
+				"status":         "firing",
+				"rule_name":      "HighMemory",
+				"rule_id":        "rule-123",
+				"total_firing":   3,
+				"total_resolved": 1,
+				"severity":       "critical",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "AlertData with dollar variables",
+			tmpl: "$rule_name: Value $value exceeded $threshold (Status: $status, Severity: $severity, Description: $description)",
+			data: &AlertData{
+				Receiver:  "webhook",
+				Status:    "resolved",
+				AlertName: "DiskFull",
+				RuleID:    "disk-001",
+				Severity:  "warning",
+				Annotations: template.KV{
+					"description": "Disk full and cannot be written to",
+				},
+				Value:      "85%",
+				Threshold:  "80%",
+				IsFiring:   false,
+				IsResolved: true,
+			},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				"{{$value := .value}}",
+				"{{$threshold := .threshold}}",
+				"{{$status := .status}}",
+				"{{$severity := .severity}}",
+				"{{$description := index .annotations \"description\"}}",
+				"{{ .rule_name }}: Value {{ .value }} exceeded {{ .threshold }} (Status: {{ .status }}, Severity: {{ .severity }}, Description: {{ .description }})",
+			},
+			expectedData: map[string]any{
+				"status":      "resolved",
+				"rule_name":   "DiskFull",
+				"rule_id":     "disk-001",
+				"severity":    "warning",
+				"value":       "85%",
+				"threshold":   "80%",
+				"description": "Disk full and cannot be written to",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "mixed dollar and dot notation with both labels and annotations",
+			tmpl: "Alert $rule_name has {{.total_firing}} firing alerts",
+			data: &NotificationTemplateData{
+				AlertName:   "HighCPU",
+				TotalFiring: 5,
+				Labels: template.KV{
+					"value": "<MASKED VALUE>",
+				},
+				Annotations: template.KV{
+					"value": "85%",
+				},
+			},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				"{{$value := index .labels \"value\"}}",
+				"Alert {{ .rule_name }} has {{.total_firing}} firing alerts",
+			},
+			expectedData: map[string]any{
+				"rule_name":    "HighCPU",
+				"total_firing": 5,
+				"value":        "<MASKED VALUE>",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "empty template",
+			tmpl: "",
+			data: &NotificationTemplateData{Receiver: "slack"},
+		},
+		{
+			name:        "invalid template syntax",
+			tmpl:        "{{invalid",
+			data:        &NotificationTemplateData{},
+			expectError: true,
+		},
+		{
+			name: "unknown dollar var in text renders empty",
+			tmpl: "alert $custom_note fired",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				`{{$custom_note := "<no value>"}}`,
+				"alert {{ .custom_note }} fired",
+			},
+			expectedUnknownVars: map[string]bool{"custom_note": true},
+		},
+		{
+			name: "unknown dollar var in action block renders empty",
+			tmpl: "alert {{ $custom_note }} fired",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				`{{$custom_note := "<no value>"}}`,
+				`alert {{$custom_note}} fired`,
+			},
+			expectedUnknownVars: map[string]bool{"custom_note": true},
+		},
+		{
+			name: "mix of known and unknown vars",
+			tmpl: "$rule_name: $custom_label",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				`{{$custom_label := "<no value>"}}`,
+				"{{ .rule_name }}: {{ .custom_label }}",
+			},
+			expectedData:        map[string]any{"rule_name": "HighCPU"},
+			expectedUnknownVars: map[string]bool{"custom_label": true},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := PreProcessTemplateAndData(tc.tmpl, tc.data)
+
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			if tc.tmpl == "" {
+				require.Equal(t, "", result.Template)
+				return
+			}
+
+			for _, substr := range tc.expectedTemplateContains {
+				require.Contains(t, result.Template, substr)
+			}
+			for k, v := range tc.expectedData {
+				require.Equal(t, v, result.Data[k])
+			}
+			if tc.expectedUnknownVars != nil {
+				require.Equal(t, tc.expectedUnknownVars, result.UnknownVars)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/types.go

@@ -0,0 +1,86 @@
+package alertmanagertemplate
+
+import (
+	"time"
+
+	"github.com/prometheus/alertmanager/template"
+)
+
+// TemplateInput carries the title/body templates
+// and their defaults to apply in case the custom templates
+// are result in empty strings.
+type TemplateInput struct {
+	TitleTemplate        string
+	BodyTemplate         string
+	DefaultTitleTemplate string
+	DefaultBodyTemplate  string
+}
+
+// ExpandedTemplates is the result of ExpandTemplates.
+type ExpandedTemplates struct {
+	Title       string
+	Body        string
+	MissingVars map[string]bool // union of unknown vars from title + body templates
+}
+
+// AlertData holds per-alert data used when expanding body templates
+type AlertData struct {
+	Receiver     string      `json:"receiver" mapstructure:"receiver"`
+	Status       string      `json:"status" mapstructure:"status"`
+	Labels       template.KV `json:"labels" mapstructure:"labels"`
+	Annotations  template.KV `json:"annotations" mapstructure:"annotations"`
+	StartsAt     time.Time   `json:"starts_at" mapstructure:"starts_at"`
+	EndsAt       time.Time   `json:"ends_at" mapstructure:"ends_at"`
+	GeneratorURL string      `json:"generator_url" mapstructure:"generator_url"`
+	Fingerprint  string      `json:"fingerprint" mapstructure:"fingerprint"`
+
+	// Convenience fields extracted from well-known labels/annotations.
+	AlertName string `json:"rule_name" mapstructure:"rule_name"`
+	RuleID    string `json:"rule_id" mapstructure:"rule_id"`
+	RuleLink  string `json:"rule_link" mapstructure:"rule_link"`
+	Severity  string `json:"severity" mapstructure:"severity"`
+
+	// Alert internal data fields
+	Value     string `json:"value" mapstructure:"value"`
+	Threshold string `json:"threshold" mapstructure:"threshold"`
+	CompareOp string `json:"compare_op" mapstructure:"compare_op"`
+	MatchType string `json:"match_type" mapstructure:"match_type"`
+
+	// Link annotations added by the rule evaluator.
+	LogLink   string `json:"log_link" mapstructure:"log_link"`
+	TraceLink string `json:"trace_link" mapstructure:"trace_link"`
+
+	// Status booleans for easy conditional templating.
+	IsFiring      bool `json:"is_firing" mapstructure:"is_firing"`
+	IsResolved    bool `json:"is_resolved" mapstructure:"is_resolved"`
+	IsMissingData bool `json:"is_missing_data" mapstructure:"is_missing_data"`
+	IsRecovering  bool `json:"is_recovering" mapstructure:"is_recovering"`
+}
+
+// NotificationTemplateData is the top-level data struct provided to custom templates.
+type NotificationTemplateData struct {
+	Receiver string `json:"receiver" mapstructure:"receiver"`
+	Status   string `json:"status" mapstructure:"status"`
+
+	// Convenience fields for title templates.
+	AlertName     string `json:"rule_name" mapstructure:"rule_name"`
+	RuleID        string `json:"rule_id" mapstructure:"rule_id"`
+	RuleLink      string `json:"rule_link" mapstructure:"rule_link"`
+	TotalFiring   int    `json:"total_firing" mapstructure:"total_firing"`
+	TotalResolved int    `json:"total_resolved" mapstructure:"total_resolved"`
+
+	// Per-alert data, also available as filtered sub-slices.
+	Alerts []AlertData `json:"-" mapstructure:"-"`
+
+	// Cross-alert aggregates, computed as intersection across all alerts.
+	GroupLabels       template.KV `json:"group_labels" mapstructure:"group_labels"`
+	CommonLabels      template.KV `json:"common_labels" mapstructure:"common_labels"`
+	CommonAnnotations template.KV `json:"common_annotations" mapstructure:"common_annotations"`
+	ExternalURL       string      `json:"external_url" mapstructure:"external_url"`
+	// Labels and Annotations that are collection of labels
+	// and annotations from all alerts, it includes only the common labels and annotations
+	// and for non-common labels and annotations, it picks some first few labels/annotations
+	// and joins them with ", " to avoid blank values in the template
+	Labels      template.KV `json:"labels" mapstructure:"labels"`
+	Annotations template.KV `json:"annotations" mapstructure:"annotations"`
+}

pkg/alertmanager/alertmanagertemplate/utils.go

@@ -0,0 +1,234 @@
+package alertmanagertemplate
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+	"text/template/parse"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+)
+
+// maxAggregatedValues is the maximum number of unique values to include
+// when aggregating non-common label/annotation values across alerts.
+const maxAggregatedValues = 5
+
+// bareVariableRegex matches bare $variable references including dotted paths like $service.name.
+var bareVariableRegex = regexp.MustCompile(`\$(\w+(?:\.\w+)*)`)
+
+// bareVariableRegexFirstSeg matches only the base $variable name, stopping before any dotted path.
+// e.g. "$labels.severity" matches "$labels", "$name" matches "$name".
+var bareVariableRegexFirstSeg = regexp.MustCompile(`\$\w+`)
+
+// ExtractTemplatesFromAnnotations computes the common annotations across all alerts
+// and returns the values for the title_template and body_template annotation keys as title and body templates.
+func ExtractTemplatesFromAnnotations(alerts []*types.Alert) (titleTemplate, bodyTemplate string) {
+	if len(alerts) == 0 {
+		return "", ""
+	}
+
+	commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+	return commonAnnotations[ruletypes.AnnotationTitleTemplate], commonAnnotations[ruletypes.AnnotationBodyTemplate]
+}
+
+// WrapDollarVariables wraps bare $variable references in Go template syntax.
+// Example transformations:
+//   - "$name is $status" -> "{{ $name }} is {{ $status }}"
+//   - "$labels.severity" -> "{{ index .labels \"severity\" }}"
+//   - "$labels.http.status" -> "{{ index .labels \"http.status\" }}"
+//   - "$annotations.summary" -> "{{ index .annotations \"summary\" }}"
+//   - "$service.name" -> "{{ index . \"service.name\" }}"
+//   - "$name is {{ .Status }}" -> "{{ $name }} is {{ .Status }}"
+func WrapDollarVariables(src string) (string, error) {
+	if src == "" {
+		return src, nil
+	}
+
+	funcMap := alertmanagertypes.AdditionalFuncMap()
+	// Create a new parse.Tree directly
+	tree := parse.New("template")
+	tree.Mode = parse.SkipFuncCheck
+
+	// Parse the template
+	_, err := tree.Parse(src, "{{", "}}", make(map[string]*parse.Tree), funcMap)
+	if err != nil {
+		return "", err
+	}
+
+	// Walk the AST and transform TextNodes
+	walkAndWrapTextNodes(tree.Root)
+
+	// Return the reassembled template
+	return tree.Root.String(), nil
+}
+
+// walkAndWrapTextNodes recursively walks the parse tree trying to find a text node
+// once text node is found it wraps the bare $variable and changes it to index based
+// element access form datamap like .key or .key.subkey
+func walkAndWrapTextNodes(node parse.Node) {
+	if reflect.ValueOf(node).IsNil() {
+		return
+	}
+
+	switch n := node.(type) {
+	// `$name is {{.Status}}` is a list node with one text and one action node
+	case *parse.ListNode:
+		// Recurse into all child nodes
+		if n.Nodes != nil {
+			for _, child := range n.Nodes {
+				walkAndWrapTextNodes(child)
+			}
+		}
+
+	// `$name is ` is a text node with plain text in root
+	// we try to find the $name variable and wrap it with template block
+	// like `{{ .name }}`, for labels and annotations we use the index to access the value
+	// so `$labels.service` becomes `{{ index .labels "service" }}`
+	case *parse.TextNode:
+		// Transform $variable based on its pattern
+		n.Text = bareVariableRegex.ReplaceAllFunc(n.Text, func(match []byte) []byte {
+			// Extract variable name without the $
+			varName := string(match[1:])
+
+			// Check if variable contains dots
+			if strings.Contains(varName, ".") {
+				// Check for reserved prefixes: labels.* or annotations.*
+				if strings.HasPrefix(varName, "labels.") {
+					key := strings.TrimPrefix(varName, "labels.")
+					return []byte(fmt.Sprintf(`{{ index .labels "%s" }}`, key))
+				}
+				if strings.HasPrefix(varName, "annotations.") {
+					key := strings.TrimPrefix(varName, "annotations.")
+					return []byte(fmt.Sprintf(`{{ index .annotations "%s" }}`, key))
+				}
+				// Other dotted variables: index into root context
+				return []byte(fmt.Sprintf(`{{ index . "%s" }}`, varName))
+			}
+
+			// Simple variables: use dot notation to directly access the field
+			// without raising any error due to missing variables
+			return []byte(fmt.Sprintf("{{ .%s }}", varName))
+		})
+
+	// `{{if pipeline}} T1 {{else}} T0 {{end}}` is a if node with T1 part of List and T0 part of ElseList
+	case *parse.IfNode:
+		// Recurse into both branches
+		walkAndWrapTextNodes(n.List)
+		walkAndWrapTextNodes(n.ElseList)
+
+	// `{{range pipeline}} T1 {{else}} T0 {{end}}` is a range node with T1 part of List and T0 part of ElseList
+	case *parse.RangeNode:
+		// Recurse into both branches
+		walkAndWrapTextNodes(n.List)
+		walkAndWrapTextNodes(n.ElseList)
+
+		// All other node types (ActionNode, PipeNode, VariableNode, etc.) are already
+		// inside {{ }} action blocks and don't need transformation
+
+		// Support for `with` can be added later when we start supporting it in editor block
+	}
+}
+
+// ExtractUsedVariables returns the set of all $variable referenced in template
+// — text nodes, action blocks, branch conditions, and loop declarations — regardless of scope.
+// After finding all variables we find the ones which are not part of our alert data and handle them so
+// Go-text-template parser doesn't rejects undefined $variables
+func ExtractUsedVariables(src string) (map[string]bool, error) {
+	if src == "" {
+		return map[string]bool{}, nil
+	}
+
+	// Regex-scan raw template string to collect all $var base names.
+	// bareVariableRegexFirstSeg stops before dots, so "$labels.severity" yields "$labels".
+	used := make(map[string]bool)
+	for _, m := range bareVariableRegexFirstSeg.FindAll([]byte(src), -1) {
+		used[string(m[1:])] = true // strip leading "$"
+	}
+
+	// Build a preamble that pre-declares every found variable.
+	// This prevents "undefined variable" parse errors for $vars used in action
+	// blocks while still letting genuine syntax errors propagate.
+	var preamble strings.Builder
+	for name := range used {
+		fmt.Fprintf(&preamble, `{{$%s := ""}}`, name)
+	}
+
+	// Validate template syntax.
+	funcMap := alertmanagertypes.AdditionalFuncMap()
+	tree := parse.New("template")
+	tree.Mode = parse.SkipFuncCheck
+	if _, err := tree.Parse(preamble.String()+src, "{{", "}}", make(map[string]*parse.Tree), funcMap); err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInternal, "failed to extract used variables")
+	}
+
+	return used, nil
+}
+
+// aggregateKV aggregates key-value pairs (labels or annotations) from all alerts into a single template.KV
+// the result is used to populate the labels and annotations in the notification template data.
+// this is done to avoid blank values in the template when labels and annotations used are not common throughout the alerts
+func aggregateKV(alerts []*types.Alert, extractFn func(*types.Alert) model.LabelSet) template.KV {
+	// track unique values per key in order of first appearance
+	valuesPerKey := make(map[string][]string)
+	// track which values have been seen for deduplication
+	seenValues := make(map[string]map[string]bool)
+
+	for _, alert := range alerts {
+		kvPairs := extractFn(alert)
+		for k, v := range kvPairs {
+			key := string(k)
+			value := string(v)
+
+			if seenValues[key] == nil {
+				seenValues[key] = make(map[string]bool)
+			}
+
+			// only add if not already seen and under the limit of maxAggregatedValues
+			if !seenValues[key][value] && len(valuesPerKey[key]) < maxAggregatedValues {
+				seenValues[key][value] = true
+				valuesPerKey[key] = append(valuesPerKey[key], value)
+			}
+		}
+	}
+
+	// build the result by joining values
+	result := make(template.KV, len(valuesPerKey))
+	for key, values := range valuesPerKey {
+		result[key] = strings.Join(values, ", ")
+	}
+
+	return result
+}
+
+// extractCommonKV returns the intersection of key-value pairs across all alerts.
+// A key/value pair is included only if it appears identically on every alert.
+func extractCommonKV(alerts []*types.Alert, extractFn func(*types.Alert) model.LabelSet) template.KV {
+	if len(alerts) == 0 {
+		return template.KV{}
+	}
+
+	common := make(template.KV, len(extractFn(alerts[0])))
+	for k, v := range extractFn(alerts[0]) {
+		common[string(k)] = string(v)
+	}
+
+	for _, a := range alerts[1:] {
+		kv := extractFn(a)
+		for k := range common {
+			if string(kv[model.LabelName(k)]) != common[k] {
+				delete(common, k)
+			}
+		}
+		if len(common) == 0 {
+			break
+		}
+	}
+
+	return common
+}

pkg/alertmanager/alertmanagertemplate/utils_test.go

@@ -0,0 +1,348 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWrapBareVars(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    string
+		expectError bool
+	}{
+		{
+			name:     "mixed variables with actions",
+			input:    "$name is {{.Status}}",
+			expected: "{{ .name }} is {{.Status}}",
+		},
+		{
+			name: "nested variables in range",
+			input: `{{range .items}}
+				$title
+				{{end}}`,
+			expected: `{{range .items}}
+				{{ .title }}
+				{{end}}`,
+		},
+		{
+			name:     "nested variables in if else",
+			input:    "{{if .ok}}$a{{else}}$b{{end}}",
+			expected: "{{if .ok}}{{ .a }}{{else}}{{ .b }}{{end}}",
+		},
+		// Labels prefix: index into .labels map
+		{
+			name:     "labels variables prefix simple",
+			input:    "$labels.service",
+			expected: `{{ index .labels "service" }}`,
+		},
+		{
+			name:     "labels variables prefix nested with multiple dots",
+			input:    "$labels.http.status",
+			expected: `{{ index .labels "http.status" }}`,
+		},
+		{
+			name:     "multiple labels variables simple and nested",
+			input:    "$labels.service and $labels.instance.id",
+			expected: `{{ index .labels "service" }} and {{ index .labels "instance.id" }}`,
+		},
+		// Annotations prefix: index into .annotations map
+		{
+			name:     "annotations variables prefix simple",
+			input:    "$annotations.summary",
+			expected: `{{ index .annotations "summary" }}`,
+		},
+		{
+			name:     "annotations variables prefix nested with multiple dots",
+			input:    "$annotations.alert.url",
+			expected: `{{ index .annotations "alert.url" }}`,
+		},
+		// Other dotted paths: index into root context
+		{
+			name:     "other variables with multiple dots",
+			input:    "$service.name",
+			expected: `{{ index . "service.name" }}`,
+		},
+		{
+			name:     "other variables with multiple dots nested",
+			input:    "$http.status.code",
+			expected: `{{ index . "http.status.code" }}`,
+		},
+		// Hybrid: all types combined
+		{
+			name:     "hybrid - all variables types",
+			input:    "Alert: $alert_name Labels: $labels.severity Annotations: $annotations.desc Service: $service.name Count: $error_count",
+			expected: `Alert: {{ .alert_name }} Labels: {{ index .labels "severity" }} Annotations: {{ index .annotations "desc" }} Service: {{ index . "service.name" }} Count: {{ .error_count }}`,
+		},
+		{
+			name:     "already wrapped should not be changed",
+			input:    "{{$status := .status}}{{.name}} is {{$status | toUpper}}",
+			expected: "{{$status := .status}}{{.name}} is {{$status | toUpper}}",
+		},
+		{
+			name:     "no variables should not be changed",
+			input:    "Hello world",
+			expected: "Hello world",
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "deeply nested",
+			input:    "{{range .items}}{{if .ok}}$deep{{end}}{{end}}",
+			expected: "{{range .items}}{{if .ok}}{{ .deep }}{{end}}{{end}}",
+		},
+		{
+			name: "complex example",
+			input: `Hello $name, your score is $score.
+				{{if .isAdmin}}
+					Welcome back $name, you have {{.unreadCount}} messages.
+				{{end}}`,
+			expected: `Hello {{ .name }}, your score is {{ .score }}.
+				{{if .isAdmin}}
+					Welcome back {{ .name }}, you have {{.unreadCount}} messages.
+				{{end}}`,
+		},
+		{
+			name:     "with custom function",
+			input:    "$name triggered at {{urlescape .url}}",
+			expected: "{{ .name }} triggered at {{urlescape .url}}",
+		},
+		{
+			name:        "invalid template",
+			input:       "{{invalid",
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := WrapDollarVariables(tc.input)
+
+			if tc.expectError {
+				require.Error(t, err, "should error on invalid template syntax")
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestExtractUsedVariables(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    map[string]bool
+		expectError bool
+	}{
+		{
+			name:     "simple usage in text",
+			input:    "$name is $status",
+			expected: map[string]bool{"name": true, "status": true},
+		},
+		{
+			name:     "declared in action block",
+			input:    "{{ $name := .name }}",
+			expected: map[string]bool{"name": true},
+		},
+		{
+			name:     "range loop vars",
+			input:    "{{ range $i, $v := .items }}{{ end }}",
+			expected: map[string]bool{"i": true, "v": true},
+		},
+		{
+			name:     "mixed text and action",
+			input:    "$x and {{ $y }}",
+			expected: map[string]bool{"x": true, "y": true},
+		},
+		{
+			name:     "dotted path in text extracts base only",
+			input:    "$labels.severity",
+			expected: map[string]bool{"labels": true},
+		},
+		{
+			name:     "nested if else",
+			input:    "{{ if .ok }}{{ $a }}{{ else }}{{ $b }}{{ end }}",
+			expected: map[string]bool{"a": true, "b": true},
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: map[string]bool{},
+		},
+		{
+			name:     "no variables",
+			input:    "Hello world",
+			expected: map[string]bool{},
+		},
+		{
+			name:        "invalid template returns error",
+			input:       "{{invalid",
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := ExtractUsedVariables(tc.input)
+
+			if tc.expectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestAggregateKV(t *testing.T) {
+	extractLabels := func(a *types.Alert) model.LabelSet { return a.Labels }
+
+	testCases := []struct {
+		name      string
+		alerts    []*types.Alert
+		extractFn func(*types.Alert) model.LabelSet
+		expected  template.KV
+	}{
+		{
+			name:      "empty alerts slice",
+			alerts:    []*types.Alert{},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "single alert",
+			alerts: []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels: model.LabelSet{
+							"env":     "production",
+							"service": "backend",
+						},
+					},
+				},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"env":     "production",
+				"service": "backend",
+			},
+		},
+		{
+			name: "varying values with duplicates deduped",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "backend"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "frontend"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "api"}}},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"env":     "production",
+				"service": "backend, api, frontend",
+			},
+		},
+		{
+			name: "more than 5 unique values truncates to 5",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc1"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc2"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc3"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc4"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc5"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc6"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc7"}}},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"service": "svc1, svc2, svc3, svc4, svc5",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := aggregateKV(tc.alerts, tc.extractFn)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestExtractCommonKV(t *testing.T) {
+	extractLabels := func(a *types.Alert) model.LabelSet { return a.Labels }
+	extractAnnotations := func(a *types.Alert) model.LabelSet { return a.Annotations }
+
+	testCases := []struct {
+		name      string
+		alerts    []*types.Alert
+		extractFn func(*types.Alert) model.LabelSet
+		expected  template.KV
+	}{
+		{
+			name:      "empty alerts slice",
+			alerts:    []*types.Alert{},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "single alert returns all labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "api"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod", "service": "api"},
+		},
+		{
+			name: "multiple alerts with fully common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "region": "us-east"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "region": "us-east"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod", "region": "us-east"},
+		},
+		{
+			name: "multiple alerts with partially common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "worker"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod"},
+		},
+		{
+			name: "multiple alerts with no common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "worker"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "annotations extract common annotations",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Annotations: model.LabelSet{"summary": "high cpu", "runbook": "http://x"}}},
+				{Alert: model.Alert{Annotations: model.LabelSet{"summary": "high cpu", "runbook": "http://y"}}},
+			},
+			extractFn: extractAnnotations,
+			expected:  template.KV{"summary": "high cpu"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := extractCommonKV(tc.alerts, tc.extractFn)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}

pkg/errors/errors.go

@@ -147,7 +147,7 @@ func Ast(cause error, typ typ) bool {
 	return t == typ
 }
 
-// Ast checks if the provided error matches the specified custom error code.
+// Asc checks if the provided error matches the specified custom error code.
 func Asc(cause error, code Code) bool {
 	_, c, _, _, _, _ := Unwrapb(cause)
 

pkg/identn/apikeyidentn/identn.go

@@ -17,28 +17,34 @@ import (
 // todo: will move this in types layer with service account integration
 type apiKeyTokenKey struct{}
 
-type resolver struct {
+type provider struct {
 	store    sqlstore.SQLStore
-	headers  []string
+	config   identn.Config
 	settings factory.ScopedProviderSettings
 	sfGroup  *singleflight.Group
 }
 
-func New(providerSettings factory.ProviderSettings, store sqlstore.SQLStore, headers []string) identn.IdentN {
-	return &resolver{
-		store:    store,
-		headers:  headers,
-		settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn/apikeyidentn"),
-		sfGroup:  &singleflight.Group{},
-	}
+func NewFactory(store sqlstore.SQLStore) factory.ProviderFactory[identn.IdentN, identn.Config] {
+	return factory.NewProviderFactory(factory.MustNewName(authtypes.IdentNProviderAPIkey.StringValue()), func(ctx context.Context, providerSettings factory.ProviderSettings, config identn.Config) (identn.IdentN, error) {
+		return New(providerSettings, store, config)
+	})
 }
 
-func (r *resolver) Name() authtypes.IdentNProvider {
+func New(providerSettings factory.ProviderSettings, store sqlstore.SQLStore, config identn.Config) (identn.IdentN, error) {
+	return &provider{
+		store:    store,
+		config:   config,
+		settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn/apikeyidentn"),
+		sfGroup:  &singleflight.Group{},
+	}, nil
+}
+
+func (provider *provider) Name() authtypes.IdentNProvider {
 	return authtypes.IdentNProviderAPIkey
 }
 
-func (r *resolver) Test(req *http.Request) bool {
-	for _, header := range r.headers {
+func (provider *provider) Test(req *http.Request) bool {
+	for _, header := range provider.config.APIKeyConfig.Headers {
 		if req.Header.Get(header) != "" {
 			return true
 		}
@@ -46,8 +52,12 @@ func (r *resolver) Test(req *http.Request) bool {
 	return false
 }
 
-func (r *resolver) Pre(req *http.Request) *http.Request {
-	token := r.extractToken(req)
+func (provider *provider) Enabled() bool {
+	return provider.config.APIKeyConfig.Enabled
+}
+
+func (provider *provider) Pre(req *http.Request) *http.Request {
+	token := provider.extractToken(req)
 	if token == "" {
 		return req
 	}
@@ -56,16 +66,16 @@ func (r *resolver) Pre(req *http.Request) *http.Request {
 	return req.WithContext(ctx)
 }
 
-func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
+func (provider *provider) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
 	ctx := req.Context()
-
 	apiKeyToken, ok := ctx.Value(apiKeyTokenKey{}).(string)
 	if !ok || apiKeyToken == "" {
 		return nil, errors.New(errors.TypeUnauthenticated, errors.CodeUnauthenticated, "missing api key")
 	}
 
 	var apiKey types.StorableAPIKey
-	err := r.store.
+	err := provider.
+		store.
 		BunDB().
 		NewSelect().
 		Model(&apiKey).
@@ -80,7 +90,8 @@ func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
 	}
 
 	var user types.User
-	err = r.store.
+	err = provider.
+		store.
 		BunDB().
 		NewSelect().
 		Model(&user).
@@ -99,14 +110,15 @@ func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
 	return &identity, nil
 }
 
-func (r *resolver) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims) {
+func (provider *provider) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims) {
 	apiKeyToken, ok := ctx.Value(apiKeyTokenKey{}).(string)
 	if !ok || apiKeyToken == "" {
 		return
 	}
 
-	_, _, _ = r.sfGroup.Do(apiKeyToken, func() (any, error) {
-		_, err := r.store.
+	_, _, _ = provider.sfGroup.Do(apiKeyToken, func() (any, error) {
+		_, err := provider.
+			store.
 			BunDB().
 			NewUpdate().
 			Model(new(types.StorableAPIKey)).
@@ -115,14 +127,14 @@ func (r *resolver) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims
 			Where("revoked = false").
 			Exec(ctx)
 		if err != nil {
-			r.settings.Logger().ErrorContext(ctx, "failed to update last used of api key", "error", err)
+			provider.settings.Logger().ErrorContext(ctx, "failed to update last used of api key", "error", err)
 		}
 		return true, nil
 	})
 }
 
-func (r *resolver) extractToken(req *http.Request) string {
-	for _, header := range r.headers {
+func (provider *provider) extractToken(req *http.Request) string {
+	for _, header := range provider.config.APIKeyConfig.Headers {
 		if v := req.Header.Get(header); v != "" {
 			return v
 		}

pkg/identn/config.go

@@ -0,0 +1,48 @@
+package identn
+
+import (
+	"github.com/SigNoz/signoz/pkg/factory"
+)
+
+type Config struct {
+	// Config for tokenizer identN resolver
+	Tokenizer TokenizerConfig `mapstructure:"tokenizer"`
+
+	// Config for apikey identN resolver
+	APIKeyConfig APIKeyConfig `mapstructure:"apikey"`
+}
+
+type TokenizerConfig struct {
+	// Toggles the identN resolver
+	Enabled bool `mapstructure:"enabled"`
+	// Headers to extract from incoming requests
+	Headers []string `mapstructure:"headers"`
+}
+
+type APIKeyConfig struct {
+	// Toggles the identN resolver
+	Enabled bool `mapstructure:"enabled"`
+	// Headers to extract from incoming requests
+	Headers []string `mapstructure:"headers"`
+}
+
+func NewConfigFactory() factory.ConfigFactory {
+	return factory.NewConfigFactory(factory.MustNewName("identn"), newConfig)
+}
+
+func newConfig() factory.Config {
+	return &Config{
+		Tokenizer: TokenizerConfig{
+			Enabled: true,
+			Headers: []string{"Authorization", "Sec-WebSocket-Protocol"},
+		},
+		APIKeyConfig: APIKeyConfig{
+			Enabled: true,
+			Headers: []string{"SIGNOZ-API-KEY"},
+		},
+	}
+}
+
+func (c Config) Validate() error {
+	return nil
+}

pkg/identn/identn.go

@@ -23,6 +23,8 @@ type IdentN interface {
 	GetIdentity(r *http.Request) (*authtypes.Identity, error)
 
 	Name() authtypes.IdentNProvider
+
+	Enabled() bool
 }
 
 // IdentNWithPreHook is optionally implemented by resolvers that need to

pkg/identn/resolver.go

@@ -12,8 +12,16 @@ type identNResolver struct {
 }
 
 func NewIdentNResolver(providerSettings factory.ProviderSettings, identNs ...IdentN) IdentNResolver {
+	enabledIdentNs := []IdentN{}
+
+	for _, identN := range identNs {
+		if identN.Enabled() {
+			enabledIdentNs = append(enabledIdentNs, identN)
+		}
+	}
+
 	return &identNResolver{
-		identNs:  identNs,
+		identNs:  enabledIdentNs,
 		settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn"),
 	}
 }

pkg/identn/tokenizeridentn/identn.go

@@ -13,28 +13,34 @@ import (
 	"golang.org/x/sync/singleflight"
 )
 
-type resolver struct {
+type provider struct {
 	tokenizer tokenizer.Tokenizer
-	headers   []string
+	config    identn.Config
 	settings  factory.ScopedProviderSettings
 	sfGroup   *singleflight.Group
 }
 
-func New(providerSettings factory.ProviderSettings, tokenizer tokenizer.Tokenizer, headers []string) identn.IdentN {
-	return &resolver{
-		tokenizer: tokenizer,
-		headers:   headers,
-		settings:  factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn/tokenizeridentn"),
-		sfGroup:   &singleflight.Group{},
-	}
+func NewFactory(tokenizer tokenizer.Tokenizer) factory.ProviderFactory[identn.IdentN, identn.Config] {
+	return factory.NewProviderFactory(factory.MustNewName(authtypes.IdentNProviderTokenizer.StringValue()), func(ctx context.Context, providerSettings factory.ProviderSettings, config identn.Config) (identn.IdentN, error) {
+		return New(providerSettings, tokenizer, config)
+	})
 }
 
-func (r *resolver) Name() authtypes.IdentNProvider {
+func New(providerSettings factory.ProviderSettings, tokenizer tokenizer.Tokenizer, config identn.Config) (identn.IdentN, error) {
+	return &provider{
+		tokenizer: tokenizer,
+		config:    config,
+		settings:  factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn/tokenizeridentn"),
+		sfGroup:   &singleflight.Group{},
+	}, nil
+}
+
+func (provider *provider) Name() authtypes.IdentNProvider {
 	return authtypes.IdentNProviderTokenizer
 }
 
-func (r *resolver) Test(req *http.Request) bool {
-	for _, header := range r.headers {
+func (provider *provider) Test(req *http.Request) bool {
+	for _, header := range provider.config.Tokenizer.Headers {
 		if req.Header.Get(header) != "" {
 			return true
 		}
@@ -42,8 +48,12 @@ func (r *resolver) Test(req *http.Request) bool {
 	return false
 }
 
-func (r *resolver) Pre(req *http.Request) *http.Request {
-	accessToken := r.extractToken(req)
+func (provider *provider) Enabled() bool {
+	return provider.config.Tokenizer.Enabled
+}
+
+func (provider *provider) Pre(req *http.Request) *http.Request {
+	accessToken := provider.extractToken(req)
 	if accessToken == "" {
 		return req
 	}
@@ -52,7 +62,7 @@ func (r *resolver) Pre(req *http.Request) *http.Request {
 	return req.WithContext(ctx)
 }
 
-func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
+func (provider *provider) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
 	ctx := req.Context()
 
 	accessToken, err := authtypes.AccessTokenFromContext(ctx)
@@ -60,41 +70,45 @@ func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
 		return nil, err
 	}
 
-	return r.tokenizer.GetIdentity(ctx, accessToken)
+	return provider.tokenizer.GetIdentity(ctx, accessToken)
 }
 
-func (r *resolver) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims) {
+func (provider *provider) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims) {
+	if !provider.config.Tokenizer.Enabled {
+		return
+	}
+
 	accessToken, err := authtypes.AccessTokenFromContext(ctx)
 	if err != nil {
 		return
 	}
 
-	_, _, _ = r.sfGroup.Do(accessToken, func() (any, error) {
-		if err := r.tokenizer.SetLastObservedAt(ctx, accessToken, time.Now()); err != nil {
-			r.settings.Logger().ErrorContext(ctx, "failed to set last observed at", "error", err)
+	_, _, _ = provider.sfGroup.Do(accessToken, func() (any, error) {
+		if err := provider.tokenizer.SetLastObservedAt(ctx, accessToken, time.Now()); err != nil {
+			provider.settings.Logger().ErrorContext(ctx, "failed to set last observed at", "error", err)
 			return false, err
 		}
 		return true, nil
 	})
 }
 
-func (r *resolver) extractToken(req *http.Request) string {
+func (provider *provider) extractToken(req *http.Request) string {
 	var value string
-	for _, header := range r.headers {
+	for _, header := range provider.config.Tokenizer.Headers {
 		if v := req.Header.Get(header); v != "" {
 			value = v
 			break
 		}
 	}
 
-	accessToken, ok := r.parseBearerAuth(value)
+	accessToken, ok := provider.parseBearerAuth(value)
 	if !ok {
 		return value
 	}
 	return accessToken
 }
 
-func (r *resolver) parseBearerAuth(auth string) (string, bool) {
+func (provider *provider) parseBearerAuth(auth string) (string, bool) {
 	const prefix = "Bearer "
 	if len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {
 		return "", false

pkg/signoz/config.go

@@ -20,6 +20,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/gateway"
 	"github.com/SigNoz/signoz/pkg/global"
+	"github.com/SigNoz/signoz/pkg/identn"
 	"github.com/SigNoz/signoz/pkg/instrumentation"
 	"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
 	"github.com/SigNoz/signoz/pkg/modules/user"
@@ -113,6 +114,9 @@ type Config struct {
 
 	// User config
 	User user.Config `mapstructure:"user"`
+
+	// IdentN config
+	IdentN identn.Config `mapstructure:"identn"`
 }
 
 // DeprecatedFlags are the flags that are deprecated and scheduled for removal.
@@ -176,6 +180,7 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
 		metricsexplorer.NewConfigFactory(),
 		flagger.NewConfigFactory(),
 		user.NewConfigFactory(),
+		identn.NewConfigFactory(),
 	}
 
 	conf, err := config.New(ctx, resolverConfig, configFactories)

pkg/signoz/provider.go

@@ -22,6 +22,9 @@ import (
 	"github.com/SigNoz/signoz/pkg/flagger/configflagger"
 	"github.com/SigNoz/signoz/pkg/global"
 	"github.com/SigNoz/signoz/pkg/global/signozglobal"
+	"github.com/SigNoz/signoz/pkg/identn"
+	"github.com/SigNoz/signoz/pkg/identn/apikeyidentn"
+	"github.com/SigNoz/signoz/pkg/identn/tokenizeridentn"
 	"github.com/SigNoz/signoz/pkg/modules/authdomain/implauthdomain"
 	"github.com/SigNoz/signoz/pkg/modules/organization"
 	"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
@@ -271,6 +274,13 @@ func NewTokenizerProviderFactories(cache cache.Cache, sqlstore sqlstore.SQLStore
 	)
 }
 
+func NewIdentNProviderFactories(sqlstore sqlstore.SQLStore, tokenizer tokenizer.Tokenizer) factory.NamedMap[factory.ProviderFactory[identn.IdentN, identn.Config]] {
+	return factory.MustNewNamedMap(
+		tokenizeridentn.NewFactory(tokenizer),
+		apikeyidentn.NewFactory(sqlstore),
+	)
+}
+
 func NewGlobalProviderFactories() factory.NamedMap[factory.ProviderFactory[global.Global, global.Config]] {
 	return factory.MustNewNamedMap(
 		signozglobal.NewFactory(),

pkg/signoz/signoz.go

@@ -17,8 +17,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/gateway"
 	"github.com/SigNoz/signoz/pkg/identn"
-	"github.com/SigNoz/signoz/pkg/identn/apikeyidentn"
-	"github.com/SigNoz/signoz/pkg/identn/tokenizeridentn"
 	"github.com/SigNoz/signoz/pkg/instrumentation"
 	"github.com/SigNoz/signoz/pkg/licensing"
 	"github.com/SigNoz/signoz/pkg/modules/dashboard"
@@ -395,9 +393,16 @@ func New(
 	modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, analytics, querier, telemetrystore, telemetryMetadataStore, authNs, authz, cache, queryParser, config, dashboard, userGetter)
 
 	// Initialize identN resolver
-	tokenizeridentN := tokenizeridentn.New(providerSettings, tokenizer, []string{"Authorization", "Sec-WebSocket-Protocol"})
-	apikeyIdentN := apikeyidentn.New(providerSettings, sqlstore, []string{"SIGNOZ-API-KEY"})
-	identNResolver := identn.NewIdentNResolver(providerSettings, tokenizeridentN, apikeyIdentN)
+	identNFactories := NewIdentNProviderFactories(sqlstore, tokenizer)
+	identNs := []identn.IdentN{}
+	for _, identNFactory := range identNFactories.GetInOrder() {
+		identN, err := identNFactory.New(ctx, providerSettings, config.IdentN)
+		if err != nil {
+			return nil, err
+		}
+		identNs = append(identNs, identN)
+	}
+	identNResolver := identn.NewIdentNResolver(providerSettings, identNs...)
 
 	userService := impluser.NewService(providerSettings, impluser.NewStore(sqlstore, providerSettings), modules.User, orgGetter, authz, config.User.Root)
 

pkg/sqlschema/index.go

@@ -8,8 +8,9 @@ import (
 )
 
 var (
-	IndexTypeUnique = IndexType{s: valuer.NewString("uq")}
-	IndexTypeIndex  = IndexType{s: valuer.NewString("ix")}
+	IndexTypeUnique        = IndexType{s: valuer.NewString("uq")}
+	IndexTypeIndex         = IndexType{s: valuer.NewString("ix")}
+	IndexTypePartialUnique = IndexType{s: valuer.NewString("puq")}
 )
 
 type IndexType struct{ s valuer.String }
@@ -22,6 +23,7 @@ type Index interface {
 	// The name of the index.
 	//   - Indexes are named as `ix_<table_name>_<column_names>`. The column names are separated by underscores.
 	//   - Unique constraints are named as `uq_<table_name>_<column_names>`. The column names are separated by underscores.
+	//   - Partial unique indexes are named as `puq_<table_name>_<column_names>_<predicate_hash>`.
 	// The name is autogenerated and should not be set by the user.
 	Name() string
 
@@ -133,3 +135,101 @@ func (index *UniqueIndex) ToDropSQL(fmter SQLFormatter) []byte {
 
 	return sql
 }
+
+type PartialUniqueIndex struct {
+	TableName   TableName
+	ColumnNames []ColumnName
+	Where       string
+	name        string
+}
+
+func (index *PartialUniqueIndex) Name() string {
+	if index.name != "" {
+		return index.name
+	}
+
+	var b strings.Builder
+	b.WriteString(IndexTypePartialUnique.String())
+	b.WriteString("_")
+	b.WriteString(string(index.TableName))
+	b.WriteString("_")
+	for i, column := range index.ColumnNames {
+		if i > 0 {
+			b.WriteString("_")
+		}
+		b.WriteString(string(column))
+	}
+	b.WriteString("_")
+	b.WriteString((&whereNormalizer{input: index.Where}).hash())
+	return b.String()
+}
+
+func (index *PartialUniqueIndex) Named(name string) Index {
+	copyOfColumnNames := make([]ColumnName, len(index.ColumnNames))
+	copy(copyOfColumnNames, index.ColumnNames)
+
+	return &PartialUniqueIndex{
+		TableName:   index.TableName,
+		ColumnNames: copyOfColumnNames,
+		Where:       index.Where,
+		name:        name,
+	}
+}
+
+func (index *PartialUniqueIndex) IsNamed() bool {
+	return index.name != ""
+}
+
+func (*PartialUniqueIndex) Type() IndexType {
+	return IndexTypePartialUnique
+}
+
+func (index *PartialUniqueIndex) Columns() []ColumnName {
+	return index.ColumnNames
+}
+
+func (index *PartialUniqueIndex) Equals(other Index) bool {
+	if other.Type() != IndexTypePartialUnique {
+		return false
+	}
+
+	otherPartial, ok := other.(*PartialUniqueIndex)
+	if !ok {
+		return false
+	}
+
+	return index.Name() == other.Name() && slices.Equal(index.Columns(), other.Columns()) && (&whereNormalizer{input: index.Where}).normalize() == (&whereNormalizer{input: otherPartial.Where}).normalize()
+}
+
+func (index *PartialUniqueIndex) ToCreateSQL(fmter SQLFormatter) []byte {
+	sql := []byte{}
+
+	sql = append(sql, "CREATE UNIQUE INDEX IF NOT EXISTS "...)
+	sql = fmter.AppendIdent(sql, index.Name())
+	sql = append(sql, " ON "...)
+	sql = fmter.AppendIdent(sql, string(index.TableName))
+	sql = append(sql, " ("...)
+
+	for i, column := range index.ColumnNames {
+		if i > 0 {
+			sql = append(sql, ", "...)
+		}
+
+		sql = fmter.AppendIdent(sql, string(column))
+	}
+
+	sql = append(sql, ") WHERE "...)
+	sql = append(sql, index.Where...)
+
+	return sql
+}
+
+func (index *PartialUniqueIndex) ToDropSQL(fmter SQLFormatter) []byte {
+	sql := []byte{}
+
+	sql = append(sql, "DROP INDEX IF EXISTS "...)
+	sql = fmter.AppendIdent(sql, index.Name())
+
+	return sql
+}
+

pkg/sqlschema/index_test.go

@@ -38,6 +38,110 @@ func TestIndexToCreateSQL(t *testing.T) {
 			},
 			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "my_index" ON "users" ("id", "name", "email")`,
 		},
+		{
+			name: "PartialUnique_1Column",
+			index: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"deleted_at" IS NULL`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_94610c77" ON "users" ("email") WHERE "deleted_at" IS NULL`,
+		},
+		{
+			name: "PartialUnique_2Columns",
+			index: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"org_id", "email"},
+				Where:       `"deleted_at" IS NULL`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_org_id_email_94610c77" ON "users" ("org_id", "email") WHERE "deleted_at" IS NULL`,
+		},
+		{
+			name: "PartialUnique_Named",
+			index: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"deleted_at" IS NULL`,
+				name:        "my_partial_index",
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "my_partial_index" ON "users" ("email") WHERE "deleted_at" IS NULL`,
+		},
+		{
+			name: "PartialUnique_WhereWithParentheses",
+			index: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `("deleted_at" IS NULL)`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_94610c77" ON "users" ("email") WHERE ("deleted_at" IS NULL)`,
+		},
+		{
+			name: "PartialUnique_WhereWithQuotedIdentifier",
+			index: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"order" IS NULL`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_14c5f5f2" ON "users" ("email") WHERE "order" IS NULL`,
+		},
+		{
+			name: "PartialUnique_WhereWithQuotedLiteral",
+			index: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `status = 'somewhere'`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_9817c709" ON "users" ("email") WHERE status = 'somewhere'`,
+		},
+		{
+			name: "PartialUnique_WhereWith2Columns",
+			index: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email", "status"},
+				Where:       `email = 'test@example.com' AND status = 'active'`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_status_e70e78c3" ON "users" ("email", "status") WHERE email = 'test@example.com' AND status = 'active'`,
+		},
+		// postgres docs example
+		{
+			name: "PartialUnique_WhereWithPostgresDocsExample_1",
+			index: &PartialUniqueIndex{
+				TableName:   "access_log",
+				ColumnNames: []ColumnName{"client_ip"},
+				Where:       `NOT (client_ip > inet '192.168.100.0' AND client_ip < inet '192.168.100.255')`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_access_log_client_ip_5a596410" ON "access_log" ("client_ip") WHERE NOT (client_ip > inet '192.168.100.0' AND client_ip < inet '192.168.100.255')`,
+		},
+		// postgres docs example
+		{
+			name: "PartialUnique_WhereWithPostgresDocsExample_2",
+			index: &PartialUniqueIndex{
+				TableName:   "orders",
+				ColumnNames: []ColumnName{"order_nr"},
+				Where:       `billed is not true`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_orders_order_nr_6d31bb0e" ON "orders" ("order_nr") WHERE billed is not true`,
+		},
+		// sqlite docs example
+		{
+			name: "PartialUnique_WhereWithSqliteDocsExample_1",
+			index: &PartialUniqueIndex{
+				TableName:   "person",
+				ColumnNames: []ColumnName{"team_id"},
+				Where:       `is_team_leader`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_person_team_id_c8604a29" ON "person" ("team_id") WHERE is_team_leader`,
+		},
+		// sqlite docs example
+		{
+			name: "PartialUnique_WhereWithSqliteDocsExample_2",
+			index: &PartialUniqueIndex{
+				TableName:   "purchaseorder",
+				ColumnNames: []ColumnName{"parent_po"},
+				Where:       `parent_po IS NOT NULL`,
+			},
+			sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_purchaseorder_parent_po_dbe2929d" ON "purchaseorder" ("parent_po") WHERE parent_po IS NOT NULL`,
+		},
 	}
 
 	for _, testCase := range testCases {
@@ -49,3 +153,109 @@ func TestIndexToCreateSQL(t *testing.T) {
 		})
 	}
 }
+
+func TestIndexEquals(t *testing.T) {
+	testCases := []struct {
+		name   string
+		a      Index
+		b      Index
+		equals bool
+	}{
+		{
+			name: "PartialUnique_Same",
+			a: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"deleted_at" IS NULL`,
+			},
+			b: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"deleted_at" IS NULL`,
+			},
+			equals: true,
+		},
+		{
+			name: "PartialUnique_NormalizedPostgresWhere",
+			a: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"deleted_at" IS NULL`,
+			},
+			b: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `(deleted_at IS NULL)`,
+			},
+			equals: true,
+		},
+		{
+			name: "PartialUnique_DifferentWhere",
+			a: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"deleted_at" IS NULL`,
+			},
+			b: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"active" = true`,
+			},
+			equals: false,
+		},
+		{
+			name: "PartialUnique_NotEqual_Unique",
+			a: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"deleted_at" IS NULL`,
+			},
+			b: &UniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+			},
+			equals: false,
+		},
+		{
+			name: "Unique_NotEqual_PartialUnique",
+			a: &UniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+			},
+			b: &PartialUniqueIndex{
+				TableName:   "users",
+				ColumnNames: []ColumnName{"email"},
+				Where:       `"deleted_at" IS NULL`,
+			},
+			equals: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			assert.Equal(t, testCase.equals, testCase.a.Equals(testCase.b))
+		})
+	}
+}
+
+func TestPartialUniqueIndexName(t *testing.T) {
+	a := &PartialUniqueIndex{
+		TableName:   "users",
+		ColumnNames: []ColumnName{"email"},
+		Where:       `"deleted_at" IS NULL`,
+	}
+	b := &PartialUniqueIndex{
+		TableName:   "users",
+		ColumnNames: []ColumnName{"email"},
+		Where:       `(deleted_at IS NULL)`,
+	}
+	c := &PartialUniqueIndex{
+		TableName:   "users",
+		ColumnNames: []ColumnName{"email"},
+		Where:       `"active" = true`,
+	}
+
+	assert.Equal(t, "puq_users_email_94610c77", a.Name())
+	assert.Equal(t, a.Name(), b.Name())
+	assert.NotEqual(t, a.Name(), c.Name())
+}

pkg/sqlschema/normalizer.go

@@ -0,0 +1,162 @@
+package sqlschema
+
+import (
+	"fmt"
+	"hash/fnv"
+	"strings"
+)
+
+type whereNormalizer struct {
+	input string
+}
+
+func (n *whereNormalizer) hash() string {
+	hasher := fnv.New32a()
+	_, _ = hasher.Write([]byte(n.normalize()))
+	return fmt.Sprintf("%08x", hasher.Sum32())
+}
+
+func (n *whereNormalizer) normalize() string {
+	where := strings.TrimSpace(n.input)
+	where = n.stripOuterParentheses(where)
+
+	var output strings.Builder
+	output.Grow(len(where))
+
+	for i := 0; i < len(where); i++ {
+		switch where[i] {
+		case ' ', '\t', '\n', '\r':
+			if output.Len() > 0 {
+				last := output.String()[output.Len()-1]
+				if last != ' ' {
+					output.WriteByte(' ')
+				}
+			}
+		case '\'':
+			end := n.consumeSingleQuotedLiteral(where, i, &output)
+			i = end
+		case '"':
+			token, end := n.consumeDoubleQuotedToken(where, i)
+			output.WriteString(token)
+			i = end
+		default:
+			output.WriteByte(where[i])
+		}
+	}
+
+	return strings.TrimSpace(output.String())
+}
+
+func (n *whereNormalizer) stripOuterParentheses(s string) string {
+	for {
+		s = strings.TrimSpace(s)
+		if len(s) < 2 || s[0] != '(' || s[len(s)-1] != ')' || !n.hasWrappingParentheses(s) {
+			return s
+		}
+		s = s[1 : len(s)-1]
+	}
+}
+
+func (n *whereNormalizer) hasWrappingParentheses(s string) bool {
+	depth := 0
+	inSingleQuotedLiteral := false
+	inDoubleQuotedToken := false
+
+	for i := 0; i < len(s); i++ {
+		switch s[i] {
+		case '\'':
+			if inDoubleQuotedToken {
+				continue
+			}
+			if inSingleQuotedLiteral && i+1 < len(s) && s[i+1] == '\'' {
+				i++
+				continue
+			}
+			inSingleQuotedLiteral = !inSingleQuotedLiteral
+		case '"':
+			if inSingleQuotedLiteral {
+				continue
+			}
+			if inDoubleQuotedToken && i+1 < len(s) && s[i+1] == '"' {
+				i++
+				continue
+			}
+			inDoubleQuotedToken = !inDoubleQuotedToken
+		case '(':
+			if inSingleQuotedLiteral || inDoubleQuotedToken {
+				continue
+			}
+			depth++
+		case ')':
+			if inSingleQuotedLiteral || inDoubleQuotedToken {
+				continue
+			}
+			depth--
+			if depth == 0 && i != len(s)-1 {
+				return false
+			}
+		}
+	}
+
+	return depth == 0
+}
+
+func (n *whereNormalizer) consumeSingleQuotedLiteral(s string, start int, output *strings.Builder) int {
+	output.WriteByte(s[start])
+	for i := start + 1; i < len(s); i++ {
+		output.WriteByte(s[i])
+		if s[i] == '\'' {
+			if i+1 < len(s) && s[i+1] == '\'' {
+				i++
+				output.WriteByte(s[i])
+				continue
+			}
+			return i
+		}
+	}
+
+	return len(s) - 1
+}
+
+func (n *whereNormalizer) consumeDoubleQuotedToken(s string, start int) (string, int) {
+	var ident strings.Builder
+
+	for i := start + 1; i < len(s); i++ {
+		if s[i] == '"' {
+			if i+1 < len(s) && s[i+1] == '"' {
+				ident.WriteByte('"')
+				i++
+				continue
+			}
+
+			if n.isSimpleUnquotedIdentifier(ident.String()) {
+				return ident.String(), i
+			}
+
+			return s[start : i+1], i
+		}
+
+		ident.WriteByte(s[i])
+	}
+
+	return s[start:], len(s) - 1
+}
+
+func (n *whereNormalizer) isSimpleUnquotedIdentifier(s string) bool {
+	if s == "" || strings.ToLower(s) != s {
+		return false
+	}
+
+	for i := 0; i < len(s); i++ {
+		ch := s[i]
+		if (ch >= 'a' && ch <= 'z') || ch == '_' {
+			continue
+		}
+		if i > 0 && ch >= '0' && ch <= '9' {
+			continue
+		}
+		return false
+	}
+
+	return true
+}

pkg/sqlschema/normalizer_test.go

@@ -0,0 +1,57 @@
+package sqlschema
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestWhereNormalizerNormalize(t *testing.T) {
+	testCases := []struct {
+		name   string
+		input  string
+		output string
+	}{
+		{
+			name:   "BooleanComparison",
+			input:  `"active" = true`,
+			output: `active = true`,
+		},
+		{
+			name:   "QuotedStringLiteralPreserved",
+			input:  `status = 'somewhere'`,
+			output: `status = 'somewhere'`,
+		},
+		{
+			name:   "EscapedStringLiteralPreserved",
+			input:  `status = 'it''s active'`,
+			output: `status = 'it''s active'`,
+		},
+		{
+			name:   "OuterParenthesesRemoved",
+			input:  `(("deleted_at" IS NULL))`,
+			output: `deleted_at IS NULL`,
+		},
+		{
+			name:   "InnerParenthesesPreserved",
+			input:  `("deleted_at" IS NULL OR ("active" = true AND "status" = 'open'))`,
+			output: `deleted_at IS NULL OR (active = true AND status = 'open')`,
+		},
+		{
+			name:   "MultipleClausesWhitespaceCollapsed",
+			input:  "  (  \"deleted_at\" IS NULL  \n AND\t\"active\" = true  AND status = 'open' )  ",
+			output: `deleted_at IS NULL AND active = true AND status = 'open'`,
+		},
+		{
+			name:   "ComplexBooleanClauses",
+			input:  `NOT ("deleted_at" IS NOT NULL AND ("active" = false OR "status" = 'archived'))`,
+			output: `NOT (deleted_at IS NOT NULL AND (active = false OR status = 'archived'))`,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			assert.Equal(t, testCase.output, (&whereNormalizer{input: testCase.input}).normalize())
+		})
+	}
+}

pkg/sqlschema/operator_test.go

@@ -1146,3 +1146,100 @@ func TestOperatorAlterTable(t *testing.T) {
 		})
 	}
 }
+
+func TestOperatorDiffIndices(t *testing.T) {
+	testCases := []struct {
+		name         string
+		oldIndices   []Index
+		newIndices   []Index
+		expectedSQLs [][]byte
+	}{
+		{
+			name: "UniqueToPartialUnique_DropAndCreate",
+			oldIndices: []Index{
+				&UniqueIndex{
+					TableName:   "users",
+					ColumnNames: []ColumnName{"email"},
+				},
+			},
+			newIndices: []Index{
+				&PartialUniqueIndex{
+					TableName:   "users",
+					ColumnNames: []ColumnName{"email"},
+					Where:       `"deleted_at" IS NULL`,
+				},
+			},
+			expectedSQLs: [][]byte{
+				[]byte(`DROP INDEX IF EXISTS "uq_users_email"`),
+				[]byte(`CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_94610c77" ON "users" ("email") WHERE "deleted_at" IS NULL`),
+			},
+		},
+		{
+			name: "PartialUnique_SameWhere_NoOp",
+			oldIndices: []Index{
+				&PartialUniqueIndex{
+					TableName:   "users",
+					ColumnNames: []ColumnName{"email"},
+					Where:       `"deleted_at" IS NULL`,
+				},
+			},
+			newIndices: []Index{
+				&PartialUniqueIndex{
+					TableName:   "users",
+					ColumnNames: []ColumnName{"email"},
+					Where:       `"deleted_at" IS NULL`,
+				},
+			},
+			expectedSQLs: [][]byte{},
+		},
+		{
+			name: "PartialUnique_NormalizedWhere_NoOp",
+			oldIndices: []Index{
+				&PartialUniqueIndex{
+					TableName:   "users",
+					ColumnNames: []ColumnName{"email"},
+					Where:       `(deleted_at IS NULL)`,
+				},
+			},
+			newIndices: []Index{
+				&PartialUniqueIndex{
+					TableName:   "users",
+					ColumnNames: []ColumnName{"email"},
+					Where:       `"deleted_at" IS NULL`,
+				},
+			},
+			expectedSQLs: [][]byte{},
+		},
+		{
+			name: "PartialUnique_DifferentWhere_DropAndCreate",
+			oldIndices: []Index{
+				&PartialUniqueIndex{
+					TableName:   "users",
+					ColumnNames: []ColumnName{"email"},
+					Where:       `"deleted_at" IS NULL`,
+				},
+			},
+			newIndices: []Index{
+				&PartialUniqueIndex{
+					TableName:   "users",
+					ColumnNames: []ColumnName{"email"},
+					Where:       `"active" = true`,
+				},
+			},
+			expectedSQLs: [][]byte{
+				[]byte(`DROP INDEX IF EXISTS "puq_users_email_94610c77"`),
+				[]byte(`CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_202121f8" ON "users" ("email") WHERE "active" = true`),
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			fmter := NewFormatter(schema.NewNopFormatter().Dialect())
+			operator := NewOperator(fmter, OperatorSupport{})
+
+			actuals := operator.DiffIndices(testCase.oldIndices, testCase.newIndices)
+			assert.Equal(t, testCase.expectedSQLs, actuals)
+		})
+	}
+}

pkg/sqlschema/sqlitesqlschema/provider.go

@@ -3,6 +3,7 @@ package sqlitesqlschema
 import (
 	"context"
 	"strconv"
+	"strings"
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
@@ -114,7 +115,29 @@ func (provider *provider) GetIndices(ctx context.Context, tableName sqlschema.Ta
 			return nil, err
 		}
 
-		if unique {
+		if unique && partial {
+			var indexSQL string
+			if err := provider.
+				sqlstore.
+				BunDB().
+				NewRaw("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", name).
+				Scan(ctx, &indexSQL); err != nil {
+				return nil, err
+			}
+
+			where := extractWhereClause(indexSQL)
+			index := &sqlschema.PartialUniqueIndex{
+				TableName:   tableName,
+				ColumnNames: columns,
+				Where:       where,
+			}
+
+			if index.Name() == name {
+				indices = append(indices, index)
+			} else {
+				indices = append(indices, index.Named(name))
+			}
+		} else if unique {
 			index := &sqlschema.UniqueIndex{
 				TableName:   tableName,
 				ColumnNames: columns,
@@ -148,3 +171,73 @@ func (provider *provider) ToggleFKEnforcement(ctx context.Context, db bun.IDB, o
 
 	return errors.NewInternalf(errors.CodeInternal, "foreign_keys(actual: %s, expected: %s), maybe a transaction is in progress?", strconv.FormatBool(val), strconv.FormatBool(on))
 }
+
+func extractWhereClause(sql string) string {
+	lastWhere := -1
+	inSingleQuotedLiteral := false
+	inDoubleQuotedIdentifier := false
+	inBacktickQuotedIdentifier := false
+	inBracketQuotedIdentifier := false
+
+	for i := 0; i < len(sql); i++ {
+		switch sql[i] {
+		case '\'':
+			if inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
+				continue
+			}
+			if inSingleQuotedLiteral && i+1 < len(sql) && sql[i+1] == '\'' {
+				i++
+				continue
+			}
+			inSingleQuotedLiteral = !inSingleQuotedLiteral
+		case '"':
+			if inSingleQuotedLiteral || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
+				continue
+			}
+			if inDoubleQuotedIdentifier && i+1 < len(sql) && sql[i+1] == '"' {
+				i++
+				continue
+			}
+			inDoubleQuotedIdentifier = !inDoubleQuotedIdentifier
+		case '`':
+			if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBracketQuotedIdentifier {
+				continue
+			}
+			inBacktickQuotedIdentifier = !inBacktickQuotedIdentifier
+		case '[':
+			if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
+				continue
+			}
+			inBracketQuotedIdentifier = true
+		case ']':
+			if inBracketQuotedIdentifier {
+				inBracketQuotedIdentifier = false
+			}
+		}
+
+		if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
+			continue
+		}
+
+		if strings.EqualFold(sql[i:min(i+5, len(sql))], "WHERE") &&
+			(i == 0 || !isSQLiteIdentifierChar(sql[i-1])) &&
+			(i+5 == len(sql) || !isSQLiteIdentifierChar(sql[i+5])) {
+			lastWhere = i
+			i += 4
+		}
+	}
+
+	if lastWhere == -1 {
+		return ""
+	}
+
+	return strings.TrimSpace(sql[lastWhere+len("WHERE"):])
+}
+
+func isSQLiteIdentifierChar(ch byte) bool {
+	return (ch >= 'a' && ch <= 'z') ||
+		(ch >= 'A' && ch <= 'Z') ||
+		(ch >= '0' && ch <= '9') ||
+		ch == '_'
+}
+

pkg/sqlschema/sqlitesqlschema/provider_test.go

@@ -0,0 +1,52 @@
+package sqlitesqlschema
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtractWhereClause(t *testing.T) {
+	testCases := []struct {
+		name  string
+		sql   string
+		where string
+	}{
+		{
+			name:  "UppercaseWhere",
+			sql:   `CREATE UNIQUE INDEX "idx" ON "users" ("email") WHERE "deleted_at" IS NULL`,
+			where: `"deleted_at" IS NULL`,
+		},
+		{
+			name:  "LowercaseWhere",
+			sql:   `CREATE UNIQUE INDEX "idx" ON "users" ("email") where "deleted_at" IS NULL`,
+			where: `"deleted_at" IS NULL`,
+		},
+		{
+			name:  "NewlineBeforeWhere",
+			sql:   "CREATE UNIQUE INDEX \"idx\" ON \"users\" (\"email\")\nWHERE \"deleted_at\" IS NULL",
+			where: `"deleted_at" IS NULL`,
+		},
+		{
+			name:  "ExtraWhitespace",
+			sql:   "CREATE UNIQUE INDEX \"idx\" ON \"users\" (\"email\")   \n \t where   \"deleted_at\" IS NULL  ",
+			where: `"deleted_at" IS NULL`,
+		},
+		{
+			name:  "WhereInStringLiteral",
+			sql:   `CREATE UNIQUE INDEX "idx" ON "users" ("email") WHERE status = 'somewhere'`,
+			where: `status = 'somewhere'`,
+		},
+		{
+			name:  "BooleanLiteral",
+			sql:   `CREATE UNIQUE INDEX "idx" ON "users" ("email") WHERE active = true`,
+			where: `active = true`,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			assert.Equal(t, testCase.where, extractWhereClause(testCase.sql))
+		})
+	}
+}

pkg/templating/markdownrenderer/html.go

@@ -0,0 +1,28 @@
+package markdownrenderer
+
+import (
+	"bytes"
+	"context"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+)
+
+// SoftLineBreakHTML is a HTML tag that is used to represent a soft line break.
+const SoftLineBreakHTML = `<p></p>`
+
+func (r *markdownRenderer) renderHTML(_ context.Context, markdown string) (string, error) {
+	var buf bytes.Buffer
+	if err := r.htmlRenderer.Convert([]byte(markdown), &buf); err != nil {
+		return "", errors.WrapInternalf(err, errors.CodeInternal, "failed to convert markdown to HTML")
+	}
+
+	// return buf.String(), nil
+
+	// TODO: check if there is another way to handle soft line breaks in HTML
+	// the idea with paragraph tags is that it will start the content in new
+	// line without using a line break tag, this works well in variety of cases
+	// but not all, for example, in case of code block, the paragraph tags will be added
+	// to the code block where newline is present.
+	return strings.ReplaceAll(buf.String(), "\n", SoftLineBreakHTML), nil
+}

pkg/templating/markdownrenderer/markdownrenderer.go

@@ -0,0 +1,51 @@
+package markdownrenderer
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+)
+
+type OutputFormat int
+
+const (
+	MarkdownFormatPlainText OutputFormat = iota
+	MarkdownFormatHTML
+	MarkdownFormatSlackMarkdown
+)
+
+// MarkdownRenderer is the interface for rendering markdown to different formats.
+type MarkdownRenderer interface {
+	// Render renders the markdown to the given output format.
+	Render(ctx context.Context, markdown string, outputFormat OutputFormat) (string, error)
+}
+
+type markdownRenderer struct {
+	logger       *slog.Logger
+	htmlRenderer goldmark.Markdown
+}
+
+func NewMarkdownRenderer(logger *slog.Logger) MarkdownRenderer {
+	htmlRenderer := goldmark.New(
+		// basic GitHub Flavored Markdown extensions
+		goldmark.WithExtensions(extension.GFM),
+	)
+	return &markdownRenderer{
+		logger:       logger,
+		htmlRenderer: htmlRenderer,
+	}
+}
+
+func (r *markdownRenderer) Render(ctx context.Context, markdown string, outputFormat OutputFormat) (string, error) {
+	switch outputFormat {
+	case MarkdownFormatPlainText:
+		return r.renderPlainText(ctx, markdown)
+	case MarkdownFormatHTML:
+		return r.renderHTML(ctx, markdown)
+	case MarkdownFormatSlackMarkdown:
+		return r.renderSlackMarkdown(ctx, markdown)
+	}
+	return "", nil
+}

pkg/templating/markdownrenderer/plaintext.go

@@ -0,0 +1,7 @@
+package markdownrenderer
+
+import "context"
+
+func (r *markdownRenderer) renderPlainText(ctx context.Context, markdown string) (string, error) {
+	return "", nil
+}

pkg/templating/markdownrenderer/slack.go

@@ -0,0 +1,7 @@
+package markdownrenderer
+
+import "context"
+
+func (r *markdownRenderer) renderSlackMarkdown(ctx context.Context, markdown string) (string, error) {
+	return "", nil
+}

pkg/types/alertmanagertypes/template.go

@@ -11,19 +11,22 @@ import (
 	alertmanagertemplate "github.com/prometheus/alertmanager/template"
 )
 
+func AdditionalFuncMap() tmpltext.FuncMap {
+	return tmpltext.FuncMap{
+		// urlescape escapes the string for use in a URL query parameter.
+		// It returns tmplhtml.HTML to prevent the template engine from escaping the already escaped string.
+		// url.QueryEscape escapes spaces as "+", and html/template escapes "+" as "+" if tmplhtml.HTML is not used.
+		"urlescape": func(value string) tmplhtml.HTML {
+			return tmplhtml.HTML(url.QueryEscape(value))
+		},
+	}
+}
+
 // customTemplateOption returns an Option that adds custom functions to the template.
 func customTemplateOption() alertmanagertemplate.Option {
 	return func(text *tmpltext.Template, html *tmplhtml.Template) {
-		funcs := tmpltext.FuncMap{
-			// urlescape escapes the string for use in a URL query parameter.
-			// It returns tmplhtml.HTML to prevent the template engine from escaping the already escaped string.
-			// url.QueryEscape escapes spaces as "+", and html/template escapes "+" as "+" if tmplhtml.HTML is not used.
-			"urlescape": func(value string) tmplhtml.HTML {
-				return tmplhtml.HTML(url.QueryEscape(value))
-			},
-		}
-		text.Funcs(funcs)
-		html.Funcs(funcs)
+		text.Funcs(AdditionalFuncMap())
+		html.Funcs(AdditionalFuncMap())
 	}
 }
 

pkg/types/ruletypes/constants.go

@@ -9,4 +9,19 @@ const (
 	LabelSeverityName     = "severity"
 	LabelLastSeen         = "lastSeen"
 	LabelRuleId           = "ruleId"
+	LabelRuleSource       = "ruleSource"
+	LabelNoData           = "nodata"
+	LabelTestAlert        = "testalert"
+	LabelAlertName        = "alertname"
+)
+
+const (
+	AnnotationRelatedLogs   = "related_logs"
+	AnnotationRelatedTraces = "related_traces"
+	AnnotationTitleTemplate = "title_template"
+	AnnotationBodyTemplate  = "body_template"
+	AnnotationValue         = "value"
+	AnnotationThreshold     = "threshold"
+	AnnotationCompareOp     = "compare_op"
+	AnnotationMatchType     = "match_type"
 )