chore: renamed the interface

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/coreos/go-oidc/v3 v3.17.0
 	github.com/dgraph-io/ristretto/v2 v2.3.0
 	github.com/dustin/go-humanize v1.0.1
+	github.com/emersion/go-smtp v0.24.0
 	github.com/gin-gonic/gin v1.11.0
 	github.com/go-co-op/gocron v1.30.1
 	github.com/go-openapi/runtime v0.29.2
@@ -108,6 +109,7 @@ require (
 	github.com/bytedance/gopkg v0.1.3 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
+	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/go-openapi/swag/cmdutils v0.25.4 // indirect
 	github.com/go-openapi/swag/conv v0.25.4 // indirect
@@ -161,7 +163,7 @@ require (
 	github.com/ClickHouse/ch-go v0.67.0 // indirect
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Yiling-J/theine-go v0.6.2 // indirect
-	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
+	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beevik/etree v1.1.0 // indirect

pkg/alertmanager/alertmanagernotify/email/email.go

@@ -0,0 +1,430 @@
+package email
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"log/slog"
+	"math/rand"
+	"mime"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net"
+	"net/mail"
+	"net/smtp"
+	"net/textproto"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "email"
+)
+
+// Email implements a Notifier for email notifications.
+type Email struct {
+	conf     *config.EmailConfig
+	tmpl     *template.Template
+	logger   *slog.Logger
+	hostname string
+}
+
+var errNoAuthUserNameConfigured = errors.NewInternalf(errors.CodeInternal, "no auth username configured")
+
+// New returns a new Email notifier.
+func New(c *config.EmailConfig, t *template.Template, l *slog.Logger) *Email {
+	if _, ok := c.Headers["Subject"]; !ok {
+		c.Headers["Subject"] = config.DefaultEmailSubject
+	}
+	if _, ok := c.Headers["To"]; !ok {
+		c.Headers["To"] = c.To
+	}
+	if _, ok := c.Headers["From"]; !ok {
+		c.Headers["From"] = c.From
+	}
+
+	h, err := os.Hostname()
+	// If we can't get the hostname, we'll use localhost
+	if err != nil {
+		h = "localhost.localdomain"
+	}
+	return &Email{conf: c, tmpl: t, logger: l, hostname: h}
+}
+
+// auth resolves a string of authentication mechanisms.
+func (n *Email) auth(mechs string) (smtp.Auth, error) {
+	username := n.conf.AuthUsername
+
+	// If no username is set, return custom error which can be ignored if needed.
+	if n.conf.AuthUsername == "" {
+		return nil, errNoAuthUserNameConfigured
+	}
+
+	err := &types.MultiError{}
+	for mech := range strings.SplitSeq(mechs, " ") {
+		switch mech {
+		case "CRAM-MD5":
+			secret, secretErr := n.getAuthSecret()
+			if secretErr != nil {
+				err.Add(secretErr)
+				continue
+			}
+			if secret == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing secret for CRAM-MD5 auth mechanism"))
+				continue
+			}
+			return smtp.CRAMMD5Auth(username, secret), nil
+
+		case "PLAIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				err.Add(passwordErr)
+				continue
+			}
+			if password == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing password for PLAIN auth mechanism"))
+				continue
+			}
+			identity := n.conf.AuthIdentity
+
+			return smtp.PlainAuth(identity, username, password, n.conf.Smarthost.Host), nil
+		case "LOGIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				err.Add(passwordErr)
+				continue
+			}
+			if password == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing password for LOGIN auth mechanism"))
+				continue
+			}
+			return LoginAuth(username, password), nil
+		}
+	}
+	if err.Len() == 0 {
+		err.Add(errors.NewInternalf(errors.CodeInternal, "unknown auth mechanism: %s", mechs))
+	}
+	return nil, err
+}
+
+// Notify implements the Notifier interface.
+func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var (
+		c       *smtp.Client
+		conn    net.Conn
+		err     error
+		success = false
+	)
+	// Determine whether to use Implicit TLS
+	var useImplicitTLS bool
+	if n.conf.ForceImplicitTLS != nil {
+		useImplicitTLS = *n.conf.ForceImplicitTLS
+	} else {
+		// Default logic: port 465 uses implicit TLS (backward compatibility)
+		useImplicitTLS = n.conf.Smarthost.Port == "465"
+	}
+
+	if useImplicitTLS {
+		tlsConfig, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "parse TLS configuration")
+		}
+		if tlsConfig.ServerName == "" {
+			tlsConfig.ServerName = n.conf.Smarthost.Host
+		}
+
+		conn, err = tls.Dial("tcp", n.conf.Smarthost.String(), tlsConfig)
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "establish TLS connection to server")
+		}
+	} else {
+		var (
+			d   = net.Dialer{}
+			err error
+		)
+		conn, err = d.DialContext(ctx, "tcp", n.conf.Smarthost.String())
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "establish connection to server")
+		}
+	}
+	c, err = smtp.NewClient(conn, n.conf.Smarthost.Host)
+	if err != nil {
+		conn.Close()
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "create SMTP client")
+	}
+	defer func() {
+		// Try to clean up after ourselves but don't log anything if something has failed.
+		if err := c.Quit(); success && err != nil {
+			n.logger.WarnContext(ctx, "failed to close SMTP connection", "err", err)
+		}
+	}()
+
+	if n.conf.Hello != "" {
+		err = c.Hello(n.conf.Hello)
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send EHLO command")
+		}
+	}
+
+	// Global Config guarantees RequireTLS is not nil.
+	if *n.conf.RequireTLS && !useImplicitTLS {
+		if ok, _ := c.Extension("STARTTLS"); !ok {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "'require_tls' is true (default) but %q does not advertise the STARTTLS extension", n.conf.Smarthost)
+		}
+
+		tlsConf, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "parse TLS configuration")
+		}
+		if tlsConf.ServerName == "" {
+			tlsConf.ServerName = n.conf.Smarthost.Host
+		}
+
+		if err := c.StartTLS(tlsConf); err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send STARTTLS command")
+		}
+	}
+
+	if ok, mech := c.Extension("AUTH"); ok {
+		auth, err := n.auth(mech)
+		if err != nil && err != errNoAuthUserNameConfigured {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "find auth mechanism")
+		} else if err == errNoAuthUserNameConfigured {
+			n.logger.DebugContext(ctx, "no auth username configured. Attempting to send email without authenticating")
+		}
+		if auth != nil {
+			if err := c.Auth(auth); err != nil {
+				return true, errors.WrapInternalf(err, errors.CodeInternal, "%T auth", auth)
+			}
+		}
+	}
+
+	var (
+		tmplErr error
+		data    = notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
+		tmpl    = notify.TmplText(n.tmpl, data, &tmplErr)
+	)
+	from := tmpl(n.conf.From)
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "execute 'from' template")
+	}
+	to := tmpl(n.conf.To)
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "execute 'to' template")
+	}
+
+	addrs, err := mail.ParseAddressList(from)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "parse 'from' addresses")
+	}
+	if len(addrs) != 1 {
+		return false, errors.NewInternalf(errors.CodeInternal, "must be exactly one 'from' address (got: %d)", len(addrs))
+	}
+	if err = c.Mail(addrs[0].Address); err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "send MAIL command")
+	}
+	addrs, err = mail.ParseAddressList(to)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "parse 'to' addresses")
+	}
+	for _, addr := range addrs {
+		if err = c.Rcpt(addr.Address); err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send RCPT command")
+		}
+	}
+
+	// Send the email headers and body.
+	message, err := c.Data()
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "send DATA command")
+	}
+	closeOnce := sync.OnceValue(func() error {
+		return message.Close()
+	})
+	// Close the message when this method exits in order to not leak resources. Even though we're calling this explicitly
+	// further down, the method may exit before then.
+	defer func() {
+		// If we try close an already-closed writer, it'll send a subsequent request to the server which is invalid.
+		_ = closeOnce()
+	}()
+
+	buffer := &bytes.Buffer{}
+	for header, t := range n.conf.Headers {
+		value, err := n.tmpl.ExecuteTextString(t, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute %q header template", header)
+		}
+		fmt.Fprintf(buffer, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", value))
+	}
+
+	if _, ok := n.conf.Headers["Message-Id"]; !ok {
+		fmt.Fprintf(buffer, "Message-Id: %s\r\n", fmt.Sprintf("<%d.%d@%s>", time.Now().UnixNano(), rand.Uint64(), n.hostname))
+	}
+
+	if n.conf.Threading.Enabled {
+		key, err := notify.ExtractGroupKey(ctx)
+		if err != nil {
+			return false, err
+		}
+		// Add threading headers. All notifications for the same alert group
+		// (identified by key hash) are threaded together.
+		threadBy := ""
+		if n.conf.Threading.ThreadByDate != "none" {
+			// ThreadByDate is 'daily':
+			// Use current date so all mails for this alert today thread together.
+			threadBy = time.Now().Format("2006-01-02")
+		}
+		keyHash := key.Hash()
+		if len(keyHash) > 16 {
+			keyHash = keyHash[:16]
+		}
+		// The thread root ID is a Message-ID that doesn't correspond to
+		// any actual email. Email clients following the (commonly used) JWZ
+		// algorithm will create a dummy container to group these messages.
+		threadRootID := fmt.Sprintf("<alert-%s-%s@alertmanager>", keyHash, threadBy)
+		fmt.Fprintf(buffer, "References: %s\r\n", threadRootID)
+		fmt.Fprintf(buffer, "In-Reply-To: %s\r\n", threadRootID)
+	}
+
+	multipartBuffer := &bytes.Buffer{}
+	multipartWriter := multipart.NewWriter(multipartBuffer)
+
+	fmt.Fprintf(buffer, "Date: %s\r\n", time.Now().Format(time.RFC1123Z))
+	fmt.Fprintf(buffer, "Content-Type: multipart/alternative;  boundary=%s\r\n", multipartWriter.Boundary())
+	fmt.Fprintf(buffer, "MIME-Version: 1.0\r\n\r\n")
+
+	// TODO: Add some useful headers here, such as URL of the alertmanager
+	// and active/resolved.
+	_, err = message.Write(buffer.Bytes())
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "write headers")
+	}
+
+	if len(n.conf.Text) > 0 {
+		// Text template
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/plain; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for text template")
+		}
+		body, err := n.tmpl.ExecuteTextString(n.conf.Text, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute text template")
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "write text part")
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "close text part")
+		}
+	}
+
+	if len(n.conf.HTML) > 0 {
+		// Html template
+		// Preferred alternative placed last per section 5.1.4 of RFC 2046
+		// https://www.ietf.org/rfc/rfc2046.txt
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/html; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for html template")
+		}
+		body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute html template")
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "write HTML part")
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "close HTML part")
+		}
+	}
+
+	err = multipartWriter.Close()
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "close multipartWriter")
+	}
+
+	_, err = message.Write(multipartBuffer.Bytes())
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "write body buffer")
+	}
+
+	// Complete the message and await response.
+	if err = closeOnce(); err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "delivery failure")
+	}
+
+	success = true
+	return false, nil
+}
+
+type loginAuth struct {
+	username, password string
+}
+
+func LoginAuth(username, password string) smtp.Auth {
+	return &loginAuth{username, password}
+}
+
+func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
+	return "LOGIN", []byte{}, nil
+}
+
+// Used for AUTH LOGIN. (Maybe password should be encrypted).
+func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
+	if more {
+		switch strings.ToLower(string(fromServer)) {
+		case "username:":
+			return []byte(a.username), nil
+		case "password:":
+			return []byte(a.password), nil
+		default:
+			return nil, errors.NewInternalf(errors.CodeInternal, "unexpected server challenge")
+		}
+	}
+	return nil, nil
+}
+
+func (n *Email) getPassword() (string, error) {
+	if len(n.conf.AuthPasswordFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthPasswordFile)
+		if err != nil {
+			return "", errors.NewInternalf(errors.CodeInternal, "could not read %s: %v", n.conf.AuthPasswordFile, err)
+		}
+		return strings.TrimSpace(string(content)), nil
+	}
+	return string(n.conf.AuthPassword), nil
+}
+
+func (n *Email) getAuthSecret() (string, error) {
+	if len(n.conf.AuthSecretFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthSecretFile)
+		if err != nil {
+			return "", errors.NewInternalf(errors.CodeInternal, "could not read %s: %v", n.conf.AuthSecretFile, err)
+		}
+		return string(content), nil
+	}
+	return string(n.conf.AuthSecret), nil
+}

pkg/alertmanager/alertmanagernotify/email/testdata/auth-local.yml

@@ -0,0 +1,4 @@
+smarthost: 127.0.0.1:1026
+server: http://127.0.0.1:1081/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/auth.yml

@@ -0,0 +1,4 @@
+smarthost: maildev-auth:1025
+server: http://maildev-auth:1080/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/noauth-local.yml

@@ -0,0 +1,2 @@
+smarthost: 127.0.0.1:1025
+server: http://127.0.0.1:1080/

pkg/alertmanager/alertmanagernotify/email/testdata/noauth.yml

@@ -0,0 +1,2 @@
+smarthost: maildev-noauth:1025
+server: http://maildev-noauth:1080/

pkg/alertmanager/alertmanagernotify/msteamsv2/msteamsv2.go

@@ -27,6 +27,10 @@ const (
 	colorGrey  = "Warning"
 )
 
+const (
+	Integration = "msteamsv2"
+)
+
 type Notifier struct {
 	conf         *config.MSTeamsV2Config
 	titleLink    string
@@ -87,7 +91,7 @@ type teamsMessage struct {
 
 // New returns a new notifier that uses the Microsoft Teams Power Platform connector.
 func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
-	client, err := commoncfg.NewClientFromConfig(*c.HTTPConfig, "msteamsv2", httpOpts...)
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
 	}

pkg/alertmanager/alertmanagernotify/opsgenie/api_key_file

@@ -0,0 +1,2 @@
+my_secret_api_key
+

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie.go

@@ -0,0 +1,290 @@
+package opsgenie
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"maps"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "opsgenie"
+)
+
+// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
+const maxMessageLenRunes = 130
+
+// Notifier implements a Notifier for OpsGenie notifications.
+type Notifier struct {
+	conf    *config.OpsGenieConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new OpsGenie notifier.
+func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:    c,
+		tmpl:    t,
+		logger:  l,
+		client:  client,
+		retrier: &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
+	}, nil
+}
+
+type opsGenieCreateMessage struct {
+	Alias       string                           `json:"alias"`
+	Message     string                           `json:"message"`
+	Description string                           `json:"description,omitempty"`
+	Details     map[string]string                `json:"details"`
+	Source      string                           `json:"source"`
+	Responders  []opsGenieCreateMessageResponder `json:"responders,omitempty"`
+	Tags        []string                         `json:"tags,omitempty"`
+	Note        string                           `json:"note,omitempty"`
+	Priority    string                           `json:"priority,omitempty"`
+	Entity      string                           `json:"entity,omitempty"`
+	Actions     []string                         `json:"actions,omitempty"`
+}
+
+type opsGenieCreateMessageResponder struct {
+	ID       string `json:"id,omitempty"`
+	Name     string `json:"name,omitempty"`
+	Username string `json:"username,omitempty"`
+	Type     string `json:"type"` // team, user, escalation, schedule etc.
+}
+
+type opsGenieCloseMessage struct {
+	Source string `json:"source"`
+}
+
+type opsGenieUpdateMessageMessage struct {
+	Message string `json:"message,omitempty"`
+}
+
+type opsGenieUpdateDescriptionMessage struct {
+	Description string `json:"description,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	requests, retry, err := n.createRequests(ctx, as...)
+	if err != nil {
+		return retry, err
+	}
+
+	for _, req := range requests {
+		req.Header.Set("User-Agent", notify.UserAgentHeader)
+		resp, err := n.client.Do(req) //nolint:bodyclose
+		if err != nil {
+			return true, err
+		}
+		shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+		notify.Drain(resp)
+		if err != nil {
+			return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+		}
+	}
+	return true, nil
+}
+
+// Like Split but filter out empty strings.
+func safeSplit(s, sep string) []string {
+	a := strings.Split(strings.TrimSpace(s), sep)
+	b := a[:0]
+	for _, x := range a {
+		if x != "" {
+			b = append(b, x)
+		}
+	}
+	return b
+}
+
+// Create requests for a list of alerts.
+func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+	logger := n.logger.With("group_key", key)
+	logger.DebugContext(ctx, "extracted group key")
+
+	data := notify.GetTemplateData(ctx, n.tmpl, as, logger)
+
+	tmpl := notify.TmplText(n.tmpl, data, &err)
+
+	details := make(map[string]string)
+
+	maps.Copy(details, data.CommonLabels)
+
+	for k, v := range n.conf.Details {
+		details[k] = tmpl(v)
+	}
+
+	requests := []*http.Request{}
+
+	var (
+		alias  = key.Hash()
+		alerts = types.Alerts(as...)
+	)
+	switch alerts.Status() {
+	case model.AlertResolved:
+		resolvedEndpointURL := n.conf.APIURL.Copy()
+		resolvedEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
+		q := resolvedEndpointURL.Query()
+		q.Set("identifierType", "alias")
+		resolvedEndpointURL.RawQuery = q.Encode()
+		msg := &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", resolvedEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+	default:
+		message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
+		if truncated {
+			logger.WarnContext(ctx, "Truncated message", "alert", key, "max_runes", maxMessageLenRunes)
+		}
+
+		createEndpointURL := n.conf.APIURL.Copy()
+		createEndpointURL.Path += "v2/alerts"
+
+		var responders []opsGenieCreateMessageResponder
+		for _, r := range n.conf.Responders {
+			responder := opsGenieCreateMessageResponder{
+				ID:       tmpl(r.ID),
+				Name:     tmpl(r.Name),
+				Username: tmpl(r.Username),
+				Type:     tmpl(r.Type),
+			}
+
+			if responder == (opsGenieCreateMessageResponder{}) {
+				// Filter out empty responders. This is useful if you want to fill
+				// responders dynamically from alert's common labels.
+				continue
+			}
+
+			if responder.Type == "teams" {
+				teams := safeSplit(responder.Name, ",")
+				for _, team := range teams {
+					newResponder := opsGenieCreateMessageResponder{
+						Name: tmpl(team),
+						Type: tmpl("team"),
+					}
+					responders = append(responders, newResponder)
+				}
+				continue
+			}
+
+			responders = append(responders, responder)
+		}
+
+		msg := &opsGenieCreateMessage{
+			Alias:       alias,
+			Message:     message,
+			Description: tmpl(n.conf.Description),
+			Details:     details,
+			Source:      tmpl(n.conf.Source),
+			Responders:  responders,
+			Tags:        safeSplit(tmpl(n.conf.Tags), ","),
+			Note:        tmpl(n.conf.Note),
+			Priority:    tmpl(n.conf.Priority),
+			Entity:      tmpl(n.conf.Entity),
+			Actions:     safeSplit(tmpl(n.conf.Actions), ","),
+		}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", createEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+
+		if n.conf.UpdateAlerts {
+			updateMessageEndpointURL := n.conf.APIURL.Copy()
+			updateMessageEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/message", alias)
+			q := updateMessageEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateMessageEndpointURL.RawQuery = q.Encode()
+			updateMsgMsg := &opsGenieUpdateMessageMessage{
+				Message: msg.Message,
+			}
+			var updateMessageBuf bytes.Buffer
+			if err := json.NewEncoder(&updateMessageBuf).Encode(updateMsgMsg); err != nil {
+				return nil, false, err
+			}
+			req, err := http.NewRequest("PUT", updateMessageEndpointURL.String(), &updateMessageBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req)
+
+			updateDescriptionEndpointURL := n.conf.APIURL.Copy()
+			updateDescriptionEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/description", alias)
+			q = updateDescriptionEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateDescriptionEndpointURL.RawQuery = q.Encode()
+			updateDescMsg := &opsGenieUpdateDescriptionMessage{
+				Description: msg.Description,
+			}
+
+			var updateDescriptionBuf bytes.Buffer
+			if err := json.NewEncoder(&updateDescriptionBuf).Encode(updateDescMsg); err != nil {
+				return nil, false, err
+			}
+			req, err = http.NewRequest("PUT", updateDescriptionEndpointURL.String(), &updateDescriptionBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req.WithContext(ctx))
+		}
+	}
+
+	var apiKey string
+	if n.conf.APIKey != "" {
+		apiKey = tmpl(string(n.conf.APIKey))
+	} else {
+		content, err := os.ReadFile(n.conf.APIKeyFile)
+		if err != nil {
+			return nil, false, errors.WrapInternalf(err, errors.CodeInternal, "read key_file error")
+		}
+		apiKey = tmpl(string(content))
+		apiKey = strings.TrimSpace(string(apiKey))
+	}
+
+	if err != nil {
+		return nil, false, errors.WrapInternalf(err, errors.CodeInternal, "templating error")
+	}
+
+	for _, req := range requests {
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
+	}
+
+	return requests, true, nil
+}

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie_test.go

@@ -0,0 +1,333 @@
+package opsgenie
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestOpsGenieRetry(t *testing.T) {
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestOpsGenieRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKey:     config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestGettingOpsGegineApikeyFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+
+	f, err := os.CreateTemp(t.TempDir(), "opsgenie_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKeyFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestOpsGenie(t *testing.T) {
+	u, err := url.Parse("https://opsgenie/api")
+	if err != nil {
+		t.Fatalf("failed to parse URL: %v", err)
+	}
+	logger := promslog.NewNopLogger()
+	tmpl := test.CreateTmpl(t)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.OpsGenieConfig
+
+		expectedEmptyAlertBody string
+		expectedBody           string
+	}{
+		{
+			title: "config without details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with multiple teams",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName3 }}`,
+						Type: `{{ .CommonLabels.ResponderType3 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"TeamB","type":"team"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1"}
+`,
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			notifier, err := New(tc.cfg, tmpl, logger)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			expectedURL, _ := url.Parse("https://opsgenie/apiv2/alerts")
+
+			// Empty alert.
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+
+			req, retry, err := notifier.createRequests(ctx, alert1)
+			require.NoError(t, err)
+			require.Len(t, req, 1)
+			require.True(t, retry)
+			require.Equal(t, expectedURL, req[0].URL)
+			require.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization"))
+			require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0]))
+
+			// Fully defined alert.
+			alert2 := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"Message":        "message",
+						"Description":    "description",
+						"Source":         "http://prometheus",
+						"ResponderName1": "TeamA",
+						"ResponderType1": "team",
+						"ResponderName2": "EscalationA",
+						"ResponderType2": "escalation",
+						"ResponderName3": "TeamA,TeamB",
+						"ResponderType3": "teams",
+						"Tags":           "tag1,tag2",
+						"Note":           "this is a note",
+						"Priority":       "P1",
+						"Entity":         "test-domain",
+						"Actions":        "doThis,doThat",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			req, retry, err = notifier.createRequests(ctx, alert2)
+			require.NoError(t, err)
+			require.True(t, retry)
+			require.Len(t, req, 1)
+			require.Equal(t, tc.expectedBody, readBody(t, req[0]))
+
+			// Broken API Key Template.
+			tc.cfg.APIKey = "{{ kaput "
+			_, _, err = notifier.createRequests(ctx, alert2)
+			require.Error(t, err)
+			require.Equal(t, "template: :1: function \"kaput\" not defined", err.Error())
+		})
+	}
+}
+
+func TestOpsGenieWithUpdate(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		Message:      `{{ .CommonLabels.Message }}`,
+		Description:  `{{ .CommonLabels.Description }}`,
+		UpdateAlerts: true,
+		APIKey:       "test-api-key",
+		APIURL:       &config.URL{URL: u},
+		HTTPConfig:   &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+	alert := &types.Alert{
+		Alert: model.Alert{
+			StartsAt: time.Now(),
+			EndsAt:   time.Now().Add(time.Hour),
+			Labels: model.LabelSet{
+				"Message":     "new message",
+				"Description": "new description",
+			},
+		},
+	}
+	require.NoError(t, err)
+	requests, retry, err := notifierWithUpdate.createRequests(ctx, alert)
+	require.NoError(t, err)
+	require.True(t, retry)
+	require.Len(t, requests, 3)
+
+	body0 := readBody(t, requests[0])
+	body1 := readBody(t, requests[1])
+	body2 := readBody(t, requests[2])
+	key, _ := notify.ExtractGroupKey(ctx)
+	alias := key.Hash()
+
+	require.Equal(t, "https://test-opsgenie-url/v2/alerts", requests[0].URL.String())
+	require.NotEmpty(t, body0)
+
+	require.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias))
+	require.JSONEq(t, `{"message":"new message"}`, body1)
+	require.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias))
+	require.JSONEq(t, `{"description":"new description"}`, body2)
+}
+
+func TestOpsGenieApiKeyFile(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		APIKeyFile: `./api_key_file`,
+		APIURL:     &config.URL{URL: u},
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+
+	require.NoError(t, err)
+	requests, _, err := notifierWithUpdate.createRequests(ctx)
+	require.NoError(t, err)
+	require.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
+}
+
+func readBody(t *testing.T, r *http.Request) string {
+	t.Helper()
+	body, err := io.ReadAll(r.Body)
+	require.NoError(t, err)
+	return string(body)
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty.go

@@ -0,0 +1,374 @@
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/alecthomas/units"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "pagerduty"
+)
+
+const (
+	maxEventSize int = 512000
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTc4-send-a-v1-event - 1024 characters or runes.
+	maxV1DescriptionLenRunes = 1024
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1024 characters or runes.
+	maxV2SummaryLenRunes = 1024
+)
+
+// Notifier implements a Notifier for PagerDuty notifications.
+type Notifier struct {
+	conf    *config.PagerdutyConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	apiV1   string // for tests.
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new PagerDuty notifier.
+func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	n := &Notifier{conf: c, tmpl: t, logger: l, client: client}
+	if c.ServiceKey != "" || c.ServiceKeyFile != "" {
+		n.apiV1 = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+		// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v1-overview#api-response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusForbidden}, CustomDetailsFunc: errDetails}
+	} else {
+		// Retrying can solve the issue on 429 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v2-overview#response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}, CustomDetailsFunc: errDetails}
+	}
+	return n, nil
+}
+
+const (
+	pagerDutyEventTrigger = "trigger"
+	pagerDutyEventResolve = "resolve"
+)
+
+type pagerDutyMessage struct {
+	RoutingKey  string            `json:"routing_key,omitempty"`
+	ServiceKey  string            `json:"service_key,omitempty"`
+	DedupKey    string            `json:"dedup_key,omitempty"`
+	IncidentKey string            `json:"incident_key,omitempty"`
+	EventType   string            `json:"event_type,omitempty"`
+	Description string            `json:"description,omitempty"`
+	EventAction string            `json:"event_action"`
+	Payload     *pagerDutyPayload `json:"payload"`
+	Client      string            `json:"client,omitempty"`
+	ClientURL   string            `json:"client_url,omitempty"`
+	Details     map[string]any    `json:"details,omitempty"`
+	Images      []pagerDutyImage  `json:"images,omitempty"`
+	Links       []pagerDutyLink   `json:"links,omitempty"`
+}
+
+type pagerDutyLink struct {
+	HRef string `json:"href"`
+	Text string `json:"text"`
+}
+
+type pagerDutyImage struct {
+	Src  string `json:"src"`
+	Alt  string `json:"alt"`
+	Href string `json:"href"`
+}
+
+type pagerDutyPayload struct {
+	Summary       string         `json:"summary"`
+	Source        string         `json:"source"`
+	Severity      string         `json:"severity"`
+	Timestamp     string         `json:"timestamp,omitempty"`
+	Class         string         `json:"class,omitempty"`
+	Component     string         `json:"component,omitempty"`
+	Group         string         `json:"group,omitempty"`
+	CustomDetails map[string]any `json:"custom_details,omitempty"`
+}
+
+func (n *Notifier) encodeMessage(ctx context.Context, msg *pagerDutyMessage) (bytes.Buffer, error) {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return buf, errors.WrapInternalf(err, errors.CodeInternal, "failed to encode PagerDuty message")
+	}
+
+	if buf.Len() > maxEventSize {
+		truncatedMsg := fmt.Sprintf("Custom details have been removed because the original event exceeds the maximum size of %s", units.MetricBytes(maxEventSize).String())
+
+		if n.apiV1 != "" {
+			msg.Details = map[string]any{"error": truncatedMsg}
+		} else {
+			msg.Payload.CustomDetails = map[string]any{"error": truncatedMsg}
+		}
+
+		n.logger.WarnContext(ctx, "Truncated Details because message of size exceeds limit", "message_size", units.MetricBytes(buf.Len()).String(), "max_size", units.MetricBytes(maxEventSize).String())
+
+		buf.Reset()
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return buf, errors.WrapInternalf(err, errors.CodeInternal, "failed to encode PagerDuty message")
+		}
+	}
+
+	return buf, nil
+}
+
+func (n *Notifier) notifyV1(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated description", "key", key, "max_runes", maxV1DescriptionLenRunes)
+	}
+
+	serviceKey := string(n.conf.ServiceKey)
+	if serviceKey == "" {
+		content, fileErr := os.ReadFile(n.conf.ServiceKeyFile)
+		if fileErr != nil {
+			return false, errors.WrapInternalf(fileErr, errors.CodeInternal, "failed to read service key from file")
+		}
+		serviceKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		ServiceKey:  tmpl(serviceKey),
+		EventType:   eventType,
+		IncidentKey: key.Hash(),
+		Description: description,
+		Details:     details,
+	}
+
+	if eventType == pagerDutyEventTrigger {
+		msg.Client = tmpl(n.conf.Client)
+		msg.ClientURL = tmpl(n.conf.ClientURL)
+	}
+
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "failed to template PagerDuty v1 message")
+	}
+
+	// Ensure that the service key isn't empty after templating.
+	if msg.ServiceKey == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "service key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(ctx, msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &encodedMsg) //nolint:bodyclose
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "failed to post message to PagerDuty v1")
+	}
+	defer notify.Drain(resp)
+
+	return n.retrier.Check(resp.StatusCode, resp.Body)
+}
+
+func (n *Notifier) notifyV2(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.Severity == "" {
+		n.conf.Severity = "error"
+	}
+
+	summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV2SummaryLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated summary", "key", key, "max_runes", maxV2SummaryLenRunes)
+	}
+
+	routingKey := string(n.conf.RoutingKey)
+	if routingKey == "" {
+		content, fileErr := os.ReadFile(n.conf.RoutingKeyFile)
+		if fileErr != nil {
+			return false, errors.WrapInternalf(fileErr, errors.CodeInternal, "failed to read routing key from file")
+		}
+		routingKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		Client:      tmpl(n.conf.Client),
+		ClientURL:   tmpl(n.conf.ClientURL),
+		RoutingKey:  tmpl(routingKey),
+		EventAction: eventType,
+		DedupKey:    key.Hash(),
+		Images:      make([]pagerDutyImage, 0, len(n.conf.Images)),
+		Links:       make([]pagerDutyLink, 0, len(n.conf.Links)),
+		Payload: &pagerDutyPayload{
+			Summary:       summary,
+			Source:        tmpl(n.conf.Source),
+			Severity:      tmpl(n.conf.Severity),
+			CustomDetails: details,
+			Class:         tmpl(n.conf.Class),
+			Component:     tmpl(n.conf.Component),
+			Group:         tmpl(n.conf.Group),
+		},
+	}
+
+	for _, item := range n.conf.Images {
+		image := pagerDutyImage{
+			Src:  tmpl(item.Src),
+			Alt:  tmpl(item.Alt),
+			Href: tmpl(item.Href),
+		}
+
+		if image.Src != "" {
+			msg.Images = append(msg.Images, image)
+		}
+	}
+
+	for _, item := range n.conf.Links {
+		link := pagerDutyLink{
+			HRef: tmpl(item.Href),
+			Text: tmpl(item.Text),
+		}
+
+		if link.HRef != "" {
+			msg.Links = append(msg.Links, link)
+		}
+	}
+
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "failed to template PagerDuty v2 message")
+	}
+
+	// Ensure that the routing key isn't empty after templating.
+	if msg.RoutingKey == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "routing key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(ctx, msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &encodedMsg) //nolint:bodyclose
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "failed to post message to PagerDuty")
+	}
+	defer notify.Drain(resp)
+
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return retry, err
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With("group_key", key)
+
+	var (
+		alerts    = types.Alerts(as...)
+		data      = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		eventType = pagerDutyEventTrigger
+	)
+
+	if alerts.Status() == model.AlertResolved {
+		eventType = pagerDutyEventResolve
+	}
+
+	logger.DebugContext(ctx, "extracted group key", "event_type", eventType)
+
+	details, err := n.renderDetails(data)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "failed to render details: %v", err)
+	}
+
+	if n.conf.Timeout > 0 {
+		nfCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeInternal, "configured pagerduty timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = nfCtx
+	}
+
+	nf := n.notifyV2
+	if n.apiV1 != "" {
+		nf = n.notifyV1
+	}
+	retry, err := nf(ctx, eventType, key, data, details)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.WrapInternalf(err, errors.CodeInternal, "failed to notify PagerDuty: %v", context.Cause(ctx))
+		}
+		return retry, err
+	}
+	return retry, nil
+}
+
+func errDetails(status int, body io.Reader) string {
+	// See https://v2.developer.pagerduty.com/docs/trigger-events for the v1 events API.
+	// See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 for the v2 events API.
+	if status != http.StatusBadRequest || body == nil {
+		return ""
+	}
+	var pgr struct {
+		Status  string   `json:"status"`
+		Message string   `json:"message"`
+		Errors  []string `json:"errors"`
+	}
+	if err := json.NewDecoder(body).Decode(&pgr); err != nil {
+		return ""
+	}
+	return fmt.Sprintf("%s: %s", pgr.Message, strings.Join(pgr.Errors, ","))
+}
+
+func (n *Notifier) renderDetails(
+	data *template.Data,
+) (map[string]any, error) {
+	var (
+		tmplTextErr  error
+		tmplText     = notify.TmplText(n.tmpl, data, &tmplTextErr)
+		tmplTextFunc = func(tmpl string) (string, error) {
+			return tmplText(tmpl), tmplTextErr
+		}
+	)
+	var err error
+	rendered := make(map[string]any, len(n.conf.Details))
+	for k, v := range n.conf.Details {
+		rendered[k], err = template.DeepCopyWithTemplate(v, tmplTextFunc)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return rendered, nil
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty_test.go

@@ -0,0 +1,879 @@
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestPagerDutyRetryV1(t *testing.T) {
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusForbidden)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "retryv1 - error on status %d", statusCode)
+	}
+}
+
+func TestPagerDutyRetryV2(t *testing.T) {
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "retryv2 - error on status %d", statusCode)
+	}
+}
+
+func TestPagerDutyRedactedURLV1(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "01234567890123456789012345678901"
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+	notifier.apiV1 = u.String()
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyRedactedURLV2(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "01234567890123456789012345678901"
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			URL:        &config.URL{URL: u},
+			RoutingKey: config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyV1ServiceKeyFromFile(t *testing.T) {
+	key := "01234567890123456789012345678901"
+	f, err := os.CreateTemp(t.TempDir(), "pagerduty_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKeyFile: f.Name(),
+			HTTPConfig:     &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+	notifier.apiV1 = u.String()
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyV2RoutingKeyFromFile(t *testing.T) {
+	key := "01234567890123456789012345678901"
+	f, err := os.CreateTemp(t.TempDir(), "pagerduty_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			URL:            &config.URL{URL: u},
+			RoutingKeyFile: f.Name(),
+			HTTPConfig:     &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyTemplating(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		dec := json.NewDecoder(r.Body)
+		out := make(map[string]any)
+		err := dec.Decode(&out)
+		if err != nil {
+			panic(err)
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.PagerdutyConfig
+
+		retry  bool
+		errMsg string
+	}{
+		{
+			title: "full-blown legacy message",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Images: []config.PagerdutyImage{
+					{
+						Src:  "{{ .Status }}",
+						Alt:  "{{ .Status }}",
+						Href: "{{ .Status }}",
+					},
+				},
+				Links: []config.PagerdutyLink{
+					{
+						Href: "{{ .Status }}",
+						Text: "{{ .Status }}",
+					},
+				},
+				Details: map[string]any{
+					"firing":       `{{ .Alerts.Firing | toJson }}`,
+					"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+		},
+		{
+			title: "full-blown legacy message",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Images: []config.PagerdutyImage{
+					{
+						Src:  "{{ .Status }}",
+						Alt:  "{{ .Status }}",
+						Href: "{{ .Status }}",
+					},
+				},
+				Links: []config.PagerdutyLink{
+					{
+						Href: "{{ .Status }}",
+						Text: "{{ .Status }}",
+					},
+				},
+				Details: map[string]any{
+					"firing":       `{{ template "pagerduty.default.instances" .Alerts.Firing }}`,
+					"resolved":     `{{ template "pagerduty.default.instances" .Alerts.Resolved }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+		},
+		{
+			title: "nested details",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": map[string]any{
+								"firing":       `{{ .Alerts.Firing | toJson }}`,
+								"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+								"num_firing":   `{{ .Alerts.Firing | len }}`,
+								"num_resolved": `{{ .Alerts.Resolved | len }}`,
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			title: "nested details with template error",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": map[string]any{
+								"firing": `{{ template "pagerduty.default.instances" .Alerts.Firing`,
+							},
+						},
+					},
+				},
+			},
+			errMsg: "failed to render details: template: :1: unclosed action",
+		},
+		{
+			title: "details with templating errors",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"firing":       `{{ .Alerts.Firing | toJson`,
+					"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+			errMsg: "failed to render details: template: :1: unclosed action",
+		},
+		{
+			title: "v2 message with templating errors",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Severity:   "{{ ",
+			},
+			errMsg: "failed to template",
+		},
+		{
+			title: "v1 message with templating errors",
+			cfg: &config.PagerdutyConfig{
+				ServiceKey: config.Secret("01234567890123456789012345678901"),
+				Client:     "{{ ",
+			},
+			errMsg: "failed to template",
+		},
+		{
+			title: "routing key cannot be empty",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret(`{{ "" }}`),
+			},
+			errMsg: "routing key cannot be empty",
+		},
+		{
+			title: "service_key cannot be empty",
+			cfg: &config.PagerdutyConfig{
+				ServiceKey: config.Secret(`{{ "" }}`),
+			},
+			errMsg: "service key cannot be empty",
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			tc.cfg.URL = &config.URL{URL: u}
+			tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
+			pd, err := New(tc.cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+			if pd.apiV1 != "" {
+				pd.apiV1 = u.String()
+			}
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			ok, err := pd.Notify(ctx, []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels: model.LabelSet{
+							"lbl1": "val1",
+						},
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}...)
+			if tc.errMsg == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				if errors.Asc(err, errors.CodeInternal) {
+					_, _, errMsg, _, _, _ := errors.Unwrapb(err)
+					require.Contains(t, errMsg, tc.errMsg)
+				} else {
+					require.Contains(t, err.Error(), tc.errMsg)
+				}
+			}
+			require.Equal(t, tc.retry, ok)
+		})
+	}
+}
+
+func TestErrDetails(t *testing.T) {
+	for _, tc := range []struct {
+		status int
+		body   io.Reader
+
+		exp string
+	}{
+		{
+			status: http.StatusBadRequest,
+			body: bytes.NewBuffer([]byte(
+				`{"status":"invalid event","message":"Event object is invalid","errors":["Length of 'routing_key' is incorrect (should be 32 characters)"]}`,
+			)),
+
+			exp: "Length of 'routing_key' is incorrect",
+		},
+		{
+			status: http.StatusBadRequest,
+			body:   bytes.NewBuffer([]byte(`{"status"}`)),
+
+			exp: "",
+		},
+		{
+			status: http.StatusBadRequest,
+
+			exp: "",
+		},
+		{
+			status: http.StatusTooManyRequests,
+
+			exp: "",
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			err := errDetails(tc.status, tc.body)
+			require.Contains(t, err, tc.exp)
+		})
+	}
+}
+
+func TestEventSizeEnforcement(t *testing.T) {
+	bigDetailsV1 := map[string]any{
+		"firing": strings.Repeat("a", 513000),
+	}
+	bigDetailsV2 := map[string]any{
+		"firing": strings.Repeat("a", 513000),
+	}
+
+	// V1 Messages
+	msgV1 := &pagerDutyMessage{
+		ServiceKey: "01234567890123456789012345678901",
+		EventType:  "trigger",
+		Details:    bigDetailsV1,
+	}
+
+	notifierV1, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV1, err := notifierV1.encodeMessage(context.Background(), msgV1)
+	require.NoError(t, err)
+	require.Contains(t, encodedV1.String(), `"details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+
+	// V2 Messages
+	msgV2 := &pagerDutyMessage{
+		RoutingKey:  "01234567890123456789012345678901",
+		EventAction: "trigger",
+		Payload: &pagerDutyPayload{
+			CustomDetails: bigDetailsV2,
+		},
+	}
+
+	notifierV2, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV2, err := notifierV2.encodeMessage(context.Background(), msgV2)
+	require.NoError(t, err)
+	require.Contains(t, encodedV2.String(), `"custom_details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+}
+
+func TestPagerDutyEmptySrcHref(t *testing.T) {
+	type pagerDutyEvent struct {
+		RoutingKey  string           `json:"routing_key"`
+		EventAction string           `json:"event_action"`
+		DedupKey    string           `json:"dedup_key"`
+		Payload     pagerDutyPayload `json:"payload"`
+		Images      []pagerDutyImage
+		Links       []pagerDutyLink
+	}
+
+	images := []config.PagerdutyImage{
+		{
+			Src:  "",
+			Alt:  "Empty src",
+			Href: "https://example.com/",
+		},
+		{
+			Src:  "https://example.com/cat.jpg",
+			Alt:  "Empty href",
+			Href: "",
+		},
+		{
+			Src:  "https://example.com/cat.jpg",
+			Alt:  "",
+			Href: "https://example.com/",
+		},
+	}
+
+	links := []config.PagerdutyLink{
+		{
+			Href: "",
+			Text: "Empty href",
+		},
+		{
+			Href: "https://example.com/",
+			Text: "",
+		},
+	}
+
+	expectedImages := make([]pagerDutyImage, 0, len(images))
+	for _, image := range images {
+		if image.Src == "" {
+			continue
+		}
+		expectedImages = append(expectedImages, pagerDutyImage{
+			Src:  image.Src,
+			Alt:  image.Alt,
+			Href: image.Href,
+		})
+	}
+
+	expectedLinks := make([]pagerDutyLink, 0, len(links))
+	for _, link := range links {
+		if link.Href == "" {
+			continue
+		}
+		expectedLinks = append(expectedLinks, pagerDutyLink{
+			HRef: link.Href,
+			Text: link.Text,
+		})
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			decoder := json.NewDecoder(r.Body)
+			var event pagerDutyEvent
+			if err := decoder.Decode(&event); err != nil {
+				panic(err)
+			}
+
+			if event.RoutingKey == "" || event.EventAction == "" {
+				http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+				return
+			}
+
+			for _, image := range event.Images {
+				if image.Src == "" {
+					http.Error(w, "Event object is invalid: 'image src' is missing or blank", http.StatusBadRequest)
+					return
+				}
+			}
+
+			for _, link := range event.Links {
+				if link.HRef == "" {
+					http.Error(w, "Event object is invalid: 'link href' is missing or blank", http.StatusBadRequest)
+					return
+				}
+			}
+
+			require.Equal(t, expectedImages, event.Images)
+			require.Equal(t, expectedLinks, event.Links)
+		},
+	))
+	defer server.Close()
+
+	url, err := url.Parse(server.URL)
+	require.NoError(t, err)
+
+	pagerDutyConfig := config.PagerdutyConfig{
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+		RoutingKey: config.Secret("01234567890123456789012345678901"),
+		URL:        &config.URL{URL: url},
+		Images:     images,
+		Links:      links,
+	}
+
+	pagerDuty, err := New(&pagerDutyConfig, test.CreateTmpl(t), promslog.NewNopLogger())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+
+	_, err = pagerDuty.Notify(ctx, []*types.Alert{
+		{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"lbl1": "val1",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		},
+	}...)
+	require.NoError(t, err)
+}
+
+func TestPagerDutyTimeout(t *testing.T) {
+	type pagerDutyEvent struct {
+		RoutingKey  string           `json:"routing_key"`
+		EventAction string           `json:"event_action"`
+		DedupKey    string           `json:"dedup_key"`
+		Payload     pagerDutyPayload `json:"payload"`
+		Images      []pagerDutyImage
+		Links       []pagerDutyLink
+	}
+
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(
+				func(w http.ResponseWriter, r *http.Request) {
+					decoder := json.NewDecoder(r.Body)
+					var event pagerDutyEvent
+					if err := decoder.Decode(&event); err != nil {
+						panic(err)
+					}
+
+					if event.RoutingKey == "" || event.EventAction == "" {
+						http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+						return
+					}
+					time.Sleep(tt.latency)
+				},
+			))
+			defer srv.Close()
+			u, err := url.Parse(srv.URL)
+			require.NoError(t, err)
+
+			cfg := config.PagerdutyConfig{
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				URL:        &config.URL{URL: u},
+				Timeout:    tt.timeout,
+			}
+
+			pd, err := New(&cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+			alert := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"lbl1": "val1",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = pd.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+func TestRenderDetails(t *testing.T) {
+	type args struct {
+		details map[string]any
+		data    *template.Data
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    map[string]any
+		wantErr bool
+	}{
+		{
+			name: "flat",
+			args: args{
+				details: map[string]any{
+					"a": "{{ .Status }}",
+					"b": "String",
+				},
+				data: &template.Data{
+					Status: "Flat",
+				},
+			},
+			want: map[string]any{
+				"a": "Flat",
+				"b": "String",
+			},
+			wantErr: false,
+		},
+		{
+			name: "flat error",
+			args: args{
+				details: map[string]any{
+					"a": "{{ .Status",
+				},
+				data: &template.Data{
+					Status: "Error",
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "nested",
+			args: args{
+				details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": "{{ .Status }}",
+							"d": "String",
+						},
+					},
+				},
+				data: &template.Data{
+					Status: "Nested",
+				},
+			},
+			want: map[string]any{
+				"a": map[string]any{
+					"b": map[string]any{
+						"c": "Nested",
+						"d": "String",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "nested error",
+			args: args{
+				details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": "{{ .Status",
+						},
+					},
+				},
+				data: &template.Data{
+					Status: "Error",
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "alerts",
+			args: args{
+				details: map[string]any{
+					"alerts": map[string]any{
+						"firing":       "{{ .Alerts.Firing | toJson }}",
+						"resolved":     "{{ .Alerts.Resolved | toJson }}",
+						"num_firing":   "{{ len .Alerts.Firing }}",
+						"num_resolved": "{{ len .Alerts.Resolved }}",
+					},
+				},
+				data: &template.Data{
+					Alerts: template.Alerts{
+						{
+							Status: "firing",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Firing1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint1",
+							GeneratorURL: "http://generator1",
+							StartsAt:     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "firing",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Firing2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint2",
+							GeneratorURL: "http://generator2",
+							StartsAt:     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "resolved",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Resolved1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint3",
+							GeneratorURL: "http://generator3",
+							StartsAt:     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "resolved",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Resolved2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint4",
+							GeneratorURL: "http://generator4",
+							StartsAt:     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+					},
+				},
+			},
+			want: map[string]any{
+				"alerts": map[string]any{
+					"firing": []any{
+						map[string]any{
+							"status": "firing",
+							"labels": map[string]any{
+								"alertname": "Firing1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint1",
+							"generatorURL": "http://generator1",
+						},
+						map[string]any{
+							"status": "firing",
+							"labels": map[string]any{
+								"alertname": "Firing2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint2",
+							"generatorURL": "http://generator2",
+						},
+					},
+					"resolved": []any{
+						map[string]any{
+							"status": "resolved",
+							"labels": map[string]any{
+								"alertname": "Resolved1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint3",
+							"generatorURL": "http://generator3",
+						},
+						map[string]any{
+							"status": "resolved",
+							"labels": map[string]any{
+								"alertname": "Resolved2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint4",
+							"generatorURL": "http://generator4",
+						},
+					},
+					"num_firing":   2,
+					"num_resolved": 2,
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			n := &Notifier{
+				conf: &config.PagerdutyConfig{
+					Details: tt.args.details,
+				},
+				tmpl: test.CreateTmpl(t),
+			}
+			got, err := n.renderDetails(tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("renderDetails() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			require.Equal(t, tt.want, got)
+		})
+	}
+}

pkg/alertmanager/alertmanagernotify/receiver.go

@@ -2,8 +2,14 @@ package alertmanagernotify
 
 import (
 	"log/slog"
+	"slices"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/email"
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/msteamsv2"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/opsgenie"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/pagerduty"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/slack"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/webhook"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/prometheus/alertmanager/config/receiver"
 	"github.com/prometheus/alertmanager/notify"
@@ -11,6 +17,15 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+var customNotifierIntegrations = []string{
+	webhook.Integration,
+	email.Integration,
+	pagerduty.Integration,
+	opsgenie.Integration,
+	slack.Integration,
+	msteamsv2.Integration,
+}
+
 func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error) {
 	upstreamIntegrations, err := receiver.BuildReceiverIntegrations(nc, tmpl, logger)
 	if err != nil {
@@ -31,14 +46,29 @@ func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Templ
 	)
 
 	for _, integration := range upstreamIntegrations {
-		// skip upstream msteamsv2 integration
-		if integration.Name() != "msteamsv2" {
+		// skip upstream integration if we support custom integration for it
+		if !slices.Contains(customNotifierIntegrations, integration.Name()) {
 			integrations = append(integrations, integration)
 		}
 	}
 
+	for i, c := range nc.WebhookConfigs {
+		add(webhook.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) })
+	}
+	for i, c := range nc.EmailConfigs {
+		add(email.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil })
+	}
+	for i, c := range nc.PagerdutyConfigs {
+		add(pagerduty.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) })
+	}
+	for i, c := range nc.OpsGenieConfigs {
+		add(opsgenie.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) })
+	}
+	for i, c := range nc.SlackConfigs {
+		add(slack.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) })
+	}
 	for i, c := range nc.MSTeamsV2Configs {
-		add("msteamsv2", i, c, func(l *slog.Logger) (notify.Notifier, error) {
+		add(msteamsv2.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
 			return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l)
 		})
 	}

pkg/alertmanager/alertmanagernotify/slack/slack.go

@@ -0,0 +1,279 @@
+package slack
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "slack"
+)
+
+// https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
+const maxTitleLenRunes = 1024
+
+// Notifier implements a Notifier for Slack notifications.
+type Notifier struct {
+	conf    *config.SlackConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+
+	postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
+}
+
+// New returns a new Slack notification handler.
+func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Notifier{
+		conf:         c,
+		tmpl:         t,
+		logger:       l,
+		client:       client,
+		retrier:      &notify.Retrier{},
+		postJSONFunc: notify.PostJSON,
+	}, nil
+}
+
+// request is the request for sending a slack notification.
+type request struct {
+	Channel     string       `json:"channel,omitempty"`
+	Username    string       `json:"username,omitempty"`
+	IconEmoji   string       `json:"icon_emoji,omitempty"`
+	IconURL     string       `json:"icon_url,omitempty"`
+	LinkNames   bool         `json:"link_names,omitempty"`
+	Text        string       `json:"text,omitempty"`
+	Attachments []attachment `json:"attachments"`
+}
+
+// attachment is used to display a richly-formatted message block.
+type attachment struct {
+	Title      string               `json:"title,omitempty"`
+	TitleLink  string               `json:"title_link,omitempty"`
+	Pretext    string               `json:"pretext,omitempty"`
+	Text       string               `json:"text"`
+	Fallback   string               `json:"fallback"`
+	CallbackID string               `json:"callback_id"`
+	Fields     []config.SlackField  `json:"fields,omitempty"`
+	Actions    []config.SlackAction `json:"actions,omitempty"`
+	ImageURL   string               `json:"image_url,omitempty"`
+	ThumbURL   string               `json:"thumb_url,omitempty"`
+	Footer     string               `json:"footer"`
+	Color      string               `json:"color,omitempty"`
+	MrkdwnIn   []string             `json:"mrkdwn_in,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var err error
+
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With("group_key", key)
+	logger.DebugContext(ctx, "extracted group key")
+
+	var (
+		data     = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		tmplText = notify.TmplText(n.tmpl, data, &err)
+	)
+
+	var markdownIn []string
+
+	if len(n.conf.MrkdwnIn) == 0 {
+		markdownIn = []string{"fallback", "pretext", "text"}
+	} else {
+		markdownIn = n.conf.MrkdwnIn
+	}
+
+	title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), maxTitleLenRunes)
+	if truncated {
+		logger.WarnContext(ctx, "Truncated title", "max_runes", maxTitleLenRunes)
+	}
+	att := &attachment{
+		Title:      title,
+		TitleLink:  tmplText(n.conf.TitleLink),
+		Pretext:    tmplText(n.conf.Pretext),
+		Text:       tmplText(n.conf.Text),
+		Fallback:   tmplText(n.conf.Fallback),
+		CallbackID: tmplText(n.conf.CallbackID),
+		ImageURL:   tmplText(n.conf.ImageURL),
+		ThumbURL:   tmplText(n.conf.ThumbURL),
+		Footer:     tmplText(n.conf.Footer),
+		Color:      tmplText(n.conf.Color),
+		MrkdwnIn:   markdownIn,
+	}
+
+	numFields := len(n.conf.Fields)
+	if numFields > 0 {
+		fields := make([]config.SlackField, numFields)
+		for index, field := range n.conf.Fields {
+			// Check if short was defined for the field otherwise fallback to the global setting
+			var short bool
+			if field.Short != nil {
+				short = *field.Short
+			} else {
+				short = n.conf.ShortFields
+			}
+
+			// Rebuild the field by executing any templates and setting the new value for short
+			fields[index] = config.SlackField{
+				Title: tmplText(field.Title),
+				Value: tmplText(field.Value),
+				Short: &short,
+			}
+		}
+		att.Fields = fields
+	}
+
+	numActions := len(n.conf.Actions)
+	if numActions > 0 {
+		actions := make([]config.SlackAction, numActions)
+		for index, action := range n.conf.Actions {
+			slackAction := config.SlackAction{
+				Type:  tmplText(action.Type),
+				Text:  tmplText(action.Text),
+				URL:   tmplText(action.URL),
+				Style: tmplText(action.Style),
+				Name:  tmplText(action.Name),
+				Value: tmplText(action.Value),
+			}
+
+			if action.ConfirmField != nil {
+				slackAction.ConfirmField = &config.SlackConfirmationField{
+					Title:       tmplText(action.ConfirmField.Title),
+					Text:        tmplText(action.ConfirmField.Text),
+					OkText:      tmplText(action.ConfirmField.OkText),
+					DismissText: tmplText(action.ConfirmField.DismissText),
+				}
+			}
+
+			actions[index] = slackAction
+		}
+		att.Actions = actions
+	}
+
+	req := &request{
+		Channel:     tmplText(n.conf.Channel),
+		Username:    tmplText(n.conf.Username),
+		IconEmoji:   tmplText(n.conf.IconEmoji),
+		IconURL:     tmplText(n.conf.IconURL),
+		LinkNames:   n.conf.LinkNames,
+		Text:        tmplText(n.conf.MessageText),
+		Attachments: []attachment{*att},
+	}
+	if err != nil {
+		return false, err
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(req); err != nil {
+		return false, err
+	}
+
+	var u string
+	if n.conf.APIURL != nil {
+		u = n.conf.APIURL.String()
+	} else {
+		content, err := os.ReadFile(n.conf.APIURLFile)
+		if err != nil {
+			return false, err
+		}
+		u = strings.TrimSpace(string(content))
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeInternal, "configured slack timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := n.postJSONFunc(ctx, n.client, u, &buf) //nolint:bodyclose
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.NewInternalf(errors.CodeInternal, "failed to post JSON to slack: %v", context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	// Use a retrier to generate an error message for non-200 responses and
+	// classify them as retriable or not.
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		err = errors.NewInternalf(errors.CodeInternal, "channel %q: %v", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+
+	// Slack web API might return errors with a 200 response code.
+	// https://slack.dev/node-slack-sdk/web-api#handle-errors
+	retry, err = checkResponseError(resp)
+	if err != nil {
+		err = errors.NewInternalf(errors.CodeInternal, "channel %q: %v", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.ClientErrorReason, err)
+	}
+
+	return retry, nil
+}
+
+// checkResponseError parses out the error message from Slack API response.
+func checkResponseError(resp *http.Response) (bool, error) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "could not read response body")
+	}
+
+	if strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") {
+		return checkJSONResponseError(body)
+	}
+	return checkTextResponseError(body)
+}
+
+// checkTextResponseError classifies plaintext responses from Slack.
+// A plaintext (non-JSON) response is successful if it's a string "ok".
+// This is typically a response for an Incoming Webhook
+// (https://api.slack.com/messaging/webhooks#handling_errors)
+func checkTextResponseError(body []byte) (bool, error) {
+	if !bytes.Equal(body, []byte("ok")) {
+		return false, errors.NewInternalf(errors.CodeInternal, "received an error response from Slack: %s", string(body))
+	}
+	return false, nil
+}
+
+// checkJSONResponseError classifies JSON responses from Slack.
+func checkJSONResponseError(body []byte) (bool, error) {
+	// response is for parsing out errors from the JSON response.
+	type response struct {
+		OK    bool   `json:"ok"`
+		Error string `json:"error"`
+	}
+
+	var data response
+	if err := json.Unmarshal(body, &data); err != nil {
+		return true, errors.NewInternalf(errors.CodeInternal, "could not unmarshal JSON response %q: %v", string(body), err)
+	}
+	if !data.OK {
+		return false, errors.NewInternalf(errors.CodeInternal, "error response from Slack: %s", data.Error)
+	}
+	return false, nil
+}

pkg/alertmanager/alertmanagernotify/slack/slack_test.go

@@ -0,0 +1,339 @@
+package slack
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestSlackRetry(t *testing.T) {
+	notifier, err := New(
+		&config.SlackConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestSlackRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURL:     &config.SecretURL{URL: u},
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestGettingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String())
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestTrimmingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test_newline")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestNotifier_Notify_WithReason(t *testing.T) {
+	tests := []struct {
+		name           string
+		statusCode     int
+		responseBody   string
+		expectedReason notify.Reason
+		expectedErr    string
+		expectedRetry  bool
+		noError        bool
+	}{
+		{
+			name:           "with a 4xx status code",
+			statusCode:     http.StatusUnauthorized,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 401",
+		},
+		{
+			name:           "with a 5xx status code",
+			statusCode:     http.StatusInternalServerError,
+			expectedReason: notify.ServerErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "unexpected status code 500",
+		},
+		{
+			name:           "with a 3xx status code",
+			statusCode:     http.StatusTemporaryRedirect,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 307",
+		},
+		{
+			name:           "with a 1xx status code",
+			statusCode:     http.StatusSwitchingProtocols,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 101",
+		},
+		{
+			name:           "2xx response with invalid JSON",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"not valid json"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "could not unmarshal",
+		},
+		{
+			name:           "2xx response with a JSON error",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"ok":false,"error":"error_message"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: error_message",
+		},
+		{
+			name:           "2xx response with a plaintext error",
+			statusCode:     http.StatusOK,
+			responseBody:   "no_channel",
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: no_channel",
+		},
+		{
+			name:         "successful JSON response",
+			statusCode:   http.StatusOK,
+			responseBody: `{"ok":true}`,
+			noError:      true,
+		},
+		{
+			name:         "successful plaintext response",
+			statusCode:   http.StatusOK,
+			responseBody: "ok",
+			noError:      true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			apiurl, _ := url.Parse("https://slack.com/post.Message")
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: apiurl},
+					Channel:        "channelname",
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				resp := httptest.NewRecorder()
+				if strings.HasPrefix(tt.responseBody, "{") {
+					resp.Header().Add("Content-Type", "application/json; charset=utf-8")
+				}
+				resp.WriteHeader(tt.statusCode)
+				_, _ = resp.WriteString(tt.responseBody)
+				return resp.Result(), nil
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			retry, err := notifier.Notify(ctx, alert1)
+			require.Equal(t, tt.expectedRetry, retry)
+			if tt.noError {
+				require.NoError(t, err)
+			} else {
+				var reasonError *notify.ErrorWithReason
+				require.ErrorAs(t, err, &reasonError)
+				require.Equal(t, tt.expectedReason, reasonError.Reason)
+				require.Contains(t, err.Error(), tt.expectedErr)
+				require.Contains(t, err.Error(), "channelname")
+			}
+		})
+	}
+}
+
+func TestSlackTimeout(t *testing.T) {
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			u, _ := url.Parse("https://slack.com/post.Message")
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: u},
+					Channel:        "channelname",
+					Timeout:        tt.timeout,
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				select {
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				case <-time.After(tt.latency):
+					resp := httptest.NewRecorder()
+					resp.Header().Set("Content-Type", "application/json; charset=utf-8")
+					resp.WriteHeader(http.StatusOK)
+					_, _ = resp.WriteString(`{"ok":true}`)
+
+					return resp.Result(), nil
+				}
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = notifier.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+func TestSlackMessageField(t *testing.T) {
+	// 1. Setup a fake Slack server
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatal(err)
+		}
+
+		// 2. VERIFY: Top-level text exists
+		if body["text"] != "My Top Level Message" {
+			t.Errorf("Expected top-level 'text' to be 'My Top Level Message', got %v", body["text"])
+		}
+
+		// 3. VERIFY: Old attachments still exist
+		attachments, ok := body["attachments"].([]any)
+		if !ok || len(attachments) == 0 {
+			t.Errorf("Expected attachments to exist")
+		} else {
+			first := attachments[0].(map[string]any)
+			if first["title"] != "Old Attachment Title" {
+				t.Errorf("Expected attachment title 'Old Attachment Title', got %v", first["title"])
+			}
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"ok": true}`))
+	}))
+	defer server.Close()
+
+	// 4. Configure Notifier with BOTH new and old fields
+	u, _ := url.Parse(server.URL)
+	conf := &config.SlackConfig{
+		APIURL:      &config.SecretURL{URL: u},
+		MessageText: "My Top Level Message", // Your NEW field
+		Title:       "Old Attachment Title", // An OLD field
+		Channel:     "#test-channel",
+		HTTPConfig:  &commoncfg.HTTPClientConfig{},
+	}
+
+	tmpl, err := template.FromGlobs([]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpl.ExternalURL = u
+
+	logger := slog.New(slog.DiscardHandler)
+	notifier, err := New(conf, tmpl, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group-key")
+
+	if _, err := notifier.Notify(ctx); err != nil {
+		t.Fatal("Notify failed:", err)
+	}
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook.go

@@ -0,0 +1,136 @@
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "webhook"
+)
+
+// Notifier implements a Notifier for generic webhooks.
+type Notifier struct {
+	conf    *config.WebhookConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new Webhook.
+func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*conf.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:   conf,
+		tmpl:   t,
+		logger: l,
+		client: client,
+		// Webhooks are assumed to respond with 2xx response codes on a successful
+		// request and 5xx response codes are assumed to be recoverable.
+		retrier: &notify.Retrier{},
+	}, nil
+}
+
+// Message defines the JSON object send to webhook endpoints.
+type Message struct {
+	*template.Data
+
+	// The protocol version.
+	Version         string `json:"version"`
+	GroupKey        string `json:"groupKey"`
+	TruncatedAlerts uint64 `json:"truncatedAlerts"`
+}
+
+func truncateAlerts(maxAlerts uint64, alerts []*types.Alert) ([]*types.Alert, uint64) {
+	if maxAlerts != 0 && uint64(len(alerts)) > maxAlerts {
+		return alerts[:maxAlerts], uint64(len(alerts)) - maxAlerts
+	}
+
+	return alerts, 0
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
+	alerts, numTruncated := truncateAlerts(n.conf.MaxAlerts, alerts)
+	data := notify.GetTemplateData(ctx, n.tmpl, alerts, n.logger)
+
+	groupKey, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	logger := n.logger.With("group_key", groupKey)
+	logger.DebugContext(ctx, "extracted group key")
+
+	msg := &Message{
+		Version:         "4",
+		Data:            data,
+		GroupKey:        groupKey.String(),
+		TruncatedAlerts: numTruncated,
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return false, err
+	}
+
+	var url string
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.URL != "" {
+		url = tmpl(string(n.conf.URL))
+	} else {
+		content, err := os.ReadFile(n.conf.URLFile)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "read url_file")
+		}
+		url = tmpl(strings.TrimSpace(string(content)))
+	}
+
+	if tmplErr != nil {
+		return false, errors.NewInternalf(errors.CodeInternal, "failed to template webhook URL: %v", tmplErr)
+	}
+
+	if url == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "webhook URL is empty after templating")
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeInternal, "configured webhook timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, url, &buf) //nolint:bodyclose
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.NewInternalf(errors.CodeInternal, "failed to post JSON to webhook: %v", context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return shouldRetry, err
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook_test.go

@@ -0,0 +1,214 @@
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestWebhookRetry(t *testing.T) {
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL("http://example.com"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	if err != nil {
+		require.NoError(t, err)
+	}
+
+	t.Run("test retry status code", func(t *testing.T) {
+		for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+			actual, _ := notifier.retrier.Check(statusCode, nil)
+			require.Equal(t, expected, actual, "error on status %d", statusCode)
+		}
+	})
+
+	t.Run("test retry error details", func(t *testing.T) {
+		for _, tc := range []struct {
+			status int
+			body   io.Reader
+
+			exp string
+		}{
+			{
+				status: http.StatusBadRequest,
+				body: bytes.NewBuffer([]byte(
+					`{"status":"invalid event"}`,
+				)),
+
+				exp: fmt.Sprintf(`unexpected status code %d: {"status":"invalid event"}`, http.StatusBadRequest),
+			},
+			{
+				status: http.StatusBadRequest,
+
+				exp: fmt.Sprintf(`unexpected status code %d`, http.StatusBadRequest),
+			},
+		} {
+			t.Run("", func(t *testing.T) {
+				_, err = notifier.retrier.Check(tc.status, tc.body)
+				require.Equal(t, tc.exp, err.Error())
+			})
+		}
+	})
+}
+
+func TestWebhookTruncateAlerts(t *testing.T) {
+	alerts := make([]*types.Alert, 10)
+
+	truncatedAlerts, numTruncated := truncateAlerts(0, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(4, alerts)
+	require.Len(t, truncatedAlerts, 4)
+	require.EqualValues(t, 6, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(100, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+}
+
+func TestWebhookRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	secret := "secret"
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL(u.String()),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, secret)
+}
+
+func TestWebhookReadingURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "webhook_url")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.WebhookConfig{
+			URLFile:    f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestWebhookURLTemplating(t *testing.T) {
+	var calledURL string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calledURL = r.URL.Path
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	tests := []struct {
+		name           string
+		url            string
+		groupLabels    model.LabelSet
+		alertLabels    model.LabelSet
+		expectError    bool
+		expectedErrMsg string
+		expectedPath   string
+	}{
+		{
+			name:         "templating with alert labels",
+			url:          srv.URL + "/{{ .GroupLabels.alertname }}/{{ .CommonLabels.severity }}",
+			groupLabels:  model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:  model.LabelSet{"alertname": "TestAlert", "severity": "critical"},
+			expectError:  false,
+			expectedPath: "/TestAlert/critical",
+		},
+		{
+			name:           "invalid template field",
+			url:            srv.URL + "/{{ .InvalidField }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "failed to template webhook URL",
+		},
+		{
+			name:           "template renders to empty string",
+			url:            "{{ if .CommonLabels.nonexistent }}http://example.com{{ end }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "webhook URL is empty after templating",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			calledURL = "" // Reset for each test
+
+			notifier, err := New(
+				&config.WebhookConfig{
+					URL:        config.SecretTemplateURL(tc.url),
+					HTTPConfig: &commoncfg.HTTPClientConfig{},
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "test-group")
+			if tc.groupLabels != nil {
+				ctx = notify.WithGroupLabels(ctx, tc.groupLabels)
+			}
+
+			alerts := []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels:   tc.alertLabels,
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}
+
+			_, err = notifier.Notify(ctx, alerts...)
+
+			if tc.expectError {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expectedErrMsg)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedPath, calledURL)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate.go

@@ -0,0 +1,262 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+)
+
+// AlertManagerTemplater processes alert notification templates.
+type AlertManagerTemplater interface {
+	// ProcessTemplates expands the title and body templates from input
+	// against the provided alerts and returns the expanded templates.
+	ProcessTemplates(ctx context.Context, input TemplateInput, alerts []*types.Alert) (*ExpandedTemplates, error)
+}
+
+// alertManagerTemplater is the private implementation of AlertTemplater.
+type alertManagerTemplater struct {
+	tmpl   *template.Template
+	logger *slog.Logger
+}
+
+// New returns a new AlertTemplater with the given template and logger.
+func New(tmpl *template.Template, logger *slog.Logger) AlertManagerTemplater {
+	return &alertManagerTemplater{tmpl: tmpl, logger: logger}
+}
+
+// ExtractTemplatesFromAnnotations computes the common annotations across all alerts
+// and returns the values for the title_template and body_template annotation keys.
+func ExtractTemplatesFromAnnotations(alerts []*types.Alert) (titleTemplate, bodyTemplate string) {
+	if len(alerts) == 0 {
+		return "", ""
+	}
+
+	commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+	return commonAnnotations[ruletypes.AnnotationTitleTemplate], commonAnnotations[ruletypes.AnnotationBodyTemplate]
+}
+
+// ProcessTemplates expands the title and body templates from input
+// against the provided alerts and returns the expanded templates.
+func (at *alertManagerTemplater) ProcessTemplates(
+	ctx context.Context,
+	input TemplateInput,
+	alerts []*types.Alert,
+) (*ExpandedTemplates, error) {
+	ntd := at.buildNotificationTemplateData(ctx, alerts)
+
+	title, titleMissingVars, err := at.expandTitle(ctx, input, alerts, ntd)
+	if err != nil {
+		return nil, err
+	}
+
+	body, bodyMissingVars, err := at.expandBody(ctx, input, alerts, ntd)
+	if err != nil {
+		return nil, err
+	}
+
+	missingVars := make(map[string]bool)
+	for k := range titleMissingVars {
+		missingVars[k] = true
+	}
+	for k := range bodyMissingVars {
+		missingVars[k] = true
+	}
+
+	return &ExpandedTemplates{Title: title, Body: body, MissingVars: missingVars}, nil
+}
+
+// expandTitle expands the title template. Falls back to the default if the custom template
+// result in empty string.
+func (at *alertManagerTemplater) expandTitle(
+	ctx context.Context,
+	input TemplateInput,
+	alerts []*types.Alert,
+	ntd *NotificationTemplateData,
+) (string, map[string]bool, error) {
+	if input.TitleTemplate != "" {
+		processRes, err := PreProcessTemplateAndData(input.TitleTemplate, ntd)
+		if err != nil {
+			return "", nil, err
+		}
+		result, err := at.tmpl.ExecuteTextString(processRes.Template, processRes.Data)
+		if err != nil {
+			return "", nil, err
+		}
+		if strings.TrimSpace(result) != "" {
+			return result, processRes.UnknownVars, nil
+		}
+	}
+
+	// Fall back to the notifier's default title template using standard template.Data.
+	if input.DefaultTitleTemplate == "" {
+		return "", nil, nil
+	}
+	data := notify.GetTemplateData(ctx, at.tmpl, alerts, at.logger)
+	result, err := at.tmpl.ExecuteTextString(input.DefaultTitleTemplate, data)
+	return result, nil, err
+}
+
+// expandBody expands the body template once per alert, concatenates the results
+// and falls back to the default if the custom template result in empty string.
+func (at *alertManagerTemplater) expandBody(
+	ctx context.Context,
+	input TemplateInput,
+	alerts []*types.Alert,
+	ntd *NotificationTemplateData,
+) (string, map[string]bool, error) {
+	if input.BodyTemplate != "" {
+		var parts []string
+		missingVars := make(map[string]bool)
+		for i := range ntd.Alerts {
+			processRes, err := PreProcessTemplateAndData(input.BodyTemplate, &ntd.Alerts[i])
+			if err != nil {
+				return "", nil, err
+			}
+			for k := range processRes.UnknownVars {
+				missingVars[k] = true
+			}
+			part, err := at.tmpl.ExecuteTextString(processRes.Template, processRes.Data)
+			if err != nil {
+				return "", nil, err
+			}
+			parts = append(parts, part)
+		}
+		result := strings.Join(parts, "<br><br>") // markdown uses html for line breaks
+		if strings.TrimSpace(result) != "" {
+			return result, missingVars, nil
+		}
+	}
+
+	// Fall back to the notifier's default body template using standard template.Data.
+	if input.DefaultBodyTemplate == "" {
+		return "", nil, nil
+	}
+	data := notify.GetTemplateData(ctx, at.tmpl, alerts, at.logger)
+	result, err := at.tmpl.ExecuteTextString(input.DefaultBodyTemplate, data)
+	return result, nil, err
+}
+
+// buildNotificationTemplateData derives a NotificationTemplateData from
+// the context, template, and the raw alerts.
+func (at *alertManagerTemplater) buildNotificationTemplateData(
+	ctx context.Context,
+	alerts []*types.Alert,
+) *NotificationTemplateData {
+	// extract the required data from the context
+	receiver, ok := notify.ReceiverName(ctx)
+	if !ok {
+		at.logger.WarnContext(ctx, "missing receiver name in context")
+	}
+
+	groupLabels, ok := notify.GroupLabels(ctx)
+	if !ok {
+		at.logger.WarnContext(ctx, "missing group labels in context")
+	}
+
+	// extract the external URL from the template
+	externalURL := ""
+	if at.tmpl.ExternalURL != nil {
+		externalURL = at.tmpl.ExternalURL.String()
+	}
+
+	commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+	commonLabels := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
+
+	// aggregate labels and annotations from all alerts
+	labels := AggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
+	annotations := AggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+
+	// build the alert data slice
+	alertDataSlice := make([]AlertData, 0, len(alerts))
+	for _, a := range alerts {
+		ad := buildAlertData(a, receiver)
+		alertDataSlice = append(alertDataSlice, ad)
+	}
+
+	// count the number of firing and resolved alerts
+	var firing, resolved int
+	for _, ad := range alertDataSlice {
+		if ad.IsFiring {
+			firing++
+		} else if ad.IsResolved {
+			resolved++
+		}
+	}
+
+	// extract the rule-level convenience fields from common labels
+	alertName := commonLabels[ruletypes.LabelAlertName]
+	ruleID := commonLabels[ruletypes.LabelRuleId]
+	ruleLink := commonLabels[ruletypes.LabelRuleSource]
+
+	// build the group labels
+	gl := make(template.KV, len(groupLabels))
+	for k, v := range groupLabels {
+		gl[string(k)] = string(v)
+	}
+
+	// build the notification template data
+	return &NotificationTemplateData{
+		Receiver:          receiver,
+		Status:            string(types.Alerts(alerts...).Status()),
+		AlertName:         alertName,
+		RuleID:            ruleID,
+		RuleLink:          ruleLink,
+		TotalFiring:       firing,
+		TotalResolved:     resolved,
+		Alerts:            alertDataSlice,
+		GroupLabels:       gl,
+		CommonLabels:      commonLabels,
+		CommonAnnotations: commonAnnotations,
+		ExternalURL:       externalURL,
+		Labels:            labels,
+		Annotations:       annotations,
+	}
+}
+
+// buildAlertData converts a single *types.Alert into an AlertData.
+func buildAlertData(a *types.Alert, receiver string) AlertData {
+	labels := make(template.KV, len(a.Labels))
+	for k, v := range a.Labels {
+		labels[string(k)] = string(v)
+	}
+
+	annotations := make(template.KV, len(a.Annotations))
+	for k, v := range a.Annotations {
+		annotations[string(k)] = string(v)
+	}
+
+	status := string(a.Status())
+	isFiring := a.Status() == model.AlertFiring
+	isResolved := a.Status() == model.AlertResolved
+	isMissingData := labels[ruletypes.LabelNoData] == "true"
+
+	return AlertData{
+		Receiver:      receiver,
+		Status:        status,
+		Labels:        labels,
+		Annotations:   annotations,
+		StartsAt:      a.StartsAt,
+		EndsAt:        a.EndsAt,
+		GeneratorURL:  a.GeneratorURL,
+		Fingerprint:   a.Fingerprint().String(),
+		AlertName:     labels[ruletypes.LabelAlertName],
+		RuleID:        labels[ruletypes.LabelRuleId],
+		RuleLink:      labels[ruletypes.LabelRuleSource],
+		Severity:      labels[ruletypes.LabelSeverityName],
+		LogLink:       annotations[ruletypes.AnnotationRelatedLogs],
+		TraceLink:     annotations[ruletypes.AnnotationRelatedTraces],
+		Value:         annotations[ruletypes.AnnotationValue],
+		Threshold:     annotations[ruletypes.AnnotationThreshold],
+		CompareOp:     annotations[ruletypes.AnnotationCompareOp],
+		MatchType:     annotations[ruletypes.AnnotationMatchType],
+		IsFiring:      isFiring,
+		IsResolved:    isResolved,
+		IsMissingData: isMissingData,
+	}
+}

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate_test.go

@@ -0,0 +1,277 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"testing"
+	"time"
+
+	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// testSetup returns an AlertTemplater and a context pre-populated with group key,
+// receiver name, and group labels for use in tests.
+func testSetup(t *testing.T) (AlertManagerTemplater, context.Context) {
+	t.Helper()
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "slack")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": "TestAlert", "severity": "critical"})
+	logger := slog.New(slog.DiscardHandler)
+	return New(tmpl, logger), ctx
+}
+
+// firingAlert returns a firing alert with the given labels/annotations.
+func firingAlert(labels, annotations map[string]string) *types.Alert {
+	ls := model.LabelSet{}
+	for k, v := range labels {
+		ls[model.LabelName(k)] = model.LabelValue(v)
+	}
+	ann := model.LabelSet{}
+	for k, v := range annotations {
+		ann[model.LabelName(k)] = model.LabelValue(v)
+	}
+	now := time.Now()
+	return &types.Alert{
+		Alert: model.Alert{
+			Labels:      ls,
+			Annotations: ann,
+			StartsAt:    now,
+			EndsAt:      now.Add(time.Hour),
+		},
+	}
+}
+
+// resolvedAlert returns a resolved alert (EndsAt in the past) with the given labels/annotations.
+func resolvedAlert(labels, annotations map[string]string) *types.Alert {
+	ls := model.LabelSet{}
+	for k, v := range labels {
+		ls[model.LabelName(k)] = model.LabelValue(v)
+	}
+	ann := model.LabelSet{}
+	for k, v := range annotations {
+		ann[model.LabelName(k)] = model.LabelValue(v)
+	}
+	now := time.Now()
+	return &types.Alert{
+		Alert: model.Alert{
+			Labels:      ls,
+			Annotations: ann,
+			StartsAt:    now.Add(-2 * time.Hour),
+			EndsAt:      now.Add(-time.Hour),
+		},
+	}
+}
+
+func TestExpandTemplates(t *testing.T) {
+	at, ctx := testSetup(t)
+
+	tests := []struct {
+		name            string
+		alerts          []*types.Alert
+		input           TemplateInput
+		wantTitle       string
+		wantBody        string
+		wantMissingVars []string
+	}{
+		{
+			// High request throughput on a service — service is a custom label.
+			// $labels.service extracts the label value; $annotations.description pulls the annotation.
+			name: "new template: high request throughput for a service",
+			alerts: []*types.Alert{
+				firingAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "HighRequestThroughput",
+						ruletypes.LabelSeverityName: "warning",
+						"service":                   "payment-service",
+					},
+					map[string]string{"description": "Request rate exceeded 10k/s"},
+				),
+			},
+			input: TemplateInput{
+				TitleTemplate: "High request throughput for $service",
+				BodyTemplate: `The service $service is getting high request. Please investigate.
+Severity: $severity
+Status: $status
+Service: $service
+Description: $description`,
+			},
+			wantTitle: "High request throughput for payment-service",
+			wantBody: `The service payment-service is getting high request. Please investigate.
+Severity: warning
+Status: firing
+Service: payment-service
+Description: Request rate exceeded 10k/s`,
+		},
+		{
+			// Disk usage alert using old Go template syntax throughout.
+			// No custom templates — both title and body use the default fallback path.
+			name: "old template: disk usage high on database host",
+			alerts: []*types.Alert{
+				firingAlert(
+					map[string]string{ruletypes.LabelAlertName: "DiskUsageHigh",
+						ruletypes.LabelSeverityName: "critical",
+						"instance":                  "db-primary-01",
+					},
+					map[string]string{
+						"summary":        "Disk usage high on database host",
+						"description":    "Disk usage is high on the database host",
+						"related_logs":   "https://logs.example.com/search?q=DiskUsageHigh",
+						"related_traces": "https://traces.example.com/search?q=DiskUsageHigh",
+					},
+				),
+			},
+			input: TemplateInput{
+				DefaultTitleTemplate: `[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
+     {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
+       {{" "}}(
+       {{- with .CommonLabels.Remove .GroupLabels.Names }}
+         {{- range $index, $label := .SortedPairs -}}
+           {{ if $index }}, {{ end }}
+           {{- $label.Name }}="{{ $label.Value -}}"
+         {{- end }}
+       {{- end -}}
+       )
+     {{- end }}`,
+				DefaultBodyTemplate: `{{ range .Alerts -}}
+     *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}
+
+     *Summary:* {{ .Annotations.summary }}
+     *Description:* {{ .Annotations.description }}
+     *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}
+     *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}
+
+     *Details:*
+       {{ range .Labels.SortedPairs }} • *{{ .Name }}:* {{ .Value }}
+       {{ end }}
+     {{ end }}`,
+			},
+			wantTitle: "[FIRING:1] DiskUsageHigh for  (instance=\"db-primary-01\")",
+			wantBody: `*Alert:* DiskUsageHigh - critical
+
+     *Summary:* Disk usage high on database host
+     *Description:* Disk usage is high on the database host
+     *RelatedLogs:* View in <https://logs.example.com/search?q=DiskUsageHigh|logs explorer>
+     *RelatedTraces:* View in <https://traces.example.com/search?q=DiskUsageHigh|traces explorer>
+
+     *Details:*
+        • *alertname:* DiskUsageHigh
+        • *instance:* db-primary-01
+        • *severity:* critical
+       
+     `,
+		},
+		{
+			// Pod crash loop on multiple pods — body is expanded once per alert
+			// and joined with "<br><br>", with the pod name pulled from labels.
+			name: "new template: pod crash loop on multiple pods, body per-alert",
+			alerts: []*types.Alert{
+				firingAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-1"}, nil),
+				firingAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-2"}, nil),
+				firingAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-3"}, nil),
+			},
+			input: TemplateInput{
+				TitleTemplate: "$rule_name: $total_firing pods affected",
+				BodyTemplate:  "$labels.pod is crash looping",
+			},
+			wantTitle: "PodCrashLoop: 3 pods affected",
+			wantBody:  "api-worker-1 is crash looping<br><br>api-worker-2 is crash looping<br><br>api-worker-3 is crash looping",
+		},
+		{
+			// Incident partially resolved — one service still down, one recovered.
+			// Title shows the aggregate counts; body shows per-service status.
+			name: "new template: service degradation with mixed firing and resolved alerts",
+			alerts: []*types.Alert{
+				firingAlert(map[string]string{ruletypes.LabelAlertName: "ServiceDown", "service": "auth-service"}, nil),
+				resolvedAlert(map[string]string{ruletypes.LabelAlertName: "ServiceDown", "service": "payment-service"}, nil),
+			},
+			input: TemplateInput{
+				TitleTemplate: "$total_firing firing, $total_resolved resolved",
+				BodyTemplate:  "$labels.service ($status)",
+			},
+			wantTitle: "1 firing, 1 resolved",
+			wantBody:  "auth-service (firing)<br><br>payment-service (resolved)",
+		},
+		{
+			// $environment is not a known AlertData or NotificationTemplateData field,
+			// so it lands in MissingVars and renders as "<no value>" in the output.
+			name: "missing vars: unknown $environment variable in title",
+			alerts: []*types.Alert{
+				firingAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil),
+			},
+			input: TemplateInput{
+				TitleTemplate: "[$environment] $rule_name",
+			},
+			wantTitle:       "[<no value>] HighCPU",
+			wantMissingVars: []string{"environment"},
+		},
+		{
+			// $runbook_url is not a known field — someone tried to embed a runbook link
+			// directly as a variable instead of via annotations.
+			name: "missing vars: unknown $runbook_url variable in body",
+			alerts: []*types.Alert{
+				firingAlert(map[string]string{ruletypes.LabelAlertName: "PodOOMKilled", ruletypes.LabelSeverityName: "warning"}, nil),
+			},
+			input: TemplateInput{
+				BodyTemplate: "$rule_name: see runbook at $runbook_url",
+			},
+			wantBody:        "PodOOMKilled: see runbook at <no value>",
+			wantMissingVars: []string{"runbook_url"},
+		},
+		{
+			// Both title and body use unknown variables; MissingVars is the union of both.
+			name: "missing vars: unknown variables in both title and body",
+			alerts: []*types.Alert{
+				firingAlert(map[string]string{ruletypes.LabelAlertName: "HighMemory", ruletypes.LabelSeverityName: "critical"}, nil),
+			},
+			input: TemplateInput{
+				TitleTemplate: "[$environment] $rule_name",
+				BodyTemplate:  "$rule_name: see runbook at $runbook_url",
+			},
+			wantTitle:       "[<no value>] HighMemory",
+			wantBody:        "HighMemory: see runbook at <no value>",
+			wantMissingVars: []string{"environment", "runbook_url"},
+		},
+		{
+			// Custom title template that expands to only whitespace triggers the fallback,
+			// so the default title template is used instead.
+			name: "fallback: whitespace-only custom title falls back to default",
+			alerts: []*types.Alert{
+				firingAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil),
+			},
+			input: TemplateInput{
+				TitleTemplate:        "   ",
+				DefaultTitleTemplate: "{{ .CommonLabels.alertname }}",
+				BodyTemplate:         "$rule_name ($severity) for $alertname",
+			},
+			wantTitle: "HighCPU",
+			wantBody:  "HighCPU (critical) for HighCPU",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := at.ProcessTemplates(ctx, tc.input, tc.alerts)
+			require.NoError(t, err)
+
+			if tc.wantTitle != "" {
+				require.Equal(t, tc.wantTitle, got.Title)
+			}
+			if tc.wantBody != "" {
+				require.Equal(t, tc.wantBody, got.Body)
+			}
+
+			require.Len(t, got.MissingVars, len(tc.wantMissingVars))
+			for _, v := range tc.wantMissingVars {
+				require.True(t, got.MissingVars[v], "expected %q in MissingVars", v)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/preprocessor.go

@@ -0,0 +1,237 @@
+package alertmanagertemplate
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/go-viper/mapstructure/v2"
+)
+
+// fieldMapping represents a mapping from a JSON tag name to its struct field name.
+type fieldMapping struct {
+	VarName   string // JSON tag name (e.g., "receiver", "rule_name")
+	FieldName string // Struct field name (e.g., "Receiver", "AlertName")
+}
+
+// extractFieldMappings uses reflection to extract field mappings from a struct.
+func extractFieldMappings(data any) []fieldMapping {
+	val := reflect.ValueOf(data)
+	// Handle pointer types
+	if val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return nil
+		}
+		val = val.Elem()
+	}
+	// return nil if the given data is not a struct
+	if val.Kind() != reflect.Struct {
+		return nil
+	}
+	typ := val.Type()
+
+	var mappings []fieldMapping
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		// Skip unexported fields
+		if !field.IsExported() {
+			continue
+		}
+		// Get JSON tag name
+		jsonTag := field.Tag.Get("json")
+		if jsonTag == "" || jsonTag == "-" {
+			continue
+		}
+		// Extract the name part (before any comma options like omitempty)
+		varName := strings.Split(jsonTag, ",")[0]
+		if varName == "" {
+			continue
+		}
+		varFieldName := field.Tag.Get("mapstructure")
+		if varFieldName == "" {
+			varFieldName = field.Name
+		}
+		// Skip complex types: slices and interfaces
+		kind := field.Type.Kind()
+		if kind == reflect.Slice || kind == reflect.Interface {
+			continue
+		}
+		// For struct types, we skip all but with few exceptions like time.Time
+		if kind == reflect.Struct {
+			// Allow time.Time which is commonly used
+			if field.Type.String() != "time.Time" {
+				continue
+			}
+		}
+
+		mappings = append(mappings, fieldMapping{
+			VarName:   varName,
+			FieldName: varFieldName,
+		})
+	}
+
+	return mappings
+}
+
+// extractNestedFieldsDefinitions extracts the labels, annotations map and adds their keys to template variable definitions
+func extractNestedFieldsDefinitions(data any) map[string]string {
+	variables := make(map[string]string)
+
+	addLabelsAndAnnotations := func(labels, annotations map[string]string) {
+		for k := range annotations {
+			variables[k] = fmt.Sprintf("index .annotations \"%s\"", k)
+		}
+		for k := range labels {
+			variables[k] = fmt.Sprintf("index .labels \"%s\"", k)
+		}
+	}
+
+	switch data := data.(type) {
+	case *NotificationTemplateData:
+		addLabelsAndAnnotations(data.Labels, data.Annotations)
+	case *AlertData:
+		addLabelsAndAnnotations(data.Labels, data.Annotations)
+	default:
+		return variables
+	}
+
+	return variables
+}
+
+// prepareDataForTemplating prepares the data for templating by adding the labels and annotations values to the resulting map
+// so they can be accessed directly from root level, the default values takes precedence over the labels and annotations values
+func prepareDataForTemplating(data any) (map[string]interface{}, error) {
+	var result map[string]interface{}
+	if err := mapstructure.Decode(data, &result); err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to prepare data for templating")
+	}
+
+	addLabelsAndAnnotationsValues := func(labels, annotations map[string]string) {
+		for k, v := range labels {
+			if _, ok := result[k]; !ok {
+				result[k] = v
+			}
+		}
+		for k, v := range annotations {
+			if _, ok := result[k]; !ok {
+				result[k] = v
+			}
+		}
+	}
+
+	switch data := data.(type) {
+	case *NotificationTemplateData:
+		addLabelsAndAnnotationsValues(data.Labels, data.Annotations)
+	case *AlertData:
+		addLabelsAndAnnotationsValues(data.Labels, data.Annotations)
+	default:
+		return result, nil
+	}
+
+	return result, nil
+}
+
+// generateVariableDefinitions creates `{{ $varname := "" }}` declarations for each variable name.
+func generateVariableDefinitions(varNames map[string]string) string {
+	if len(varNames) == 0 {
+		return ""
+	}
+	var sb strings.Builder
+	for name := range varNames {
+		fmt.Fprintf(&sb, `{{ $%s := %s }}`, name, varNames[name])
+	}
+	return sb.String()
+}
+
+// buildVariableDefinitions constructs the full variable definition preamble for a template.
+// containing all known and unknown variables, the reason to include unknown variables is to
+// populate them with "<no value>" in template so go-text-template don't throw errors
+// when these variables are used in the template.
+func buildVariableDefinitions(tmpl string, data any) (string, map[string]bool, error) {
+	// Extract the initial fields from the data struct and add to the definitions
+	mappings := extractFieldMappings(data)
+
+	variables := make(map[string]string)
+	for _, m := range mappings {
+		variables[m.VarName] = fmt.Sprintf(".%s", m.FieldName)
+	}
+
+	// Extract the nested fields definitions from the data struct
+	// nested fields are all maps that are present in data like labels, annotations, etc.
+	// once extracted we add them to the variables map along with the field address
+	nestedVariables := extractNestedFieldsDefinitions(data)
+	for k, v := range nestedVariables {
+		variables[k] = v
+	}
+
+	// variables that are used throughout the template
+	usedVars, err := ExtractUsedVariables(tmpl)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// Compute unknown variables: used in template but not covered by a field mapping
+	probableUnknownVars := make(map[string]bool)
+	for name := range usedVars {
+		_, ok := variables[name]
+		if !ok {
+			probableUnknownVars[name] = true
+		}
+	}
+
+	// Add the remaining variables to the definitions with missing values
+	// as "<no value>" indicating that the variable is not available in the data
+	for name := range probableUnknownVars {
+		variables[name] = `"<no value>"`
+	}
+
+	return generateVariableDefinitions(variables), probableUnknownVars, nil
+}
+
+type ProcessingResult struct {
+	Template string
+	Data     map[string]interface{}
+	// UnknownVars is the set of possible unknown variables exptracted using regex
+	UnknownVars map[string]bool
+}
+
+// PreProcessTemplateAndData prepares a template string and struct data for Go template execution.
+//
+//	Input:  "$receiver has $rule_name in $status state"
+//	Output: "{{ $receiver := .Receiver }}...{{ $receiver }} has {{ $rule_name }} in {{ $status }} state"
+func PreProcessTemplateAndData(tmpl string, data any) (*ProcessingResult, error) {
+	// Handle empty template
+	unknownVars := make(map[string]bool)
+	if tmpl == "" {
+		result, err := prepareDataForTemplating(data)
+		if err != nil {
+			return nil, err
+		}
+		return &ProcessingResult{Data: result, UnknownVars: unknownVars}, nil
+	}
+
+	// Build variable definitions: known struct fields + fallback empty-string declarations
+	definitions, unknownVars, err := buildVariableDefinitions(tmpl, data)
+	if err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to build template definitions")
+	}
+
+	// Attach definitions prefix so WrapDollarVariables can parse the AST without "undefined variable" errors.
+	finalTmpl := definitions + tmpl
+
+	// Call WrapDollarVariables to transform bare $variable references to go-text-template format
+	// with {{ $variable }} syntax from $variable syntax
+	wrappedTmpl, err := WrapDollarVariables(finalTmpl)
+	if err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to prepare template for templating")
+	}
+
+	// Convert struct to map using mapstructure to be used for template execution
+	result, err := prepareDataForTemplating(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ProcessingResult{Template: wrappedTmpl, Data: result, UnknownVars: unknownVars}, nil
+}

pkg/alertmanager/alertmanagertemplate/preprocessor_test.go

@@ -0,0 +1,316 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/alertmanager/template"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExtractFieldMappings(t *testing.T) {
+	// Struct with various field types to test extraction logic
+	type TestStruct struct {
+		Name       string    `json:"name"`
+		Status     string    `json:"status"`
+		Count      int       `json:"count"`
+		IsActive   bool      `json:"is_active"`
+		CreatedAt  time.Time `json:"created_at"` // time.Time allowed
+		Items      []string  `json:"items"`      // slice skipped
+		unexported string    // unexported skipped (no tag needed)
+		NoTag      string    // no json tag skipped
+		SkippedTag string    `json:"-"` // json:"-" skipped
+	}
+
+	testCases := []struct {
+		name     string
+		data     any
+		expected []fieldMapping
+	}{
+		{
+			name: "struct with mixed field types",
+			data: TestStruct{Name: "test", Count: 5, unexported: ""},
+			expected: []fieldMapping{
+				{VarName: "name", FieldName: "Name"},
+				{VarName: "status", FieldName: "Status"},
+				{VarName: "count", FieldName: "Count"},
+				{VarName: "is_active", FieldName: "IsActive"},
+				{VarName: "created_at", FieldName: "CreatedAt"},
+			},
+		},
+		{
+			name:     "nil data",
+			data:     nil,
+			expected: nil,
+		},
+		{
+			name:     "non-struct type",
+			data:     "string",
+			expected: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := extractFieldMappings(tc.data)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestBuildVariableDefinitions(t *testing.T) {
+	testCases := []struct {
+		name         string
+		tmpl         string
+		data         any
+		expectedVars []string // substrings that must appear in result
+		expectError  bool
+	}{
+		{
+			name: "empty template still returns struct field definitions",
+			tmpl: "",
+			data: &NotificationTemplateData{Receiver: "test"},
+			expectedVars: []string{
+				"{{ $receiver := .receiver }}",
+				"{{ $status := .status }}",
+			},
+		},
+		{
+			name: "mix of known and unknown vars",
+			tmpl: "$rule_name: $custom_label",
+			data: &AlertData{AlertName: "test", Status: "ok", Severity: "critical"},
+			expectedVars: []string{
+				"{{ $rule_name := .rule_name }}",
+				"{{ $status := .status }}",
+				"{{ $severity := .severity }}",
+				`{{ $custom_label := "<no value>" }}`,
+			},
+		},
+		{
+			name: "nested fields definitions coming from NotificationTemplateData",
+			tmpl: "$severity for $service",
+			data: &NotificationTemplateData{Labels: template.KV{
+				"severity": "critical",
+				"service":  "test",
+			}},
+			expectedVars: []string{
+				"{{ $severity := index .labels \"severity\" }}",
+				"{{ $service := index .labels \"service\" }}",
+			},
+		},
+		{
+			name: "nested fields definitions coming from AlertData",
+			tmpl: "$severity for $service",
+			data: &AlertData{Labels: template.KV{
+				"severity": "critical",
+				"service":  "test",
+			}},
+			expectedVars: []string{
+				"{{ $severity := index .labels \"severity\" }}",
+				"{{ $service := index .labels \"service\" }}",
+			},
+		},
+		{
+			name:        "invalid template syntax returns error",
+			tmpl:        "{{invalid",
+			data:        &NotificationTemplateData{},
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, _, err := buildVariableDefinitions(tc.tmpl, tc.data)
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			if len(tc.expectedVars) == 0 {
+				require.Empty(t, result)
+				return
+			}
+			for _, expected := range tc.expectedVars {
+				require.Contains(t, result, expected)
+			}
+		})
+	}
+}
+
+func TestPreProcessTemplateAndData(t *testing.T) {
+	testCases := []struct {
+		name                     string
+		tmpl                     string
+		data                     any
+		expectedTemplateContains []string
+		expectedData             map[string]any
+		expectedUnknownVars      map[string]bool
+		expectError              bool
+	}{
+		{
+			name: "NotificationTemplateData with dollar variables",
+			tmpl: "[$status] $rule_name (ID: $rule_id) - Firing: $total_firing, Resolved: $total_resolved, Severity: $severity",
+			data: &NotificationTemplateData{
+				Receiver:  "pagerduty",
+				Status:    "firing",
+				AlertName: "HighMemory",
+				RuleID:    "rule-123",
+				Labels: template.KV{
+					"severity": "critical",
+				},
+				TotalFiring:   3,
+				TotalResolved: 1,
+			},
+			expectedTemplateContains: []string{
+				"{{$status := .status}}",
+				"{{$rule_name := .rule_name}}",
+				"{{$rule_id := .rule_id}}",
+				"{{$total_firing := .total_firing}}",
+				"{{$total_resolved := .total_resolved}}",
+				"{{$severity := index .labels \"severity\"}}",
+				"[{{ .status }}] {{ .rule_name }} (ID: {{ .rule_id }}) - Firing: {{ .total_firing }}, Resolved: {{ .total_resolved }}",
+			},
+			expectedData: map[string]any{
+				"status":         "firing",
+				"rule_name":      "HighMemory",
+				"rule_id":        "rule-123",
+				"total_firing":   3,
+				"total_resolved": 1,
+				"severity":       "critical",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "AlertData with dollar variables",
+			tmpl: "$rule_name: Value $value exceeded $threshold (Status: $status, Severity: $severity, Description: $description)",
+			data: &AlertData{
+				Receiver:  "webhook",
+				Status:    "resolved",
+				AlertName: "DiskFull",
+				RuleID:    "disk-001",
+				Severity:  "warning",
+				Annotations: template.KV{
+					"description": "Disk full and cannot be written to",
+				},
+				Value:      "85%",
+				Threshold:  "80%",
+				IsFiring:   false,
+				IsResolved: true,
+			},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				"{{$value := .value}}",
+				"{{$threshold := .threshold}}",
+				"{{$status := .status}}",
+				"{{$severity := .severity}}",
+				"{{$description := index .annotations \"description\"}}",
+				"{{ .rule_name }}: Value {{ .value }} exceeded {{ .threshold }} (Status: {{ .status }}, Severity: {{ .severity }}, Description: {{ .description }})",
+			},
+			expectedData: map[string]any{
+				"status":      "resolved",
+				"rule_name":   "DiskFull",
+				"rule_id":     "disk-001",
+				"severity":    "warning",
+				"value":       "85%",
+				"threshold":   "80%",
+				"description": "Disk full and cannot be written to",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "mixed dollar and dot notation with both labels and annotations",
+			tmpl: "Alert $rule_name has {{.total_firing}} firing alerts",
+			data: &NotificationTemplateData{
+				AlertName:   "HighCPU",
+				TotalFiring: 5,
+				Labels: template.KV{
+					"value": "<MASKED VALUE>",
+				},
+				Annotations: template.KV{
+					"value": "85%",
+				},
+			},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				"{{$value := index .labels \"value\"}}",
+				"Alert {{ .rule_name }} has {{.total_firing}} firing alerts",
+			},
+			expectedData: map[string]any{
+				"rule_name":    "HighCPU",
+				"total_firing": 5,
+				"value":        "<MASKED VALUE>",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "empty template",
+			tmpl: "",
+			data: &NotificationTemplateData{Receiver: "slack"},
+		},
+		{
+			name:        "invalid template syntax",
+			tmpl:        "{{invalid",
+			data:        &NotificationTemplateData{},
+			expectError: true,
+		},
+		{
+			name: "unknown dollar var in text renders empty",
+			tmpl: "alert $custom_note fired",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				`{{$custom_note := "<no value>"}}`,
+				"alert {{ .custom_note }} fired",
+			},
+			expectedUnknownVars: map[string]bool{"custom_note": true},
+		},
+		{
+			name: "unknown dollar var in action block renders empty",
+			tmpl: "alert {{ $custom_note }} fired",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				`{{$custom_note := "<no value>"}}`,
+				`alert {{$custom_note}} fired`,
+			},
+			expectedUnknownVars: map[string]bool{"custom_note": true},
+		},
+		{
+			name: "mix of known and unknown vars",
+			tmpl: "$rule_name: $custom_label",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				`{{$custom_label := "<no value>"}}`,
+				"{{ .rule_name }}: {{ .custom_label }}",
+			},
+			expectedData:        map[string]any{"rule_name": "HighCPU"},
+			expectedUnknownVars: map[string]bool{"custom_label": true},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := PreProcessTemplateAndData(tc.tmpl, tc.data)
+
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			if tc.tmpl == "" {
+				require.Equal(t, "", result.Template)
+				return
+			}
+
+			for _, substr := range tc.expectedTemplateContains {
+				require.Contains(t, result.Template, substr)
+			}
+			for k, v := range tc.expectedData {
+				require.Equal(t, v, result.Data[k])
+			}
+			if tc.expectedUnknownVars != nil {
+				require.Equal(t, tc.expectedUnknownVars, result.UnknownVars)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/types.go

@@ -0,0 +1,86 @@
+package alertmanagertemplate
+
+import (
+	"time"
+
+	"github.com/prometheus/alertmanager/template"
+)
+
+// TemplateInput carries the title/body templates
+// and their defaults to apply in case the custom templates
+// are result in empty strings.
+type TemplateInput struct {
+	TitleTemplate        string
+	BodyTemplate         string
+	DefaultTitleTemplate string
+	DefaultBodyTemplate  string
+}
+
+// ExpandedTemplates is the result of ExpandTemplates.
+type ExpandedTemplates struct {
+	Title       string
+	Body        string
+	MissingVars map[string]bool // union of unknown vars from title + body templates
+}
+
+// AlertData holds per-alert data used when expanding body templates
+type AlertData struct {
+	Receiver     string      `json:"receiver" mapstructure:"receiver"`
+	Status       string      `json:"status" mapstructure:"status"`
+	Labels       template.KV `json:"labels" mapstructure:"labels"`
+	Annotations  template.KV `json:"annotations" mapstructure:"annotations"`
+	StartsAt     time.Time   `json:"starts_at" mapstructure:"starts_at"`
+	EndsAt       time.Time   `json:"ends_at" mapstructure:"ends_at"`
+	GeneratorURL string      `json:"generator_url" mapstructure:"generator_url"`
+	Fingerprint  string      `json:"fingerprint" mapstructure:"fingerprint"`
+
+	// Convenience fields extracted from well-known labels/annotations.
+	AlertName string `json:"rule_name" mapstructure:"rule_name"`
+	RuleID    string `json:"rule_id" mapstructure:"rule_id"`
+	RuleLink  string `json:"rule_link" mapstructure:"rule_link"`
+	Severity  string `json:"severity" mapstructure:"severity"`
+
+	// Alert internal data fields
+	Value     string `json:"value" mapstructure:"value"`
+	Threshold string `json:"threshold" mapstructure:"threshold"`
+	CompareOp string `json:"compare_op" mapstructure:"compare_op"`
+	MatchType string `json:"match_type" mapstructure:"match_type"`
+
+	// Link annotations added by the rule evaluator.
+	LogLink   string `json:"log_link" mapstructure:"log_link"`
+	TraceLink string `json:"trace_link" mapstructure:"trace_link"`
+
+	// Status booleans for easy conditional templating.
+	IsFiring      bool `json:"is_firing" mapstructure:"is_firing"`
+	IsResolved    bool `json:"is_resolved" mapstructure:"is_resolved"`
+	IsMissingData bool `json:"is_missing_data" mapstructure:"is_missing_data"`
+	IsRecovering  bool `json:"is_recovering" mapstructure:"is_recovering"`
+}
+
+// NotificationTemplateData is the top-level data struct provided to custom templates.
+type NotificationTemplateData struct {
+	Receiver string `json:"receiver" mapstructure:"receiver"`
+	Status   string `json:"status" mapstructure:"status"`
+
+	// Convenience fields for title templates.
+	AlertName     string `json:"rule_name" mapstructure:"rule_name"`
+	RuleID        string `json:"rule_id" mapstructure:"rule_id"`
+	RuleLink      string `json:"rule_link" mapstructure:"rule_link"`
+	TotalFiring   int    `json:"total_firing" mapstructure:"total_firing"`
+	TotalResolved int    `json:"total_resolved" mapstructure:"total_resolved"`
+
+	// Per-alert data, also available as filtered sub-slices.
+	Alerts []AlertData `json:"-" mapstructure:"-"`
+
+	// Cross-alert aggregates, computed as intersection across all alerts.
+	GroupLabels       template.KV `json:"group_labels" mapstructure:"group_labels"`
+	CommonLabels      template.KV `json:"common_labels" mapstructure:"common_labels"`
+	CommonAnnotations template.KV `json:"common_annotations" mapstructure:"common_annotations"`
+	ExternalURL       string      `json:"external_url" mapstructure:"external_url"`
+	// Labels and Annotations that are collection of labels
+	// and annotations from all alerts, it includes only the common labels and annotations
+	// and for non-common labels and annotations, it picks some first few labels/annotations
+	// and joins them with ", " to avoid blank values in the template
+	Labels      template.KV `json:"labels" mapstructure:"labels"`
+	Annotations template.KV `json:"annotations" mapstructure:"annotations"`
+}

pkg/alertmanager/alertmanagertemplate/utils.go

@@ -0,0 +1,212 @@
+package alertmanagertemplate
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+	"text/template/parse"
+
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+)
+
+// maxAggregatedValues is the maximum number of unique values to include
+// when aggregating non-common label/annotation values across alerts.
+const maxAggregatedValues = 5
+
+// bareVariableRegex matches bare $variable references including dotted paths like $service.name.
+var bareVariableRegex = regexp.MustCompile(`\$(\w+(?:\.\w+)*)`)
+
+// bareVariableRegexFirstSeg matches only the base $variable name, stopping before any dotted path.
+// e.g. "$labels.severity" matches "$labels", "$name" matches "$name".
+var bareVariableRegexFirstSeg = regexp.MustCompile(`\$\w+`)
+
+// WrapDollarVariables wraps bare $variable references in Go template syntax.
+// Example transformations:
+//   - "$name is $status" -> "{{ $name }} is {{ $status }}"
+//   - "$labels.severity" -> "{{ index .labels \"severity\" }}"
+//   - "$labels.http.status" -> "{{ index .labels \"http.status\" }}"
+//   - "$annotations.summary" -> "{{ index .annotations \"summary\" }}"
+//   - "$service.name" -> "{{ index . \"service.name\" }}"
+//   - "$name is {{ .Status }}" -> "{{ $name }} is {{ .Status }}"
+func WrapDollarVariables(src string) (string, error) {
+	if src == "" {
+		return src, nil
+	}
+
+	funcMap := alertmanagertypes.AdditionalFuncMap()
+	// Create a new parse.Tree directly
+	tree := parse.New("template")
+	tree.Mode = parse.SkipFuncCheck
+
+	// Parse the template
+	_, err := tree.Parse(src, "{{", "}}", make(map[string]*parse.Tree), funcMap)
+	if err != nil {
+		return "", err
+	}
+
+	// Walk the AST and transform TextNodes
+	walkAndWrapTextNodes(tree.Root)
+
+	// Return the reassembled template
+	return tree.Root.String(), nil
+}
+
+// walkAndWrapTextNodes recursively walks the parse tree and wraps bare $variable
+func walkAndWrapTextNodes(node parse.Node) {
+	if reflect.ValueOf(node).IsNil() {
+		return
+	}
+
+	switch n := node.(type) {
+	case *parse.ListNode:
+		// Recurse into all child nodes
+		if n.Nodes != nil {
+			for _, child := range n.Nodes {
+				walkAndWrapTextNodes(child)
+			}
+		}
+
+	case *parse.TextNode:
+		// Transform $variable based on its pattern
+		n.Text = bareVariableRegex.ReplaceAllFunc(n.Text, func(match []byte) []byte {
+			// Extract variable name without the $
+			varName := string(match[1:])
+
+			// Check if variable contains dots
+			if strings.Contains(varName, ".") {
+				// Check for reserved prefixes: labels.* or annotations.*
+				if strings.HasPrefix(varName, "labels.") {
+					key := strings.TrimPrefix(varName, "labels.")
+					return []byte(fmt.Sprintf(`{{ index .labels "%s" }}`, key))
+				}
+				if strings.HasPrefix(varName, "annotations.") {
+					key := strings.TrimPrefix(varName, "annotations.")
+					return []byte(fmt.Sprintf(`{{ index .annotations "%s" }}`, key))
+				}
+				// Other dotted variables: index into root context
+				return []byte(fmt.Sprintf(`{{ index . "%s" }}`, varName))
+			}
+
+			// Simple variables: use dot notation to directly access the field
+			// without raising any error due to missing variables
+			return []byte(fmt.Sprintf("{{ .%s }}", varName))
+		})
+
+	case *parse.IfNode:
+		// Recurse into both branches
+		walkAndWrapTextNodes(n.List)
+		walkAndWrapTextNodes(n.ElseList)
+
+	case *parse.RangeNode:
+		// Recurse into both branches
+		walkAndWrapTextNodes(n.List)
+		walkAndWrapTextNodes(n.ElseList)
+
+		// All other node types (ActionNode, PipeNode, VariableNode, etc.) are already
+		// inside {{ }} action blocks and don't need transformation
+
+		// Support for `with` can be added later when we start supporting it in editor block
+	}
+}
+
+// ExtractUsedVariables returns the set of all $variable referenced in template
+// — text nodes, action blocks, branch conditions, and loop declarations — regardless of scope.
+// After finding all variables we find the ones which are not part of our alert data and handle them so
+// Go-text-template parser doesn't rejects undefined $variables
+func ExtractUsedVariables(src string) (map[string]bool, error) {
+	if src == "" {
+		return map[string]bool{}, nil
+	}
+
+	// Regex-scan raw template string to collect all $var base names.
+	// bareVariableRegexFirstSeg stops before dots, so "$labels.severity" yields "$labels".
+	used := make(map[string]bool)
+	for _, m := range bareVariableRegexFirstSeg.FindAll([]byte(src), -1) {
+		used[string(m[1:])] = true // strip leading "$"
+	}
+
+	// Build a preamble that pre-declares every found variable.
+	// This prevents "undefined variable" parse errors for $vars used in action
+	// blocks while still letting genuine syntax errors propagate.
+	var preamble strings.Builder
+	for name := range used {
+		fmt.Fprintf(&preamble, `{{$%s := ""}}`, name)
+	}
+
+	// Validate template syntax.
+	funcMap := alertmanagertypes.AdditionalFuncMap()
+	tree := parse.New("template")
+	tree.Mode = parse.SkipFuncCheck
+	if _, err := tree.Parse(preamble.String()+src, "{{", "}}", make(map[string]*parse.Tree), funcMap); err != nil {
+		return nil, err
+	}
+
+	return used, nil
+}
+
+// AggregateKV aggregates key-value pairs (labels or annotations) from all alerts into a single template.KV
+// the result is used to populate the labels and annotations in the notification template data.
+// this is done to avoid blank values in the template when labels and annotations used are not common throughout the alerts
+func AggregateKV(alerts []*types.Alert, extractFn func(*types.Alert) model.LabelSet) template.KV {
+	// track unique values per key in order of first appearance
+	valuesPerKey := make(map[string][]string)
+	// track which values have been seen for deduplication
+	seenValues := make(map[string]map[string]bool)
+
+	for _, alert := range alerts {
+		kvPairs := extractFn(alert)
+		for k, v := range kvPairs {
+			key := string(k)
+			value := string(v)
+
+			if seenValues[key] == nil {
+				seenValues[key] = make(map[string]bool)
+			}
+
+			// only add if not already seen and under the limit of maxAggregatedValues
+			if !seenValues[key][value] && len(valuesPerKey[key]) < maxAggregatedValues {
+				seenValues[key][value] = true
+				valuesPerKey[key] = append(valuesPerKey[key], value)
+			}
+		}
+	}
+
+	// build the result by joining values
+	result := make(template.KV, len(valuesPerKey))
+	for key, values := range valuesPerKey {
+		result[key] = strings.Join(values, ", ")
+	}
+
+	return result
+}
+
+// extractCommonKV returns the intersection of key-value pairs across all alerts.
+// A key/value pair is included only if it appears identically on every alert.
+func extractCommonKV(alerts []*types.Alert, extractFn func(*types.Alert) model.LabelSet) template.KV {
+	if len(alerts) == 0 {
+		return template.KV{}
+	}
+
+	common := make(template.KV, len(extractFn(alerts[0])))
+	for k, v := range extractFn(alerts[0]) {
+		common[string(k)] = string(v)
+	}
+
+	for _, a := range alerts[1:] {
+		kv := extractFn(a)
+		for k := range common {
+			if string(kv[model.LabelName(k)]) != common[k] {
+				delete(common, k)
+			}
+		}
+		if len(common) == 0 {
+			break
+		}
+	}
+
+	return common
+}

pkg/alertmanager/alertmanagertemplate/utils_test.go

@@ -0,0 +1,348 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWrapBareVars(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    string
+		expectError bool
+	}{
+		{
+			name:     "mixed variables with actions",
+			input:    "$name is {{.Status}}",
+			expected: "{{ .name }} is {{.Status}}",
+		},
+		{
+			name: "nested variables in range",
+			input: `{{range .items}}
+				$title
+				{{end}}`,
+			expected: `{{range .items}}
+				{{ .title }}
+				{{end}}`,
+		},
+		{
+			name:     "nested variables in if else",
+			input:    "{{if .ok}}$a{{else}}$b{{end}}",
+			expected: "{{if .ok}}{{ .a }}{{else}}{{ .b }}{{end}}",
+		},
+		// Labels prefix: index into .labels map
+		{
+			name:     "labels variables prefix simple",
+			input:    "$labels.service",
+			expected: `{{ index .labels "service" }}`,
+		},
+		{
+			name:     "labels variables prefix nested with multiple dots",
+			input:    "$labels.http.status",
+			expected: `{{ index .labels "http.status" }}`,
+		},
+		{
+			name:     "multiple labels variables simple and nested",
+			input:    "$labels.service and $labels.instance.id",
+			expected: `{{ index .labels "service" }} and {{ index .labels "instance.id" }}`,
+		},
+		// Annotations prefix: index into .annotations map
+		{
+			name:     "annotations variables prefix simple",
+			input:    "$annotations.summary",
+			expected: `{{ index .annotations "summary" }}`,
+		},
+		{
+			name:     "annotations variables prefix nested with multiple dots",
+			input:    "$annotations.alert.url",
+			expected: `{{ index .annotations "alert.url" }}`,
+		},
+		// Other dotted paths: index into root context
+		{
+			name:     "other variables with multiple dots",
+			input:    "$service.name",
+			expected: `{{ index . "service.name" }}`,
+		},
+		{
+			name:     "other variables with multiple dots nested",
+			input:    "$http.status.code",
+			expected: `{{ index . "http.status.code" }}`,
+		},
+		// Hybrid: all types combined
+		{
+			name:     "hybrid - all variables types",
+			input:    "Alert: $alert_name Labels: $labels.severity Annotations: $annotations.desc Service: $service.name Count: $error_count",
+			expected: `Alert: {{ .alert_name }} Labels: {{ index .labels "severity" }} Annotations: {{ index .annotations "desc" }} Service: {{ index . "service.name" }} Count: {{ .error_count }}`,
+		},
+		{
+			name:     "already wrapped should not be changed",
+			input:    "{{$status := .status}}{{.name}} is {{$status | toUpper}}",
+			expected: "{{$status := .status}}{{.name}} is {{$status | toUpper}}",
+		},
+		{
+			name:     "no variables should not be changed",
+			input:    "Hello world",
+			expected: "Hello world",
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "deeply nested",
+			input:    "{{range .items}}{{if .ok}}$deep{{end}}{{end}}",
+			expected: "{{range .items}}{{if .ok}}{{ .deep }}{{end}}{{end}}",
+		},
+		{
+			name: "complex example",
+			input: `Hello $name, your score is $score.
+				{{if .isAdmin}}
+					Welcome back $name, you have {{.unreadCount}} messages.
+				{{end}}`,
+			expected: `Hello {{ .name }}, your score is {{ .score }}.
+				{{if .isAdmin}}
+					Welcome back {{ .name }}, you have {{.unreadCount}} messages.
+				{{end}}`,
+		},
+		{
+			name:     "with custom function",
+			input:    "$name triggered at {{urlescape .url}}",
+			expected: "{{ .name }} triggered at {{urlescape .url}}",
+		},
+		{
+			name:        "invalid template",
+			input:       "{{invalid",
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := WrapDollarVariables(tc.input)
+
+			if tc.expectError {
+				require.Error(t, err, "should error on invalid template syntax")
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestExtractUsedVariables(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    map[string]bool
+		expectError bool
+	}{
+		{
+			name:     "simple usage in text",
+			input:    "$name is $status",
+			expected: map[string]bool{"name": true, "status": true},
+		},
+		{
+			name:     "declared in action block",
+			input:    "{{ $name := .name }}",
+			expected: map[string]bool{"name": true},
+		},
+		{
+			name:     "range loop vars",
+			input:    "{{ range $i, $v := .items }}{{ end }}",
+			expected: map[string]bool{"i": true, "v": true},
+		},
+		{
+			name:     "mixed text and action",
+			input:    "$x and {{ $y }}",
+			expected: map[string]bool{"x": true, "y": true},
+		},
+		{
+			name:     "dotted path in text extracts base only",
+			input:    "$labels.severity",
+			expected: map[string]bool{"labels": true},
+		},
+		{
+			name:     "nested if else",
+			input:    "{{ if .ok }}{{ $a }}{{ else }}{{ $b }}{{ end }}",
+			expected: map[string]bool{"a": true, "b": true},
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: map[string]bool{},
+		},
+		{
+			name:     "no variables",
+			input:    "Hello world",
+			expected: map[string]bool{},
+		},
+		{
+			name:        "invalid template returns error",
+			input:       "{{invalid",
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := ExtractUsedVariables(tc.input)
+
+			if tc.expectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestAggregateKV(t *testing.T) {
+	extractLabels := func(a *types.Alert) model.LabelSet { return a.Labels }
+
+	testCases := []struct {
+		name      string
+		alerts    []*types.Alert
+		extractFn func(*types.Alert) model.LabelSet
+		expected  template.KV
+	}{
+		{
+			name:      "empty alerts slice",
+			alerts:    []*types.Alert{},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "single alert",
+			alerts: []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels: model.LabelSet{
+							"env":     "production",
+							"service": "backend",
+						},
+					},
+				},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"env":     "production",
+				"service": "backend",
+			},
+		},
+		{
+			name: "varying values with duplicates deduped",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "backend"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "frontend"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "api"}}},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"env":     "production",
+				"service": "backend, api, frontend",
+			},
+		},
+		{
+			name: "more than 5 unique values truncates to 5",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc1"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc2"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc3"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc4"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc5"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc6"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc7"}}},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"service": "svc1, svc2, svc3, svc4, svc5",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := AggregateKV(tc.alerts, tc.extractFn)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestExtractCommonKV(t *testing.T) {
+	extractLabels := func(a *types.Alert) model.LabelSet { return a.Labels }
+	extractAnnotations := func(a *types.Alert) model.LabelSet { return a.Annotations }
+
+	testCases := []struct {
+		name      string
+		alerts    []*types.Alert
+		extractFn func(*types.Alert) model.LabelSet
+		expected  template.KV
+	}{
+		{
+			name:      "empty alerts slice",
+			alerts:    []*types.Alert{},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "single alert returns all labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "api"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod", "service": "api"},
+		},
+		{
+			name: "multiple alerts with fully common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "region": "us-east"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "region": "us-east"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod", "region": "us-east"},
+		},
+		{
+			name: "multiple alerts with partially common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "worker"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod"},
+		},
+		{
+			name: "multiple alerts with no common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "worker"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "annotations extract common annotations",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Annotations: model.LabelSet{"summary": "high cpu", "runbook": "http://x"}}},
+				{Alert: model.Alert{Annotations: model.LabelSet{"summary": "high cpu", "runbook": "http://y"}}},
+			},
+			extractFn: extractAnnotations,
+			expected:  template.KV{"summary": "high cpu"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := extractCommonKV(tc.alerts, tc.extractFn)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}

pkg/errors/errors.go

@@ -147,7 +147,7 @@ func Ast(cause error, typ typ) bool {
 	return t == typ
 }
 
-// Ast checks if the provided error matches the specified custom error code.
+// Asc checks if the provided error matches the specified custom error code.
 func Asc(cause error, code Code) bool {
 	_, c, _, _, _, _ := Unwrapb(cause)
 

pkg/types/alertmanagertypes/template.go

@@ -11,19 +11,22 @@ import (
 	alertmanagertemplate "github.com/prometheus/alertmanager/template"
 )
 
+func AdditionalFuncMap() tmpltext.FuncMap {
+	return tmpltext.FuncMap{
+		// urlescape escapes the string for use in a URL query parameter.
+		// It returns tmplhtml.HTML to prevent the template engine from escaping the already escaped string.
+		// url.QueryEscape escapes spaces as "+", and html/template escapes "+" as "+" if tmplhtml.HTML is not used.
+		"urlescape": func(value string) tmplhtml.HTML {
+			return tmplhtml.HTML(url.QueryEscape(value))
+		},
+	}
+}
+
 // customTemplateOption returns an Option that adds custom functions to the template.
 func customTemplateOption() alertmanagertemplate.Option {
 	return func(text *tmpltext.Template, html *tmplhtml.Template) {
-		funcs := tmpltext.FuncMap{
-			// urlescape escapes the string for use in a URL query parameter.
-			// It returns tmplhtml.HTML to prevent the template engine from escaping the already escaped string.
-			// url.QueryEscape escapes spaces as "+", and html/template escapes "+" as "+" if tmplhtml.HTML is not used.
-			"urlescape": func(value string) tmplhtml.HTML {
-				return tmplhtml.HTML(url.QueryEscape(value))
-			},
-		}
-		text.Funcs(funcs)
-		html.Funcs(funcs)
+		text.Funcs(AdditionalFuncMap())
+		html.Funcs(AdditionalFuncMap())
 	}
 }
 

pkg/types/ruletypes/constants.go

@@ -9,4 +9,19 @@ const (
 	LabelSeverityName     = "severity"
 	LabelLastSeen         = "lastSeen"
 	LabelRuleId           = "ruleId"
+	LabelRuleSource       = "ruleSource"
+	LabelNoData           = "nodata"
+	LabelTestAlert        = "testalert"
+	LabelAlertName        = "alertname"
+)
+
+const (
+	AnnotationRelatedLogs   = "related_logs"
+	AnnotationRelatedTraces = "related_traces"
+	AnnotationTitleTemplate = "title_template"
+	AnnotationBodyTemplate  = "body_template"
+	AnnotationValue         = "value"
+	AnnotationThreshold     = "threshold"
+	AnnotationCompareOp     = "compare_op"
+	AnnotationMatchType     = "match_type"
 )