test: added alert manager notifier e2e

ee/query-service/rules/anomaly.go

@@ -410,8 +410,23 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 
 		annotations := make(labels.Labels, 0, len(r.Annotations().Map()))
 		for name, value := range r.Annotations().Map() {
+			// no need to expand custom templating annotations — they get expanded in the notifier layer
+			if ruletypes.IsCustomTemplatingAnnotation(name) {
+				annotations = append(annotations, labels.Label{Name: name, Value: value})
+				continue
+			}
 			annotations = append(annotations, labels.Label{Name: name, Value: expand(value)})
 		}
+		// Add values to be used in notifier layer for notification templates
+		annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationValue, Value: value})
+		annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationThresholdValue, Value: threshold})
+		annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationCompareOp, Value: smpl.CompareOp.String()})
+		annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationMatchType, Value: smpl.MatchType.String()})
+
+		if smpl.IsRecovering {
+			lb.Set(ruletypes.LabelIsRecovering, "true")
+		}
+
 		if smpl.IsMissing {
 			lb.Set(labels.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(labels.NoDataLabel, "true")

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/coreos/go-oidc/v3 v3.17.0
 	github.com/dgraph-io/ristretto/v2 v2.3.0
 	github.com/dustin/go-humanize v1.0.1
+	github.com/emersion/go-smtp v0.24.0
 	github.com/gin-gonic/gin v1.11.0
 	github.com/go-co-op/gocron v1.30.1
 	github.com/go-openapi/runtime v0.29.2
@@ -63,6 +64,7 @@ require (
 	github.com/uptrace/bun/dialect/pgdialect v1.2.9
 	github.com/uptrace/bun/dialect/sqlitedialect v1.2.9
 	github.com/uptrace/bun/extra/bunotel v1.2.9
+	github.com/yuin/goldmark v1.7.16
 	go.opentelemetry.io/collector/confmap v1.51.0
 	go.opentelemetry.io/collector/otelcol v0.144.0
 	go.opentelemetry.io/collector/pdata v1.51.0
@@ -111,6 +113,7 @@ require (
 	github.com/bytedance/sonic v1.14.1 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
+	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/go-openapi/swag/cmdutils v0.25.4 // indirect
 	github.com/go-openapi/swag/conv v0.25.4 // indirect
@@ -164,7 +167,7 @@ require (
 	github.com/ClickHouse/ch-go v0.67.0 // indirect
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Yiling-J/theine-go v0.6.2 // indirect
-	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
+	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beevik/etree v1.1.0 // indirect

go.sum

@@ -1144,6 +1144,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE=
+github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A=

pkg/alertmanager/alertmanagernotify/email/email.go

@@ -0,0 +1,538 @@
+// Copyright 2019 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package email
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"log/slog"
+	"math/rand"
+	"mime"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net"
+	"net/mail"
+	"net/smtp"
+	"net/textproto"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "email"
+)
+
+// Email implements a Notifier for email notifications.
+type Email struct {
+	conf      *config.EmailConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	hostname  string
+	processor alertmanagertypes.NotificationProcessor
+}
+
+var errNoAuthUsernameConfigured = errors.NewInternalf(errors.CodeInternal, "no auth username configured")
+
+// New returns a new Email notifier.
+func New(c *config.EmailConfig, t *template.Template, l *slog.Logger, proc alertmanagertypes.NotificationProcessor) *Email {
+	if _, ok := c.Headers["Subject"]; !ok {
+		c.Headers["Subject"] = config.DefaultEmailSubject
+	}
+	if _, ok := c.Headers["To"]; !ok {
+		c.Headers["To"] = c.To
+	}
+	if _, ok := c.Headers["From"]; !ok {
+		c.Headers["From"] = c.From
+	}
+
+	h, err := os.Hostname()
+	// If we can't get the hostname, we'll use localhost
+	if err != nil {
+		h = "localhost.localdomain"
+	}
+	return &Email{conf: c, tmpl: t, logger: l, hostname: h, processor: proc}
+}
+
+// auth resolves a string of authentication mechanisms.
+func (n *Email) auth(mechs string) (smtp.Auth, error) {
+	username := n.conf.AuthUsername
+
+	// If no username is set, return custom error which can be ignored if needed.
+	if strings.TrimSpace(username) == "" {
+		return nil, errNoAuthUsernameConfigured
+	}
+
+	var errs error
+	for mech := range strings.SplitSeq(mechs, " ") {
+		switch mech {
+		case "CRAM-MD5":
+			secret, secretErr := n.getAuthSecret()
+			if secretErr != nil {
+				errs = errors.Join(errs, secretErr)
+				continue
+			}
+			if secret == "" {
+				errs = errors.Join(errs, errors.NewInternalf(errors.CodeInternal, "missing secret for CRAM-MD5 auth mechanism"))
+				continue
+			}
+			return smtp.CRAMMD5Auth(username, secret), nil
+
+		case "PLAIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				errs = errors.Join(errs, passwordErr)
+				continue
+			}
+			if password == "" {
+				errs = errors.Join(errs, errors.NewInternalf(errors.CodeInternal, "missing password for PLAIN auth mechanism"))
+				continue
+			}
+			return smtp.PlainAuth(n.conf.AuthIdentity, username, password, n.conf.Smarthost.Host), nil
+		case "LOGIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				errs = errors.Join(errs, passwordErr)
+				continue
+			}
+			if password == "" {
+				errs = errors.Join(errs, errors.NewInternalf(errors.CodeInternal, "missing password for LOGIN auth mechanism"))
+				continue
+			}
+			return LoginAuth(username, password), nil
+		default:
+			errs = errors.Join(errs, errors.NewInternalf(errors.CodeUnsupported, "unknown auth mechanism: %s", mech))
+		}
+	}
+	return nil, errs
+}
+
+// Notify implements the Notifier interface.
+func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var (
+		c       *smtp.Client
+		conn    net.Conn
+		err     error
+		success = false
+	)
+	// Determine whether to use Implicit TLS
+	var useImplicitTLS bool
+	if n.conf.ForceImplicitTLS != nil {
+		useImplicitTLS = *n.conf.ForceImplicitTLS
+	} else {
+		// Default logic: port 465 uses implicit TLS (backward compatibility)
+		useImplicitTLS = n.conf.Smarthost.Port == "465"
+	}
+
+	if useImplicitTLS {
+		tlsConfig, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "parse TLS configuration")
+		}
+		if tlsConfig.ServerName == "" {
+			tlsConfig.ServerName = n.conf.Smarthost.Host
+		}
+
+		conn, err = tls.Dial("tcp", n.conf.Smarthost.String(), tlsConfig)
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "establish TLS connection to server")
+		}
+	} else {
+		var (
+			d   = net.Dialer{}
+			err error
+		)
+		conn, err = d.DialContext(ctx, "tcp", n.conf.Smarthost.String())
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "establish connection to server")
+		}
+	}
+	c, err = smtp.NewClient(conn, n.conf.Smarthost.Host)
+	if err != nil {
+		conn.Close()
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "create SMTP client")
+	}
+	defer func() {
+		// Try to clean up after ourselves but don't log anything if something has failed.
+		if err := c.Quit(); success && err != nil {
+			n.logger.WarnContext(ctx, "failed to close SMTP connection", slog.Any("err", err))
+		}
+	}()
+
+	if n.conf.Hello != "" {
+		err = c.Hello(n.conf.Hello)
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send EHLO command")
+		}
+	}
+
+	// Global Config guarantees RequireTLS is not nil.
+	if *n.conf.RequireTLS && !useImplicitTLS {
+		if ok, _ := c.Extension("STARTTLS"); !ok {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "'require_tls' is true (default) but %q does not advertise the STARTTLS extension", n.conf.Smarthost)
+		}
+
+		tlsConf, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "parse TLS configuration")
+		}
+		if tlsConf.ServerName == "" {
+			tlsConf.ServerName = n.conf.Smarthost.Host
+		}
+
+		if err := c.StartTLS(tlsConf); err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send STARTTLS command")
+		}
+	}
+
+	if ok, mech := c.Extension("AUTH"); ok {
+		auth, err := n.auth(mech)
+		if err != nil && err != errNoAuthUsernameConfigured {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "find auth mechanism")
+		} else if err == errNoAuthUsernameConfigured {
+			n.logger.DebugContext(ctx, "no auth username configured. Attempting to send email without authenticating")
+		}
+		if auth != nil {
+			if err := c.Auth(auth); err != nil {
+				return true, errors.WrapInternalf(err, errors.CodeInternal, "%T auth", auth)
+			}
+		}
+	}
+
+	var (
+		tmplErr error
+		data    = notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
+		tmpl    = notify.TmplText(n.tmpl, data, &tmplErr)
+	)
+	from := tmpl(n.conf.From)
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "execute 'from' template")
+	}
+	to := tmpl(n.conf.To)
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "execute 'to' template")
+	}
+
+	addrs, err := mail.ParseAddressList(from)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "parse 'from' addresses")
+	}
+	if len(addrs) != 1 {
+		return false, errors.NewInternalf(errors.CodeInternal, "must be exactly one 'from' address (got: %d)", len(addrs))
+	}
+	if err = c.Mail(addrs[0].Address); err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "send MAIL command")
+	}
+	addrs, err = mail.ParseAddressList(to)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "parse 'to' addresses")
+	}
+	for _, addr := range addrs {
+		if err = c.Rcpt(addr.Address); err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send RCPT command")
+		}
+	}
+
+	// Prepare the content for the email
+	title, htmlBody, err := n.prepareContent(ctx, as, data)
+	if err != nil {
+		return false, err
+	}
+	if title != "" {
+		n.conf.Headers["Subject"] = title
+	}
+
+	// Send the email headers and body.
+	message, err := c.Data()
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "send DATA command")
+	}
+	closeOnce := sync.OnceValue(func() error {
+		return message.Close()
+	})
+	// Close the message when this method exits in order to not leak resources. Even though we're calling this explicitly
+	// further down, the method may exit before then.
+	defer func() {
+		// If we try close an already-closed writer, it'll send a subsequent request to the server which is invalid.
+		_ = closeOnce()
+	}()
+
+	buffer := &bytes.Buffer{}
+	for header, t := range n.conf.Headers {
+		value, err := n.tmpl.ExecuteTextString(t, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute %q header template", header)
+		}
+		fmt.Fprintf(buffer, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", value))
+	}
+
+	if _, ok := n.conf.Headers["Message-Id"]; !ok {
+		fmt.Fprintf(buffer, "Message-Id: %s\r\n", fmt.Sprintf("<%d.%d@%s>", time.Now().UnixNano(), rand.Uint64(), n.hostname))
+	}
+
+	if n.conf.Threading.Enabled {
+		key, err := notify.ExtractGroupKey(ctx)
+		if err != nil {
+			return false, err
+		}
+		// Add threading headers. All notifications for the same alert group
+		// (identified by key hash) are threaded together.
+		threadBy := ""
+		if n.conf.Threading.ThreadByDate != "none" {
+			// ThreadByDate is 'daily':
+			// Use current date so all mails for this alert today thread together.
+			threadBy = time.Now().Format("2006-01-02")
+		}
+		keyHash := key.Hash()
+		if len(keyHash) > 16 {
+			keyHash = keyHash[:16]
+		}
+		// The thread root ID is a Message-ID that doesn't correspond to
+		// any actual email. Email clients following the (commonly used) JWZ
+		// algorithm will create a dummy container to group these messages.
+		threadRootID := fmt.Sprintf("<alert-%s-%s@alertmanager>", keyHash, threadBy)
+		fmt.Fprintf(buffer, "References: %s\r\n", threadRootID)
+		fmt.Fprintf(buffer, "In-Reply-To: %s\r\n", threadRootID)
+	}
+
+	multipartBuffer := &bytes.Buffer{}
+	multipartWriter := multipart.NewWriter(multipartBuffer)
+
+	fmt.Fprintf(buffer, "Date: %s\r\n", time.Now().Format(time.RFC1123Z))
+	fmt.Fprintf(buffer, "Content-Type: multipart/alternative;  boundary=%s\r\n", multipartWriter.Boundary())
+	fmt.Fprintf(buffer, "MIME-Version: 1.0\r\n\r\n")
+
+	// TODO: Add some useful headers here, such as URL of the alertmanager
+	// and active/resolved.
+	_, err = message.Write(buffer.Bytes())
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "write headers")
+	}
+
+	if len(n.conf.Text) > 0 {
+		// Text template
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/plain; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for text template")
+		}
+		body, err := n.tmpl.ExecuteTextString(n.conf.Text, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute text template")
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "write text part")
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "close text part")
+		}
+	}
+
+	// TODO: handle case where both custom and default body templates
+	// result in empty body, write no message in the email body
+
+	if htmlBody != "" {
+		// Html template
+		// Preferred alternative placed last per section 5.1.4 of RFC 2046
+		// https://www.ietf.org/rfc/rfc2046.txt
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/html; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for html template")
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(htmlBody))
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "write HTML part")
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "close HTML part")
+		}
+	}
+
+	err = multipartWriter.Close()
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "close multipartWriter")
+	}
+
+	_, err = message.Write(multipartBuffer.Bytes())
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "write body buffer")
+	}
+
+	// Complete the message and await response.
+	if err = closeOnce(); err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "delivery failure")
+	}
+
+	success = true
+	return false, nil
+}
+
+// prepareContent extracts custom templates from alerts, runs the notification processor,
+// and returns the resolved subject title (if any) and the HTML body for the email.
+func (n *Email) prepareContent(ctx context.Context, alerts []*types.Alert, data *template.Data) (string, string, error) {
+	// run the notification processor to get the title and body
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.processor.ProcessAlertNotification(ctx, alertmanagertypes.NotificationProcessorInput{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: n.conf.Headers["Subject"],
+		// no templating needed for email body as it will be handled with legacy templating
+		DefaultBodyTemplate: "NO_OP",
+	}, alerts, markdownrenderer.MarkdownFormatHTML)
+	if err != nil {
+		return "", "", err
+	}
+
+	title := result.Title
+
+	// If custom templated, render via the HTML layout template
+	if result.IsCustomTemplated() {
+		// Add buttons to each of the bodies if the related logs and traces links are present in annotations
+		for i := range result.Body {
+			relatedLogsLink := alerts[i].Annotations[ruletypes.AnnotationRelatedLogs]
+			relatedTracesLink := alerts[i].Annotations[ruletypes.AnnotationRelatedTraces]
+			if relatedLogsLink != "" {
+				result.Body[i] += htmlButton("View Related Logs", string(relatedLogsLink))
+			}
+			if relatedTracesLink != "" {
+				result.Body[i] += htmlButton("View Related Traces", string(relatedTracesLink))
+			}
+		}
+
+		htmlContent, renderErr := n.processor.RenderEmailNotification(ctx, emailtypes.TemplateNameAlertEmailNotification, result, alerts)
+		if renderErr == nil {
+			return title, htmlContent, nil
+		}
+		n.logger.WarnContext(ctx, "custom email template rendering failed, falling back to default", errors.Attr(renderErr))
+	}
+
+	// Default templated body: use the HTML config template if available
+	if len(n.conf.HTML) > 0 {
+		body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
+		if err != nil {
+			return "", "", errors.WrapInternalf(err, errors.CodeInternal, "execute html template")
+		}
+		return title, body, nil
+	}
+
+	// No HTML template configured, fallback to plain HTML templating
+	if result.IsCustomTemplated() {
+		var b strings.Builder
+		for _, part := range result.Body {
+			b.WriteString("<div>")
+			b.WriteString(part)
+			b.WriteString("</div>")
+		}
+		return title, b.String(), nil
+	}
+	return title, "", nil
+}
+
+func htmlButton(text, url string) string {
+	return fmt.Sprintf(`
+	<a href="%s" target="_blank" style="text-decoration: none;">
+<button style="
+    padding: 6px 16px;
+    /* Default System Font */
+    font-family: sans-serif;
+    font-size: 14px;
+    font-weight: 500;
+    line-height: 1.5;
+    /* Light Theme & Dynamic Background (Solid) */
+    color: #111827;
+    background-color: #f9fafb;
+    /* Static Outline */
+    border: 1px solid #d1d5db;
+    border-radius: 4px;
+    cursor: pointer;
+">
+  %s
+</button>
+</a>`, url, text)
+}
+
+type loginAuth struct {
+	username, password string
+}
+
+func LoginAuth(username, password string) smtp.Auth {
+	return &loginAuth{username, password}
+}
+
+func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
+	return "LOGIN", []byte{}, nil
+}
+
+// Used for AUTH LOGIN. (Maybe password should be encrypted).
+func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
+	if more {
+		switch strings.ToLower(string(fromServer)) {
+		case "username:":
+			return []byte(a.username), nil
+		case "password:":
+			return []byte(a.password), nil
+		default:
+			return nil, errors.NewInternalf(errors.CodeInternal, "unexpected server challenge")
+		}
+	}
+	return nil, nil
+}
+
+func (n *Email) getPassword() (string, error) {
+	if len(n.conf.AuthPasswordFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthPasswordFile)
+		if err != nil {
+			return "", errors.NewInternalf(errors.CodeInternal, "could not read %s: %v", n.conf.AuthPasswordFile, err)
+		}
+		return strings.TrimSpace(string(content)), nil
+	}
+	return string(n.conf.AuthPassword), nil
+}
+
+func (n *Email) getAuthSecret() (string, error) {
+	if len(n.conf.AuthSecretFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthSecretFile)
+		if err != nil {
+			return "", errors.NewInternalf(errors.CodeInternal, "could not read %s: %v", n.conf.AuthSecretFile, err)
+		}
+		return string(content), nil
+	}
+	return string(n.conf.AuthSecret), nil
+}

pkg/alertmanager/alertmanagernotify/email/testdata/auth-local.yml

@@ -0,0 +1,4 @@
+smarthost: 127.0.0.1:1026
+server: http://127.0.0.1:1081/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/auth.yml

@@ -0,0 +1,4 @@
+smarthost: maildev-auth:1025
+server: http://maildev-auth:1080/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/noauth-local.yml

@@ -0,0 +1,2 @@
+smarthost: 127.0.0.1:1025
+server: http://127.0.0.1:1080/

pkg/alertmanager/alertmanagernotify/email/testdata/noauth.yml

@@ -0,0 +1,2 @@
+smarthost: maildev-noauth:1025
+server: http://maildev-noauth:1080/

pkg/alertmanager/alertmanagernotify/msteamsv2/msteamsv2.go

@@ -1,3 +1,16 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package msteamsv2
 
 import (
@@ -11,7 +24,10 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -27,6 +43,10 @@ const (
 	colorGrey  = "Warning"
 )
 
+const (
+	Integration = "msteamsv2"
+)
+
 type Notifier struct {
 	conf         *config.MSTeamsV2Config
 	titleLink    string
@@ -36,6 +56,7 @@ type Notifier struct {
 	retrier      *notify.Retrier
 	webhookURL   *config.SecretURL
 	postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
+	processor    alertmanagertypes.NotificationProcessor
 }
 
 // https://learn.microsoft.com/en-us/connectors/teams/?tabs=text1#adaptivecarditemschema
@@ -86,8 +107,8 @@ type teamsMessage struct {
 }
 
 // New returns a new notifier that uses the Microsoft Teams Power Platform connector.
-func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
-	client, err := commoncfg.NewClientFromConfig(*c.HTTPConfig, "msteamsv2", httpOpts...)
+func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, proc alertmanagertypes.NotificationProcessor, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
 	}
@@ -101,6 +122,7 @@ func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *s
 		retrier:      &notify.Retrier{},
 		webhookURL:   c.WebhookURL,
 		postJSONFunc: notify.PostJSON,
+		processor:    proc,
 	}
 
 	return n, nil
@@ -120,25 +142,11 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		return false, err
 	}
 
-	title := tmpl(n.conf.Title)
-	if err != nil {
-		return false, err
-	}
-
 	titleLink := tmpl(n.titleLink)
 	if err != nil {
 		return false, err
 	}
 
-	alerts := types.Alerts(as...)
-	color := colorGrey
-	switch alerts.Status() {
-	case model.AlertFiring:
-		color = colorRed
-	case model.AlertResolved:
-		color = colorGreen
-	}
-
 	var url string
 	if n.conf.WebhookURL != nil {
 		url = n.conf.WebhookURL.String()
@@ -150,6 +158,11 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		url = strings.TrimSpace(string(content))
 	}
 
+	bodyBlocks, err := n.prepareContent(ctx, as)
+	if err != nil {
+		return false, err
+	}
+
 	// A message as referenced in https://learn.microsoft.com/en-us/connectors/teams/?tabs=text1%2Cdotnet#request-body-schema
 	t := teamsMessage{
 		Type: "message",
@@ -161,17 +174,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 					Schema:  "http://adaptivecards.io/schemas/adaptive-card.json",
 					Type:    "AdaptiveCard",
 					Version: "1.2",
-					Body: []Body{
-						{
-							Type:   "TextBlock",
-							Text:   title,
-							Weight: "Bolder",
-							Size:   "Medium",
-							Wrap:   true,
-							Style:  "heading",
-							Color:  color,
-						},
-					},
+					Body:    bodyBlocks,
 					Actions: []Action{
 						{
 							Type:  "Action.OpenUrl",
@@ -187,20 +190,6 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		},
 	}
 
-	// add labels and annotations to the body of all alerts
-	for _, alert := range as {
-		t.Attachments[0].Content.Body = append(t.Attachments[0].Content.Body, Body{
-			Type:   "TextBlock",
-			Text:   "Alerts",
-			Weight: "Bolder",
-			Size:   "Medium",
-			Wrap:   true,
-			Color:  color,
-		})
-
-		t.Attachments[0].Content.Body = append(t.Attachments[0].Content.Body, n.createLabelsAndAnnotationsBody(alert)...)
-	}
-
 	var payload bytes.Buffer
 	if err = json.NewEncoder(&payload).Encode(t); err != nil {
 		return false, err
@@ -220,6 +209,79 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	return shouldRetry, err
 }
 
+// prepareContent prepares the body blocks for the templated title and body.
+func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert) ([]Body, error) {
+	// run the notification processor to get the title and body
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.processor.ProcessAlertNotification(ctx, alertmanagertypes.NotificationProcessorInput{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: n.conf.Title,
+		// the default body template is not used and instead we add collection of labels and annotations for each alert
+		DefaultBodyTemplate: "NO_OP",
+	}, alerts, markdownrenderer.MarkdownFormatNoop)
+	if err != nil {
+		return nil, err
+	}
+
+	blocks := []Body{}
+
+	// common color for the title block
+	aggregateAlerts := types.Alerts(alerts...)
+	color := colorGrey
+	switch aggregateAlerts.Status() {
+	case model.AlertFiring:
+		color = colorRed
+	case model.AlertResolved:
+		color = colorGreen
+	}
+
+	// add title block
+	blocks = append(blocks, Body{
+		Type:   "TextBlock",
+		Text:   result.Title,
+		Weight: "Bolder",
+		Size:   "Medium",
+		Wrap:   true,
+		Style:  "heading",
+		Color:  color,
+	})
+
+	// handle default templated body
+	if result.IsDefaultTemplatedBody {
+		for _, alert := range alerts {
+			blocks = append(blocks, Body{
+				Type:   "TextBlock",
+				Text:   "Alerts",
+				Weight: "Bolder",
+				Size:   "Medium",
+				Wrap:   true,
+				Color:  color,
+			})
+			blocks = append(blocks, n.createLabelsAndAnnotationsBody(alert)...)
+		}
+	} else {
+		for i, body := range result.Body {
+			b := Body{
+				Type:  "TextBlock",
+				Text:  body,
+				Wrap:  true,
+				Color: colorGrey,
+			}
+			if i < len(alerts) {
+				if alerts[i].Resolved() {
+					b.Color = colorGreen
+				} else {
+					b.Color = colorRed
+				}
+			}
+			blocks = append(blocks, b)
+		}
+	}
+
+	return blocks, nil
+}
+
 func (*Notifier) createLabelsAndAnnotationsBody(alert *types.Alert) []Body {
 	bodies := []Body{}
 	bodies = append(bodies, Body{

pkg/alertmanager/alertmanagernotify/msteamsv2/msteamsv2_test.go

@@ -1,9 +1,23 @@
+// Copyright 2024 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package msteamsv2
 
 import (
 	"context"
 	"encoding/json"
 	"io"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -11,6 +25,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertnotificationprocessor"
+	"github.com/SigNoz/signoz/pkg/emailing/templatestore/filetemplatestore"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
@@ -19,21 +39,31 @@ import (
 	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
 	"github.com/prometheus/alertmanager/types"
 )
 
+func newTestProcessor(tmpl *template.Template) alertmanagertypes.NotificationProcessor {
+	logger := slog.Default()
+	templater := alertmanagertemplate.New(tmpl, logger)
+	renderer := markdownrenderer.NewMarkdownRenderer(logger)
+	return alertnotificationprocessor.New(templater, renderer, filetemplatestore.NewEmptyStore(), logger)
+}
+
 // This is a test URL that has been modified to not be valid.
 var testWebhookURL, _ = url.Parse("https://example.westeurope.logic.azure.com:443/workflows/xxx/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=xxx")
 
 func TestMSTeamsV2Retry(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.MSTeamsV2Config{
 			WebhookURL: &config.SecretURL{URL: testWebhookURL},
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		`{{ template "msteamsv2.default.titleLink" . }}`,
 		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -60,14 +90,16 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			tmpl := test.CreateTmpl(t)
 			notifier, err := New(
 				&config.MSTeamsV2Config{
 					WebhookURL: &config.SecretURL{URL: testWebhookURL},
 					HTTPConfig: &commoncfg.HTTPClientConfig{},
 				},
-				test.CreateTmpl(t),
+				tmpl,
 				`{{ template "msteamsv2.default.titleLink" . }}`,
 				promslog.NewNopLogger(),
+				newTestProcessor(tmpl),
 			)
 			require.NoError(t, err)
 
@@ -149,7 +181,8 @@ func TestMSTeamsV2Templating(t *testing.T) {
 		t.Run(tc.title, func(t *testing.T) {
 			tc.cfg.WebhookURL = &config.SecretURL{URL: u}
 			tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
-			pd, err := New(tc.cfg, test.CreateTmpl(t), tc.titleLink, promslog.NewNopLogger())
+			tmpl := test.CreateTmpl(t)
+			pd, err := New(tc.cfg, tmpl, tc.titleLink, promslog.NewNopLogger(), newTestProcessor(tmpl))
 			require.NoError(t, err)
 
 			ctx := context.Background()
@@ -182,20 +215,124 @@ func TestMSTeamsV2RedactedURL(t *testing.T) {
 	defer fn()
 
 	secret := "secret"
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.MSTeamsV2Config{
 			WebhookURL: &config.SecretURL{URL: u},
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		`{{ template "msteamsv2.default.titleLink" . }}`,
 		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
 	)
 	require.NoError(t, err)
 
 	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, secret)
 }
 
+func TestPrepareContent(t *testing.T) {
+	t.Run("default template - firing alerts", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		notifier, err := New(
+			&config.MSTeamsV2Config{
+				WebhookURL: &config.SecretURL{URL: testWebhookURL},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+				Title:      "Alertname: {{ .CommonLabels.alertname }}",
+			},
+			tmpl,
+			`{{ template "msteamsv2.default.titleLink" . }}`,
+			promslog.NewNopLogger(),
+			newTestProcessor(tmpl),
+		)
+		require.NoError(t, err)
+
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{"alertname": "test"},
+					// Custom body template
+					Annotations: model.LabelSet{
+						ruletypes.AnnotationBodyTemplate: "Firing alert: $alertname",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+		}
+		blocks, err := notifier.prepareContent(ctx, alerts)
+		require.NoError(t, err)
+		require.NotEmpty(t, blocks)
+		// First block should be the title with color (firing = red)
+		require.Equal(t, "Bolder", blocks[0].Weight)
+		require.Equal(t, colorRed, blocks[0].Color)
+		// verify title text
+		require.Equal(t, "Alertname: test", blocks[0].Text)
+		// verify body text
+		require.Equal(t, "Firing alert: test", blocks[1].Text)
+	})
+
+	t.Run("custom template - per-alert color", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		notifier, err := New(
+			&config.MSTeamsV2Config{
+				WebhookURL: &config.SecretURL{URL: testWebhookURL},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			tmpl,
+			`{{ template "msteamsv2.default.titleLink" . }}`,
+			promslog.NewNopLogger(),
+			newTestProcessor(tmpl),
+		)
+		require.NoError(t, err)
+
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{"alertname": "test1"},
+					Annotations: model.LabelSet{
+						"summary":                         "test",
+						ruletypes.AnnotationTitleTemplate: "Custom Title",
+						ruletypes.AnnotationBodyTemplate:  "custom body $alertname",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{"alertname": "test2"},
+					Annotations: model.LabelSet{
+						"summary":                         "test",
+						ruletypes.AnnotationTitleTemplate: "Custom Title",
+						ruletypes.AnnotationBodyTemplate:  "custom body $alertname",
+					},
+					StartsAt: time.Now().Add(-time.Hour),
+					EndsAt:   time.Now().Add(-time.Minute),
+				},
+			},
+		}
+		blocks, err := notifier.prepareContent(ctx, alerts)
+		require.NoError(t, err)
+		require.NotEmpty(t, blocks)
+		// total 3 blocks: title and 2 body blocks
+		require.True(t, len(blocks) == 3)
+		// First block: title color is overall color of the alerts
+		require.Equal(t, colorRed, blocks[0].Color)
+		// verify title text
+		require.Equal(t, "Custom Title", blocks[0].Text)
+		// Body blocks should have per-alert color
+		require.Equal(t, colorRed, blocks[1].Color)   // firing
+		require.Equal(t, colorGreen, blocks[2].Color) // resolved
+	})
+}
+
 func TestMSTeamsV2ReadingURLFromFile(t *testing.T) {
 	ctx, u, fn := test.GetContextWithCancelingURL()
 	defer fn()
@@ -205,14 +342,16 @@ func TestMSTeamsV2ReadingURLFromFile(t *testing.T) {
 	_, err = f.WriteString(u.String() + "\n")
 	require.NoError(t, err, "writing to temp file failed")
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.MSTeamsV2Config{
 			WebhookURLFile: f.Name(),
 			HTTPConfig:     &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		`{{ template "msteamsv2.default.titleLink" . }}`,
 		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
 	)
 	require.NoError(t, err)
 

pkg/alertmanager/alertmanagernotify/opsgenie/api_key_file

@@ -0,0 +1,2 @@
+my_secret_api_key
+

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie.go

@@ -0,0 +1,349 @@
+// Copyright 2019 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package opsgenie
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"maps"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "opsgenie"
+)
+
+// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
+const maxMessageLenRunes = 130
+
+// Notifier implements a Notifier for OpsGenie notifications.
+type Notifier struct {
+	conf      *config.OpsGenieConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	client    *http.Client
+	retrier   *notify.Retrier
+	processor alertmanagertypes.NotificationProcessor
+}
+
+// New returns a new OpsGenie notifier.
+func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, proc alertmanagertypes.NotificationProcessor, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:      c,
+		tmpl:      t,
+		logger:    l,
+		client:    client,
+		retrier:   &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
+		processor: proc,
+	}, nil
+}
+
+type opsGenieCreateMessage struct {
+	Alias       string                           `json:"alias"`
+	Message     string                           `json:"message"`
+	Description string                           `json:"description,omitempty"`
+	Details     map[string]string                `json:"details"`
+	Source      string                           `json:"source"`
+	Responders  []opsGenieCreateMessageResponder `json:"responders,omitempty"`
+	Tags        []string                         `json:"tags,omitempty"`
+	Note        string                           `json:"note,omitempty"`
+	Priority    string                           `json:"priority,omitempty"`
+	Entity      string                           `json:"entity,omitempty"`
+	Actions     []string                         `json:"actions,omitempty"`
+}
+
+type opsGenieCreateMessageResponder struct {
+	ID       string `json:"id,omitempty"`
+	Name     string `json:"name,omitempty"`
+	Username string `json:"username,omitempty"`
+	Type     string `json:"type"` // team, user, escalation, schedule etc.
+}
+
+type opsGenieCloseMessage struct {
+	Source string `json:"source"`
+}
+
+type opsGenieUpdateMessageMessage struct {
+	Message string `json:"message,omitempty"`
+}
+
+type opsGenieUpdateDescriptionMessage struct {
+	Description string `json:"description,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	requests, retry, err := n.createRequests(ctx, as...)
+	if err != nil {
+		return retry, err
+	}
+
+	for _, req := range requests {
+		req.Header.Set("User-Agent", notify.UserAgentHeader)
+		resp, err := n.client.Do(req) //nolint:bodyclose
+		if err != nil {
+			return true, err
+		}
+		shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+		notify.Drain(resp)
+		if err != nil {
+			return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+		}
+	}
+	return true, nil
+}
+
+// Like Split but filter out empty strings.
+func safeSplit(s, sep string) []string {
+	a := strings.Split(strings.TrimSpace(s), sep)
+	b := a[:0]
+	for _, x := range a {
+		if x != "" {
+			b = append(b, x)
+		}
+	}
+	return b
+}
+
+// prepareContent extracts custom templates from alert annotations, runs the
+// notification processor, and returns a ready-to-use title (truncated to the
+// OpsGenie 130-rune limit) and description.
+func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert) (string, string, error) {
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.processor.ProcessAlertNotification(ctx, alertmanagertypes.NotificationProcessorInput{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: n.conf.Message,
+		DefaultBodyTemplate:  n.conf.Description,
+	}, alerts, markdownrenderer.MarkdownFormatHTML)
+	if err != nil {
+		return "", "", err
+	}
+
+	title := result.Title
+	description := strings.Join(result.Body, "\n")
+
+	if result.IsCustomTemplated() {
+		// OpsGenie uses basic HTML for alert description previews, so we
+		// separate each per-alert body with an <hr> divider.
+		var b strings.Builder
+		for i, part := range result.Body {
+			if i > 0 {
+				b.WriteString("<hr>")
+			}
+			b.WriteString("<div>")
+			b.WriteString(part)
+			b.WriteString("</div>")
+		}
+		description = b.String()
+	}
+
+	title, truncated := notify.TruncateInRunes(title, maxMessageLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated message", slog.Int("max_runes", maxMessageLenRunes))
+	}
+
+	return title, description, nil
+}
+
+// Create requests for a list of alerts.
+func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+	logger := n.logger.With(slog.Any("group_key", key))
+	logger.DebugContext(ctx, "extracted group key")
+
+	data := notify.GetTemplateData(ctx, n.tmpl, as, logger)
+
+	tmpl := notify.TmplText(n.tmpl, data, &err)
+
+	details := make(map[string]string)
+
+	maps.Copy(details, data.CommonLabels)
+
+	for k, v := range n.conf.Details {
+		details[k] = tmpl(v)
+	}
+
+	requests := []*http.Request{}
+
+	var (
+		alias  = key.Hash()
+		alerts = types.Alerts(as...)
+	)
+	switch alerts.Status() {
+	case model.AlertResolved:
+		resolvedEndpointURL := n.conf.APIURL.Copy()
+		resolvedEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
+		q := resolvedEndpointURL.Query()
+		q.Set("identifierType", "alias")
+		resolvedEndpointURL.RawQuery = q.Encode()
+		msg := &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", resolvedEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+	default:
+		message, description, err := n.prepareContent(ctx, as)
+		if err != nil {
+			return nil, false, err
+		}
+
+		createEndpointURL := n.conf.APIURL.Copy()
+		createEndpointURL.Path += "v2/alerts"
+
+		var responders []opsGenieCreateMessageResponder
+		for _, r := range n.conf.Responders {
+			responder := opsGenieCreateMessageResponder{
+				ID:       tmpl(r.ID),
+				Name:     tmpl(r.Name),
+				Username: tmpl(r.Username),
+				Type:     tmpl(r.Type),
+			}
+
+			if responder == (opsGenieCreateMessageResponder{}) {
+				// Filter out empty responders. This is useful if you want to fill
+				// responders dynamically from alert's common labels.
+				continue
+			}
+
+			if responder.Type == "teams" {
+				teams := safeSplit(responder.Name, ",")
+				for _, team := range teams {
+					newResponder := opsGenieCreateMessageResponder{
+						Name: tmpl(team),
+						Type: tmpl("team"),
+					}
+					responders = append(responders, newResponder)
+				}
+				continue
+			}
+
+			responders = append(responders, responder)
+		}
+
+		msg := &opsGenieCreateMessage{
+			Alias:       alias,
+			Message:     message,
+			Description: description,
+			Details:     details,
+			Source:      tmpl(n.conf.Source),
+			Responders:  responders,
+			Tags:        safeSplit(tmpl(n.conf.Tags), ","),
+			Note:        tmpl(n.conf.Note),
+			Priority:    tmpl(n.conf.Priority),
+			Entity:      tmpl(n.conf.Entity),
+			Actions:     safeSplit(tmpl(n.conf.Actions), ","),
+		}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", createEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+
+		if n.conf.UpdateAlerts {
+			updateMessageEndpointURL := n.conf.APIURL.Copy()
+			updateMessageEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/message", alias)
+			q := updateMessageEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateMessageEndpointURL.RawQuery = q.Encode()
+			updateMsgMsg := &opsGenieUpdateMessageMessage{
+				Message: msg.Message,
+			}
+			var updateMessageBuf bytes.Buffer
+			if err := json.NewEncoder(&updateMessageBuf).Encode(updateMsgMsg); err != nil {
+				return nil, false, err
+			}
+			req, err := http.NewRequest("PUT", updateMessageEndpointURL.String(), &updateMessageBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req)
+
+			updateDescriptionEndpointURL := n.conf.APIURL.Copy()
+			updateDescriptionEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/description", alias)
+			q = updateDescriptionEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateDescriptionEndpointURL.RawQuery = q.Encode()
+			updateDescMsg := &opsGenieUpdateDescriptionMessage{
+				Description: msg.Description,
+			}
+
+			var updateDescriptionBuf bytes.Buffer
+			if err := json.NewEncoder(&updateDescriptionBuf).Encode(updateDescMsg); err != nil {
+				return nil, false, err
+			}
+			req, err = http.NewRequest("PUT", updateDescriptionEndpointURL.String(), &updateDescriptionBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req.WithContext(ctx))
+		}
+	}
+
+	var apiKey string
+	if n.conf.APIKey != "" {
+		apiKey = tmpl(string(n.conf.APIKey))
+	} else {
+		content, err := os.ReadFile(n.conf.APIKeyFile)
+		if err != nil {
+			return nil, false, errors.WrapInternalf(err, errors.CodeInternal, "read key_file error")
+		}
+		apiKey = tmpl(string(content))
+		apiKey = strings.TrimSpace(string(apiKey))
+	}
+
+	if err != nil {
+		return nil, false, errors.WrapInternalf(err, errors.CodeInternal, "templating error")
+	}
+
+	for _, req := range requests {
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
+	}
+
+	return requests, true, nil
+}

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie_test.go

@@ -0,0 +1,462 @@
+// Copyright 2019 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package opsgenie
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertnotificationprocessor"
+	"github.com/SigNoz/signoz/pkg/emailing/templatestore/filetemplatestore"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func newTestProcessor(tmpl *template.Template) alertmanagertypes.NotificationProcessor {
+	logger := slog.Default()
+	templater := alertmanagertemplate.New(tmpl, logger)
+	renderer := markdownrenderer.NewMarkdownRenderer(logger)
+	return alertnotificationprocessor.New(templater, renderer, filetemplatestore.NewEmptyStore(), logger)
+}
+
+func TestOpsGenieRetry(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestOpsGenieRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKey:     config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestGettingOpsGegineApikeyFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+
+	f, err := os.CreateTemp(t.TempDir(), "opsgenie_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKeyFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestOpsGenie(t *testing.T) {
+	u, err := url.Parse("https://opsgenie/api")
+	if err != nil {
+		t.Fatalf("failed to parse URL: %v", err)
+	}
+	logger := promslog.NewNopLogger()
+	tmpl := test.CreateTmpl(t)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.OpsGenieConfig
+
+		expectedEmptyAlertBody string
+		expectedBody           string
+	}{
+		{
+			title: "config without details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with multiple teams",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName3 }}`,
+						Type: `{{ .CommonLabels.ResponderType3 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"TeamB","type":"team"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1"}
+`,
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			notifier, err := New(tc.cfg, tmpl, logger, newTestProcessor(tmpl))
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			expectedURL, _ := url.Parse("https://opsgenie/apiv2/alerts")
+
+			// Empty alert.
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+
+			req, retry, err := notifier.createRequests(ctx, alert1)
+			require.NoError(t, err)
+			require.Len(t, req, 1)
+			require.True(t, retry)
+			require.Equal(t, expectedURL, req[0].URL)
+			require.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization"))
+			require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0]))
+
+			// Fully defined alert.
+			alert2 := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"Message":        "message",
+						"Description":    "description",
+						"Source":         "http://prometheus",
+						"ResponderName1": "TeamA",
+						"ResponderType1": "team",
+						"ResponderName2": "EscalationA",
+						"ResponderType2": "escalation",
+						"ResponderName3": "TeamA,TeamB",
+						"ResponderType3": "teams",
+						"Tags":           "tag1,tag2",
+						"Note":           "this is a note",
+						"Priority":       "P1",
+						"Entity":         "test-domain",
+						"Actions":        "doThis,doThat",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			req, retry, err = notifier.createRequests(ctx, alert2)
+			require.NoError(t, err)
+			require.True(t, retry)
+			require.Len(t, req, 1)
+			require.Equal(t, tc.expectedBody, readBody(t, req[0]))
+
+			// Broken API Key Template.
+			tc.cfg.APIKey = "{{ kaput "
+			_, _, err = notifier.createRequests(ctx, alert2)
+			require.Error(t, err)
+			require.Equal(t, "template: :1: function \"kaput\" not defined", err.Error())
+		})
+	}
+}
+
+func TestOpsGenieWithUpdate(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		Message:      `{{ .CommonLabels.Message }}`,
+		Description:  `{{ .CommonLabels.Description }}`,
+		UpdateAlerts: true,
+		APIKey:       "test-api-key",
+		APIURL:       &config.URL{URL: u},
+		HTTPConfig:   &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger(), newTestProcessor(tmpl))
+	alert := &types.Alert{
+		Alert: model.Alert{
+			StartsAt: time.Now(),
+			EndsAt:   time.Now().Add(time.Hour),
+			Labels: model.LabelSet{
+				"Message":     "new message",
+				"Description": "new description",
+			},
+		},
+	}
+	require.NoError(t, err)
+	requests, retry, err := notifierWithUpdate.createRequests(ctx, alert)
+	require.NoError(t, err)
+	require.True(t, retry)
+	require.Len(t, requests, 3)
+
+	body0 := readBody(t, requests[0])
+	body1 := readBody(t, requests[1])
+	body2 := readBody(t, requests[2])
+	key, _ := notify.ExtractGroupKey(ctx)
+	alias := key.Hash()
+
+	require.Equal(t, "https://test-opsgenie-url/v2/alerts", requests[0].URL.String())
+	require.NotEmpty(t, body0)
+
+	require.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias))
+	require.JSONEq(t, `{"message":"new message"}`, body1)
+	require.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias))
+	require.JSONEq(t, `{"description":"new description"}`, body2)
+}
+
+func TestOpsGenieApiKeyFile(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		APIKeyFile: `./api_key_file`,
+		APIURL:     &config.URL{URL: u},
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger(), newTestProcessor(tmpl))
+
+	require.NoError(t, err)
+	requests, _, err := notifierWithUpdate.createRequests(ctx)
+	require.NoError(t, err)
+	require.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
+}
+
+func TestPrepareContent(t *testing.T) {
+	t.Run("default template", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		logger := promslog.NewNopLogger()
+		proc := newTestProcessor(tmpl)
+
+		notifier := &Notifier{
+			conf: &config.OpsGenieConfig{
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+			},
+			tmpl:      tmpl,
+			logger:    logger,
+			processor: proc,
+		}
+
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+
+		alert := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"Message":     "Firing alert: test",
+					"Description": "Check runbook for more details",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		}
+
+		alerts := []*types.Alert{alert}
+
+		title, desc, prepErr := notifier.prepareContent(ctx, alerts)
+		require.NoError(t, prepErr)
+		require.Equal(t, "Firing alert: test", title)
+		require.Equal(t, "Check runbook for more details", desc)
+	})
+
+	t.Run("custom template", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		logger := promslog.NewNopLogger()
+		proc := newTestProcessor(tmpl)
+
+		notifier := &Notifier{
+			conf: &config.OpsGenieConfig{
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+			},
+			tmpl:      tmpl,
+			logger:    logger,
+			processor: proc,
+		}
+
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+
+		alert1 := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"service":   "payment",
+					"namespace": "potter-the-harry",
+				},
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: "High request throughput for $service",
+					ruletypes.AnnotationBodyTemplate:  "Alert firing in NS: $labels.namespace",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		}
+		alert2 := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"service":   "payment",
+					"namespace": "smart-the-rat",
+				},
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: "High request throughput for $service",
+					ruletypes.AnnotationBodyTemplate:  "Alert firing in NS: $labels.namespace",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		}
+
+		alerts := []*types.Alert{alert1, alert2}
+
+		title, desc, err := notifier.prepareContent(ctx, alerts)
+		require.NoError(t, err)
+		require.Equal(t, "High request throughput for payment", title)
+		// Each alert body wrapped in <div>, separated by <hr>
+		require.Equal(t, "<div><p>Alert firing in NS: potter-the-harry</p><p></p></div><hr><div><p>Alert firing in NS: smart-the-rat</p><p></p></div>", desc)
+	})
+}
+
+func readBody(t *testing.T, r *http.Request) string {
+	t.Helper()
+	body, err := io.ReadAll(r.Body)
+	require.NoError(t, err)
+	return string(body)
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty.go

@@ -0,0 +1,414 @@
+// Copyright 2019 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/alecthomas/units"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "pagerduty"
+)
+
+const (
+	maxEventSize int = 512000
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTc4-send-a-v1-event - 1024 characters or runes.
+	maxV1DescriptionLenRunes = 1024
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1024 characters or runes.
+	maxV2SummaryLenRunes = 1024
+)
+
+// Notifier implements a Notifier for PagerDuty notifications.
+type Notifier struct {
+	conf      *config.PagerdutyConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	apiV1     string // for tests.
+	client    *http.Client
+	retrier   *notify.Retrier
+	processor alertmanagertypes.NotificationProcessor
+}
+
+// New returns a new PagerDuty notifier.
+func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, proc alertmanagertypes.NotificationProcessor, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	n := &Notifier{conf: c, tmpl: t, logger: l, client: client, processor: proc}
+	if c.ServiceKey != "" || c.ServiceKeyFile != "" {
+		n.apiV1 = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+		// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v1-overview#api-response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusForbidden}, CustomDetailsFunc: errDetails}
+	} else {
+		// Retrying can solve the issue on 429 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v2-overview#response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}, CustomDetailsFunc: errDetails}
+	}
+	return n, nil
+}
+
+const (
+	pagerDutyEventTrigger = "trigger"
+	pagerDutyEventResolve = "resolve"
+)
+
+type pagerDutyMessage struct {
+	RoutingKey  string            `json:"routing_key,omitempty"`
+	ServiceKey  string            `json:"service_key,omitempty"`
+	DedupKey    string            `json:"dedup_key,omitempty"`
+	IncidentKey string            `json:"incident_key,omitempty"`
+	EventType   string            `json:"event_type,omitempty"`
+	Description string            `json:"description,omitempty"`
+	EventAction string            `json:"event_action"`
+	Payload     *pagerDutyPayload `json:"payload"`
+	Client      string            `json:"client,omitempty"`
+	ClientURL   string            `json:"client_url,omitempty"`
+	Details     map[string]any    `json:"details,omitempty"`
+	Images      []pagerDutyImage  `json:"images,omitempty"`
+	Links       []pagerDutyLink   `json:"links,omitempty"`
+}
+
+type pagerDutyLink struct {
+	HRef string `json:"href"`
+	Text string `json:"text"`
+}
+
+type pagerDutyImage struct {
+	Src  string `json:"src"`
+	Alt  string `json:"alt"`
+	Href string `json:"href"`
+}
+
+type pagerDutyPayload struct {
+	Summary       string         `json:"summary"`
+	Source        string         `json:"source"`
+	Severity      string         `json:"severity"`
+	Timestamp     string         `json:"timestamp,omitempty"`
+	Class         string         `json:"class,omitempty"`
+	Component     string         `json:"component,omitempty"`
+	Group         string         `json:"group,omitempty"`
+	CustomDetails map[string]any `json:"custom_details,omitempty"`
+}
+
+func (n *Notifier) encodeMessage(ctx context.Context, msg *pagerDutyMessage) (bytes.Buffer, error) {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return buf, errors.WrapInternalf(err, errors.CodeInternal, "failed to encode PagerDuty message")
+	}
+
+	if buf.Len() > maxEventSize {
+		truncatedMsg := fmt.Sprintf("Custom details have been removed because the original event exceeds the maximum size of %s", units.MetricBytes(maxEventSize).String())
+
+		if n.apiV1 != "" {
+			msg.Details = map[string]any{"error": truncatedMsg}
+		} else {
+			msg.Payload.CustomDetails = map[string]any{"error": truncatedMsg}
+		}
+
+		n.logger.WarnContext(ctx, "Truncated Details because message of size exceeds limit", slog.String("message_size", units.MetricBytes(buf.Len()).String()), slog.String("max_size", units.MetricBytes(maxEventSize).String()))
+
+		buf.Reset()
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return buf, errors.WrapInternalf(err, errors.CodeInternal, "failed to encode PagerDuty message")
+		}
+	}
+
+	return buf, nil
+}
+
+func (n *Notifier) notifyV1(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+	title string,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	description, truncated := notify.TruncateInRunes(title, maxV1DescriptionLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated description", slog.Any("key", key), slog.Int("max_runes", maxV1DescriptionLenRunes))
+	}
+
+	serviceKey := string(n.conf.ServiceKey)
+	if serviceKey == "" {
+		content, fileErr := os.ReadFile(n.conf.ServiceKeyFile)
+		if fileErr != nil {
+			return false, errors.WrapInternalf(fileErr, errors.CodeInternal, "failed to read service key from file")
+		}
+		serviceKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		ServiceKey:  tmpl(serviceKey),
+		EventType:   eventType,
+		IncidentKey: key.Hash(),
+		Description: description,
+		Details:     details,
+	}
+
+	if eventType == pagerDutyEventTrigger {
+		msg.Client = tmpl(n.conf.Client)
+		msg.ClientURL = tmpl(n.conf.ClientURL)
+	}
+
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "failed to template PagerDuty v1 message")
+	}
+
+	// Ensure that the service key isn't empty after templating.
+	if msg.ServiceKey == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "service key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(ctx, msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &encodedMsg) //nolint:bodyclose
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "failed to post message to PagerDuty v1")
+	}
+	defer notify.Drain(resp)
+
+	return n.retrier.Check(resp.StatusCode, resp.Body)
+}
+
+func (n *Notifier) notifyV2(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+	title string,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.Severity == "" {
+		n.conf.Severity = "error"
+	}
+
+	summary, truncated := notify.TruncateInRunes(title, maxV2SummaryLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated summary", slog.Any("key", key), slog.Int("max_runes", maxV2SummaryLenRunes))
+	}
+
+	routingKey := string(n.conf.RoutingKey)
+	if routingKey == "" {
+		content, fileErr := os.ReadFile(n.conf.RoutingKeyFile)
+		if fileErr != nil {
+			return false, errors.WrapInternalf(fileErr, errors.CodeInternal, "failed to read routing key from file")
+		}
+		routingKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		Client:      tmpl(n.conf.Client),
+		ClientURL:   tmpl(n.conf.ClientURL),
+		RoutingKey:  tmpl(routingKey),
+		EventAction: eventType,
+		DedupKey:    key.Hash(),
+		Images:      make([]pagerDutyImage, 0, len(n.conf.Images)),
+		Links:       make([]pagerDutyLink, 0, len(n.conf.Links)),
+		Payload: &pagerDutyPayload{
+			Summary:       summary,
+			Source:        tmpl(n.conf.Source),
+			Severity:      tmpl(n.conf.Severity),
+			CustomDetails: details,
+			Class:         tmpl(n.conf.Class),
+			Component:     tmpl(n.conf.Component),
+			Group:         tmpl(n.conf.Group),
+		},
+	}
+
+	for _, item := range n.conf.Images {
+		image := pagerDutyImage{
+			Src:  tmpl(item.Src),
+			Alt:  tmpl(item.Alt),
+			Href: tmpl(item.Href),
+		}
+
+		if image.Src != "" {
+			msg.Images = append(msg.Images, image)
+		}
+	}
+
+	for _, item := range n.conf.Links {
+		link := pagerDutyLink{
+			HRef: tmpl(item.Href),
+			Text: tmpl(item.Text),
+		}
+
+		if link.HRef != "" {
+			msg.Links = append(msg.Links, link)
+		}
+	}
+
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "failed to template PagerDuty v2 message")
+	}
+
+	// Ensure that the routing key isn't empty after templating.
+	if msg.RoutingKey == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "routing key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(ctx, msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &encodedMsg) //nolint:bodyclose
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "failed to post message to PagerDuty")
+	}
+	defer notify.Drain(resp)
+
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return retry, err
+}
+
+// prepareContent extracts custom templates from alert annotations, runs the
+// notification processor, and returns the processed title ready for use.
+func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert) (string, error) {
+	customTitle, _ := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.processor.ProcessAlertNotification(ctx, alertmanagertypes.NotificationProcessorInput{
+		TitleTemplate:        customTitle,
+		DefaultTitleTemplate: n.conf.Description,
+		BodyTemplate:         "NO_OP",
+		DefaultBodyTemplate:  "NO_OP",
+	}, alerts, markdownrenderer.MarkdownFormatNoop)
+	if err != nil {
+		return "", err
+	}
+	return result.Title, nil
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With(slog.Any("group_key", key))
+
+	var (
+		alerts    = types.Alerts(as...)
+		data      = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		eventType = pagerDutyEventTrigger
+	)
+
+	if alerts.Status() == model.AlertResolved {
+		eventType = pagerDutyEventResolve
+	}
+
+	logger.DebugContext(ctx, "extracted group key", slog.String("event_type", eventType))
+
+	details, err := n.renderDetails(data)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "failed to render details: %v", err)
+	}
+
+	if n.conf.Timeout > 0 {
+		nfCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeTimeout, "configured pagerduty timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = nfCtx
+	}
+
+	title, err := n.prepareContent(ctx, as)
+	if err != nil {
+		return false, err
+	}
+
+	nf := n.notifyV2
+	if n.apiV1 != "" {
+		nf = n.notifyV1
+	}
+	retry, err := nf(ctx, eventType, key, data, details, title)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.WrapInternalf(err, errors.CodeInternal, "failed to notify PagerDuty: %v", context.Cause(ctx))
+		}
+		return retry, err
+	}
+	return retry, nil
+}
+
+func errDetails(status int, body io.Reader) string {
+	// See https://v2.developer.pagerduty.com/docs/trigger-events for the v1 events API.
+	// See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 for the v2 events API.
+	if status != http.StatusBadRequest || body == nil {
+		return ""
+	}
+	var pgr struct {
+		Status  string   `json:"status"`
+		Message string   `json:"message"`
+		Errors  []string `json:"errors"`
+	}
+	if err := json.NewDecoder(body).Decode(&pgr); err != nil {
+		return ""
+	}
+	return fmt.Sprintf("%s: %s", pgr.Message, strings.Join(pgr.Errors, ","))
+}
+
+func (n *Notifier) renderDetails(
+	data *template.Data,
+) (map[string]any, error) {
+	var (
+		tmplTextErr  error
+		tmplText     = notify.TmplText(n.tmpl, data, &tmplTextErr)
+		tmplTextFunc = func(tmpl string) (string, error) {
+			return tmplText(tmpl), tmplTextErr
+		}
+	)
+	var err error
+	rendered := make(map[string]any, len(n.conf.Details))
+	for k, v := range n.conf.Details {
+		rendered[k], err = template.DeepCopyWithTemplate(v, tmplTextFunc)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return rendered, nil
+}

pkg/alertmanager/alertmanagernotify/receiver.go

@@ -2,8 +2,14 @@ package alertmanagernotify
 
 import (
 	"log/slog"
+	"slices"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/email"
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/msteamsv2"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/opsgenie"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/pagerduty"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/slack"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/webhook"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/prometheus/alertmanager/config/receiver"
 	"github.com/prometheus/alertmanager/notify"
@@ -11,7 +17,16 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
-func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error) {
+var customNotifierIntegrations = []string{
+	webhook.Integration,
+	email.Integration,
+	pagerduty.Integration,
+	opsgenie.Integration,
+	slack.Integration,
+	msteamsv2.Integration,
+}
+
+func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger, proc alertmanagertypes.NotificationProcessor) ([]notify.Integration, error) {
 	upstreamIntegrations, err := receiver.BuildReceiverIntegrations(nc, tmpl, logger)
 	if err != nil {
 		return nil, err
@@ -31,15 +46,30 @@ func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Templ
 	)
 
 	for _, integration := range upstreamIntegrations {
-		// skip upstream msteamsv2 integration
-		if integration.Name() != "msteamsv2" {
+		// skip upstream integration if we support custom integration for it
+		if !slices.Contains(customNotifierIntegrations, integration.Name()) {
 			integrations = append(integrations, integration)
 		}
 	}
 
+	for i, c := range nc.WebhookConfigs {
+		add(webhook.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l, proc) })
+	}
+	for i, c := range nc.EmailConfigs {
+		add(email.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l, proc), nil })
+	}
+	for i, c := range nc.PagerdutyConfigs {
+		add(pagerduty.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l, proc) })
+	}
+	for i, c := range nc.OpsGenieConfigs {
+		add(opsgenie.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l, proc) })
+	}
+	for i, c := range nc.SlackConfigs {
+		add(slack.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l, proc) })
+	}
 	for i, c := range nc.MSTeamsV2Configs {
-		add("msteamsv2", i, c, func(l *slog.Logger) (notify.Notifier, error) {
-			return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l)
+		add(msteamsv2.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
+			return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l, proc)
 		})
 	}
 

pkg/alertmanager/alertmanagernotify/slack/slack.go

@@ -0,0 +1,377 @@
+// Copyright 2019 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package slack
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "slack"
+	colorRed    = "#FF0000"
+	colorGreen  = "#00FF00"
+)
+
+// https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
+const maxTitleLenRunes = 1024
+
+// Notifier implements a Notifier for Slack notifications.
+type Notifier struct {
+	conf      *config.SlackConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	client    *http.Client
+	retrier   *notify.Retrier
+	processor alertmanagertypes.NotificationProcessor
+
+	postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
+}
+
+// New returns a new Slack notification handler.
+func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, proc alertmanagertypes.NotificationProcessor, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Notifier{
+		conf:         c,
+		tmpl:         t,
+		logger:       l,
+		client:       client,
+		retrier:      &notify.Retrier{},
+		processor:    proc,
+		postJSONFunc: notify.PostJSON,
+	}, nil
+}
+
+// request is the request for sending a slack notification.
+type request struct {
+	Channel     string       `json:"channel,omitempty"`
+	Username    string       `json:"username,omitempty"`
+	IconEmoji   string       `json:"icon_emoji,omitempty"`
+	IconURL     string       `json:"icon_url,omitempty"`
+	LinkNames   bool         `json:"link_names,omitempty"`
+	Text        string       `json:"text,omitempty"`
+	Attachments []attachment `json:"attachments"`
+}
+
+// attachment is used to display a richly-formatted message block.
+type attachment struct {
+	Title      string               `json:"title,omitempty"`
+	TitleLink  string               `json:"title_link,omitempty"`
+	Pretext    string               `json:"pretext,omitempty"`
+	Text       string               `json:"text,omitempty"`
+	Fallback   string               `json:"fallback,omitempty"`
+	CallbackID string               `json:"callback_id,omitempty"`
+	Fields     []config.SlackField  `json:"fields,omitempty"`
+	Actions    []config.SlackAction `json:"actions,omitempty"`
+	ImageURL   string               `json:"image_url,omitempty"`
+	ThumbURL   string               `json:"thumb_url,omitempty"`
+	Footer     string               `json:"footer,omitempty"`
+	Color      string               `json:"color,omitempty"`
+	MrkdwnIn   []string             `json:"mrkdwn_in,omitempty"`
+	Blocks     []any                `json:"blocks,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With(slog.Any("group_key", key))
+	logger.DebugContext(ctx, "extracted group key")
+
+	var (
+		data     = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		tmplText = notify.TmplText(n.tmpl, data, &err)
+	)
+
+	attachments, err := n.prepareContent(ctx, as, tmplText)
+	if err != nil {
+		return false, err
+	}
+
+	if len(attachments) > 0 {
+		n.addFieldsAndActions(&attachments[0], tmplText)
+	}
+
+	req := &request{
+		Channel:     tmplText(n.conf.Channel),
+		Username:    tmplText(n.conf.Username),
+		IconEmoji:   tmplText(n.conf.IconEmoji),
+		IconURL:     tmplText(n.conf.IconURL),
+		LinkNames:   n.conf.LinkNames,
+		Text:        tmplText(n.conf.MessageText),
+		Attachments: attachments,
+	}
+	if err != nil {
+		return false, err
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(req); err != nil {
+		return false, err
+	}
+
+	var u string
+	if n.conf.APIURL != nil {
+		u = n.conf.APIURL.String()
+	} else {
+		content, err := os.ReadFile(n.conf.APIURLFile)
+		if err != nil {
+			return false, err
+		}
+		u = strings.TrimSpace(string(content))
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeTimeout, "configured slack timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := n.postJSONFunc(ctx, n.client, u, &buf) //nolint:bodyclose
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.NewInternalf(errors.CodeInternal, "failed to post JSON to slack: %v", context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	// Use a retrier to generate an error message for non-200 responses and
+	// classify them as retriable or not.
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		err = errors.NewInternalf(errors.CodeInternal, "channel %q: %v", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+
+	// Slack web API might return errors with a 200 response code.
+	// https://docs.slack.dev/tools/node-slack-sdk/web-api/#handle-errors
+	retry, err = checkResponseError(resp)
+	if err != nil {
+		err = errors.NewInternalf(errors.CodeInternal, "channel %q: %v", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.ClientErrorReason, err)
+	}
+
+	return retry, nil
+}
+
+// prepareContent extracts custom templates from alert annotations, runs the
+// notification processor, and returns the Slack attachment(s) ready to send.
+func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert, tmplText func(string) string) ([]attachment, error) {
+	// Extract custom templates and process them
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.processor.ProcessAlertNotification(ctx, alertmanagertypes.NotificationProcessorInput{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: n.conf.Title,
+		// use default body templating to prepare the attachment
+		// as default template uses plain text markdown rendering instead of blockkit
+		DefaultBodyTemplate: "NO_OP",
+	}, alerts, markdownrenderer.MarkdownFormatSlackMrkdwn)
+	if err != nil {
+		return nil, err
+	}
+
+	title, truncated := notify.TruncateInRunes(result.Title, maxTitleLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated title", slog.Int("max_runes", maxTitleLenRunes))
+	}
+
+	if result.IsDefaultTemplatedBody {
+		var markdownIn []string
+		if len(n.conf.MrkdwnIn) == 0 {
+			markdownIn = []string{"fallback", "pretext", "text"}
+		} else {
+			markdownIn = n.conf.MrkdwnIn
+		}
+		attachments := []attachment{
+			{
+				Title:      title,
+				TitleLink:  tmplText(n.conf.TitleLink),
+				Pretext:    tmplText(n.conf.Pretext),
+				Text:       tmplText(n.conf.Text),
+				Fallback:   tmplText(n.conf.Fallback),
+				CallbackID: tmplText(n.conf.CallbackID),
+				ImageURL:   tmplText(n.conf.ImageURL),
+				ThumbURL:   tmplText(n.conf.ThumbURL),
+				Footer:     tmplText(n.conf.Footer),
+				Color:      tmplText(n.conf.Color),
+				MrkdwnIn:   markdownIn,
+			},
+		}
+		return attachments, nil
+	}
+
+	// Custom template path: one title attachment + one attachment per alert body.
+	// Each alert body gets its own attachment so we can set per-alert color
+	// (red for firing, green for resolved).
+	attachments := make([]attachment, 0, 1+len(result.Body))
+
+	// Title-only attachment (no color)
+	attachments = append(attachments, attachment{
+		Title:     title,
+		TitleLink: tmplText(n.conf.TitleLink),
+	})
+
+	for i, body := range result.Body {
+		color := colorRed // red for firing
+		if i < len(alerts) && alerts[i].Resolved() {
+			color = colorGreen // green for resolved
+		}
+
+		// If alert has related logs and traces, add them to the attachment as action buttons
+		var actionButtons []config.SlackAction
+		relatedLogsLink := alerts[i].Annotations[ruletypes.AnnotationRelatedLogs]
+		relatedTracesLink := alerts[i].Annotations[ruletypes.AnnotationRelatedTraces]
+		if relatedLogsLink != "" {
+			actionButtons = append(actionButtons, config.SlackAction{
+				Type: "button",
+				Text: "View Related Logs",
+				URL:  string(relatedLogsLink),
+			})
+		}
+		if relatedTracesLink != "" {
+			actionButtons = append(actionButtons, config.SlackAction{
+				Type: "button",
+				Text: "View Related Traces",
+				URL:  string(relatedTracesLink),
+			})
+		}
+
+		attachments = append(attachments, attachment{
+			Text:     body,
+			Color:    color,
+			MrkdwnIn: []string{"text"},
+			Actions:  actionButtons,
+		})
+	}
+
+	return attachments, nil
+}
+
+// addFieldsAndActions populates fields and actions on the attachment from the Slack config.
+func (n *Notifier) addFieldsAndActions(att *attachment, tmplText func(string) string) {
+	numFields := len(n.conf.Fields)
+	if numFields > 0 {
+		fields := make([]config.SlackField, numFields)
+		for index, field := range n.conf.Fields {
+			var short bool
+			if field.Short != nil {
+				short = *field.Short
+			} else {
+				short = n.conf.ShortFields
+			}
+			fields[index] = config.SlackField{
+				Title: tmplText(field.Title),
+				Value: tmplText(field.Value),
+				Short: &short,
+			}
+		}
+		att.Fields = fields
+	}
+
+	numActions := len(n.conf.Actions)
+	if numActions > 0 {
+		actions := make([]config.SlackAction, numActions)
+		for index, action := range n.conf.Actions {
+			slackAction := config.SlackAction{
+				Type:  tmplText(action.Type),
+				Text:  tmplText(action.Text),
+				URL:   tmplText(action.URL),
+				Style: tmplText(action.Style),
+				Name:  tmplText(action.Name),
+				Value: tmplText(action.Value),
+			}
+
+			if action.ConfirmField != nil {
+				slackAction.ConfirmField = &config.SlackConfirmationField{
+					Title:       tmplText(action.ConfirmField.Title),
+					Text:        tmplText(action.ConfirmField.Text),
+					OkText:      tmplText(action.ConfirmField.OkText),
+					DismissText: tmplText(action.ConfirmField.DismissText),
+				}
+			}
+
+			actions[index] = slackAction
+		}
+		att.Actions = actions
+	}
+}
+
+// checkResponseError parses out the error message from Slack API response.
+func checkResponseError(resp *http.Response) (bool, error) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "could not read response body")
+	}
+
+	if strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") {
+		return checkJSONResponseError(body)
+	}
+	return checkTextResponseError(body)
+}
+
+// checkTextResponseError classifies plaintext responses from Slack.
+// A plaintext (non-JSON) response is successful if it's a string "ok".
+// This is typically a response for an Incoming Webhook
+// (https://api.slack.com/messaging/webhooks#handling_errors)
+func checkTextResponseError(body []byte) (bool, error) {
+	if !bytes.Equal(body, []byte("ok")) {
+		return false, errors.NewInternalf(errors.CodeInternal, "received an error response from Slack: %s", string(body))
+	}
+	return false, nil
+}
+
+// checkJSONResponseError classifies JSON responses from Slack.
+func checkJSONResponseError(body []byte) (bool, error) {
+	// response is for parsing out errors from the JSON response.
+	type response struct {
+		OK    bool   `json:"ok"`
+		Error string `json:"error"`
+	}
+
+	var data response
+	if err := json.Unmarshal(body, &data); err != nil {
+		return true, errors.NewInternalf(errors.CodeInternal, "could not unmarshal JSON response %q: %v", string(body), err)
+	}
+	if !data.OK {
+		return false, errors.NewInternalf(errors.CodeInternal, "error response from Slack: %s", data.Error)
+	}
+	return false, nil
+}

pkg/alertmanager/alertmanagernotify/slack/slack_test.go

@@ -0,0 +1,596 @@
+// Copyright 2019 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package slack
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertnotificationprocessor"
+	"github.com/SigNoz/signoz/pkg/emailing/templatestore/filetemplatestore"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func newTestProcessor(tmpl *template.Template) alertmanagertypes.NotificationProcessor {
+	logger := slog.Default()
+	templater := alertmanagertemplate.New(tmpl, logger)
+	renderer := markdownrenderer.NewMarkdownRenderer(logger)
+	return alertnotificationprocessor.New(templater, renderer, filetemplatestore.NewEmptyStore(), logger)
+}
+
+func TestSlackRetry(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.SlackConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestSlackRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURL:     &config.SecretURL{URL: u},
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestGettingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String())
+	require.NoError(t, err, "writing to temp file failed")
+
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestTrimmingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test_newline")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestNotifier_Notify_WithReason(t *testing.T) {
+	tests := []struct {
+		name           string
+		statusCode     int
+		responseBody   string
+		expectedReason notify.Reason
+		expectedErr    string
+		expectedRetry  bool
+		noError        bool
+	}{
+		{
+			name:           "with a 4xx status code",
+			statusCode:     http.StatusUnauthorized,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 401",
+		},
+		{
+			name:           "with a 5xx status code",
+			statusCode:     http.StatusInternalServerError,
+			expectedReason: notify.ServerErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "unexpected status code 500",
+		},
+		{
+			name:           "with a 3xx status code",
+			statusCode:     http.StatusTemporaryRedirect,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 307",
+		},
+		{
+			name:           "with a 1xx status code",
+			statusCode:     http.StatusSwitchingProtocols,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 101",
+		},
+		{
+			name:           "2xx response with invalid JSON",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"not valid json"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "could not unmarshal",
+		},
+		{
+			name:           "2xx response with a JSON error",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"ok":false,"error":"error_message"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: error_message",
+		},
+		{
+			name:           "2xx response with a plaintext error",
+			statusCode:     http.StatusOK,
+			responseBody:   "no_channel",
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: no_channel",
+		},
+		{
+			name:         "successful JSON response",
+			statusCode:   http.StatusOK,
+			responseBody: `{"ok":true}`,
+			noError:      true,
+		},
+		{
+			name:         "successful plaintext response",
+			statusCode:   http.StatusOK,
+			responseBody: "ok",
+			noError:      true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			apiurl, _ := url.Parse("https://slack.com/post.Message")
+			tmpl := test.CreateTmpl(t)
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: apiurl},
+					Channel:        "channelname",
+				},
+				tmpl,
+				promslog.NewNopLogger(),
+				newTestProcessor(tmpl),
+			)
+			require.NoError(t, err)
+
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				resp := httptest.NewRecorder()
+				if strings.HasPrefix(tt.responseBody, "{") {
+					resp.Header().Add("Content-Type", "application/json; charset=utf-8")
+				}
+				resp.WriteHeader(tt.statusCode)
+				_, _ = resp.WriteString(tt.responseBody)
+				return resp.Result(), nil
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			retry, err := notifier.Notify(ctx, alert1)
+			require.Equal(t, tt.expectedRetry, retry)
+			if tt.noError {
+				require.NoError(t, err)
+			} else {
+				var reasonError *notify.ErrorWithReason
+				require.ErrorAs(t, err, &reasonError)
+				require.Equal(t, tt.expectedReason, reasonError.Reason)
+				require.Contains(t, err.Error(), tt.expectedErr)
+				require.Contains(t, err.Error(), "channelname")
+			}
+		})
+	}
+}
+
+func TestSlackTimeout(t *testing.T) {
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			u, _ := url.Parse("https://slack.com/post.Message")
+			tmpl := test.CreateTmpl(t)
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: u},
+					Channel:        "channelname",
+					Timeout:        tt.timeout,
+				},
+				tmpl,
+				promslog.NewNopLogger(),
+				newTestProcessor(tmpl),
+			)
+			require.NoError(t, err)
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				select {
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				case <-time.After(tt.latency):
+					resp := httptest.NewRecorder()
+					resp.Header().Set("Content-Type", "application/json; charset=utf-8")
+					resp.WriteHeader(http.StatusOK)
+					_, _ = resp.WriteString(`{"ok":true}`)
+
+					return resp.Result(), nil
+				}
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = notifier.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+// setupTestContext creates a context with group key, receiver name, and group labels
+// required by the notification processor.
+func setupTestContext() context.Context {
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "slack")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{
+		"alertname": "TestAlert",
+		"severity":  "critical",
+	})
+	return ctx
+}
+
+func TestPrepareContent(t *testing.T) {
+	t.Run("default template uses go text template config for title and body", func(t *testing.T) {
+		// When alerts have no custom annotation templates (title_template / body_template),
+		tmpl := test.CreateTmpl(t)
+		proc := newTestProcessor(tmpl)
+		notifier := &Notifier{
+			conf: &config.SlackConfig{
+				Title:     `{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})`,
+				Text:      `{{ range .Alerts }}Alert: {{ .Labels.alertname }} - severity {{ .Labels.severity }}{{ end }}`,
+				Color:     `{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}`,
+				TitleLink: "https://alertmanager.signoz.com",
+			},
+			tmpl:      tmpl,
+			logger:    slog.New(slog.DiscardHandler),
+			processor: proc,
+		}
+
+		ctx := setupTestContext()
+		alerts := []*types.Alert{
+			{Alert: model.Alert{
+				Labels:   model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			}},
+		}
+
+		// Build tmplText the same way Notify does
+		var err error
+		data := notify.GetTemplateData(ctx, tmpl, alerts, slog.New(slog.DiscardHandler))
+		tmplText := notify.TmplText(tmpl, data, &err)
+
+		atts, attErr := notifier.prepareContent(ctx, alerts, tmplText)
+		require.NoError(t, attErr)
+		require.NoError(t, err)
+		require.Len(t, atts, 1)
+
+		require.Equal(t, "HighCPU (FIRING)", atts[0].Title)
+		require.Equal(t, "Alert: HighCPU - severity critical", atts[0].Text)
+		// Color is templated — firing alert should be "danger"
+		require.Equal(t, "danger", atts[0].Color)
+		// No BlockKit blocks for default template
+		require.Nil(t, atts[0].Blocks)
+		// Default markdownIn when config has none
+		require.Equal(t, []string{"fallback", "pretext", "text"}, atts[0].MrkdwnIn)
+	})
+
+	t.Run("custom template produces 1+N attachments with per-alert color", func(t *testing.T) {
+		// When alerts carry custom $variable annotation templates (title_template / body_template)
+		tmpl := test.CreateTmpl(t)
+		proc := newTestProcessor(tmpl)
+		notifier := &Notifier{
+			conf: &config.SlackConfig{
+				Title:     "default title fallback",
+				Text:      "default text fallback",
+				TitleLink: "https://alertmanager.signoz.com",
+			},
+			tmpl:      tmpl,
+			logger:    slog.New(slog.DiscardHandler),
+			processor: proc,
+		}
+		tmplText := func(s string) string { return s }
+
+		bodyTemplate := `## $rule_name
+
+**Service:** *$labels.service*
+**Instance:** *$labels.instance*
+**Region:** *$labels.region*
+**Method:** *$labels.http_method*
+
+---
+
+| Metric | Value |
+|--------|-------|
+| **Current** | *$value* |
+| **Threshold** | *$threshold* |
+
+**Status:** $status | **Severity:** $severity`
+		titleTemplate := "[$status] $rule_name — $labels.service"
+
+		ctx := setupTestContext()
+		firingAlert := &types.Alert{
+			Alert: model.Alert{
+				Labels:   model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical", "service": "api-server", "instance": "i-0abc123", "region": "us-east-1", "http_method": "GET"},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: model.LabelValue(titleTemplate),
+					ruletypes.AnnotationBodyTemplate:  model.LabelValue(bodyTemplate),
+					"value":                           "100",
+					"threshold":                       "200",
+				},
+			},
+		}
+		resolvedAlert := &types.Alert{
+			Alert: model.Alert{
+				Labels:   model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical", "service": "api-server", "instance": "i-0abc123", "region": "us-east-1", "http_method": "GET"},
+				StartsAt: time.Now().Add(-2 * time.Hour),
+				EndsAt:   time.Now().Add(-time.Hour),
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: model.LabelValue(titleTemplate),
+					ruletypes.AnnotationBodyTemplate:  model.LabelValue(bodyTemplate),
+					"value":                           "50",
+					"threshold":                       "200",
+				},
+			},
+		}
+
+		atts, err := notifier.prepareContent(ctx, []*types.Alert{firingAlert, resolvedAlert}, tmplText)
+		require.NoError(t, err)
+
+		// 1 title attachment + 2 body attachments (one per alert)
+		require.Len(t, atts, 3)
+
+		// First attachment: title-only, no color, no blocks
+		require.Equal(t, "[firing] HighCPU — api-server", atts[0].Title)
+		require.Empty(t, atts[0].Color)
+		require.Nil(t, atts[0].Blocks)
+		require.Equal(t, "https://alertmanager.signoz.com", atts[0].TitleLink)
+
+		expectedFiringBody := "*HighCPU*\n\n" +
+			"*Service:* _api-server_\n*Instance:* _i-0abc123_\n*Region:* _us-east-1_\n*Method:* _GET_\n\n" +
+			"---\n\n" +
+			"```\nMetric    | Value\n----------|------\nCurrent   | 100  \nThreshold | 200  \n```\n\n" +
+			"*Status:* firing | *Severity:* critical\n\n"
+
+		expectedResolvedBody := "*HighCPU*\n\n" +
+			"*Service:* _api-server_\n*Instance:* _i-0abc123_\n*Region:* _us-east-1_\n*Method:* _GET_\n\n" +
+			"---\n\n" +
+			"```\nMetric    | Value\n----------|------\nCurrent   | 50   \nThreshold | 200  \n```\n\n" +
+			"*Status:* resolved | *Severity:* critical\n\n"
+
+		// Second attachment: firing alert body rendered as slack mrkdwn text, red color
+		require.Nil(t, atts[1].Blocks)
+		require.Equal(t, "#FF0000", atts[1].Color)
+		require.Equal(t, []string{"text"}, atts[1].MrkdwnIn)
+		require.Equal(t, expectedFiringBody, atts[1].Text)
+
+		// Third attachment: resolved alert body rendered as slack mrkdwn text, green color
+		require.Nil(t, atts[2].Blocks)
+		require.Equal(t, "#00FF00", atts[2].Color)
+		require.Equal(t, []string{"text"}, atts[2].MrkdwnIn)
+		require.Equal(t, expectedResolvedBody, atts[2].Text)
+	})
+
+	t.Run("default template with fields and actions", func(t *testing.T) {
+		// Verifies that addFieldsAndActions (called from Notify after prepareContent)
+		// correctly populates fields and actions on the attachment from config.
+		tmpl := test.CreateTmpl(t)
+		proc := newTestProcessor(tmpl)
+		short := true
+		notifier := &Notifier{
+			conf: &config.SlackConfig{
+				Title: `{{ .CommonLabels.alertname }}`,
+				Text:  "alert text",
+				Color: "warning",
+				Fields: []*config.SlackField{
+					{Title: "Severity", Value: "critical", Short: &short},
+					{Title: "Service", Value: "api-server", Short: &short},
+				},
+				Actions: []*config.SlackAction{
+					{Type: "button", Text: "View Alert", URL: "https://alertmanager.signoz.com"},
+				},
+				TitleLink: "https://alertmanager.signoz.com",
+			},
+			tmpl:      tmpl,
+			logger:    slog.New(slog.DiscardHandler),
+			processor: proc,
+		}
+		tmplText := func(s string) string { return s }
+
+		ctx := setupTestContext()
+		alerts := []*types.Alert{
+			{Alert: model.Alert{
+				Labels:   model.LabelSet{ruletypes.LabelAlertName: "TestAlert"},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			}},
+		}
+		atts, err := notifier.prepareContent(ctx, alerts, tmplText)
+		require.NoError(t, err)
+		require.Len(t, atts, 1)
+
+		// prepareContent does not populate fields/actions — that's done by
+		// addFieldsAndActions which is called from Notify.
+		require.Nil(t, atts[0].Fields)
+		require.Nil(t, atts[0].Actions)
+
+		// Simulate what Notify does after prepareContent
+		notifier.addFieldsAndActions(&atts[0], tmplText)
+
+		// Verify fields
+		require.Len(t, atts[0].Fields, 2)
+		require.Equal(t, "Severity", atts[0].Fields[0].Title)
+		require.Equal(t, "critical", atts[0].Fields[0].Value)
+		require.True(t, *atts[0].Fields[0].Short)
+		require.Equal(t, "Service", atts[0].Fields[1].Title)
+		require.Equal(t, "api-server", atts[0].Fields[1].Value)
+
+		// Verify actions
+		require.Len(t, atts[0].Actions, 1)
+		require.Equal(t, "button", atts[0].Actions[0].Type)
+		require.Equal(t, "View Alert", atts[0].Actions[0].Text)
+		require.Equal(t, "https://alertmanager.signoz.com", atts[0].Actions[0].URL)
+	})
+}
+
+func TestSlackMessageField(t *testing.T) {
+	// 1. Setup a fake Slack server
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatal(err)
+		}
+
+		// 2. VERIFY: Top-level text exists
+		if body["text"] != "My Top Level Message" {
+			t.Errorf("Expected top-level 'text' to be 'My Top Level Message', got %v", body["text"])
+		}
+
+		// 3. VERIFY: Old attachments still exist
+		attachments, ok := body["attachments"].([]any)
+		if !ok || len(attachments) == 0 {
+			t.Errorf("Expected attachments to exist")
+		} else {
+			first := attachments[0].(map[string]any)
+			if first["title"] != "Old Attachment Title" {
+				t.Errorf("Expected attachment title 'Old Attachment Title', got %v", first["title"])
+			}
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"ok": true}`))
+	}))
+	defer server.Close()
+
+	// 4. Configure Notifier with BOTH new and old fields
+	u, _ := url.Parse(server.URL)
+	conf := &config.SlackConfig{
+		APIURL:      &config.SecretURL{URL: u},
+		MessageText: "My Top Level Message", // Your NEW field
+		Title:       "Old Attachment Title", // An OLD field
+		Channel:     "#test-channel",
+		HTTPConfig:  &commoncfg.HTTPClientConfig{},
+	}
+
+	tmpl, err := template.FromGlobs([]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpl.ExternalURL = u
+
+	logger := slog.New(slog.DiscardHandler)
+	notifier, err := New(conf, tmpl, logger, newTestProcessor(tmpl))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group-key")
+
+	if _, err := notifier.Notify(ctx); err != nil {
+		t.Fatal("Notify failed:", err)
+	}
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook.go

@@ -0,0 +1,192 @@
+// Copyright 2019 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "webhook"
+)
+
+// Notifier implements a Notifier for generic webhooks.
+type Notifier struct {
+	conf      *config.WebhookConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	client    *http.Client
+	retrier   *notify.Retrier
+	processor alertmanagertypes.NotificationProcessor
+}
+
+// New returns a new Webhook.
+func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, proc alertmanagertypes.NotificationProcessor, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*conf.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:      conf,
+		tmpl:      t,
+		logger:    l,
+		client:    client,
+		processor: proc,
+		// Webhooks are assumed to respond with 2xx response codes on a successful
+		// request and 5xx response codes are assumed to be recoverable.
+		retrier: &notify.Retrier{},
+	}, nil
+}
+
+// Message defines the JSON object send to webhook endpoints.
+type Message struct {
+	*template.Data
+
+	// The protocol version.
+	Version         string `json:"version"`
+	GroupKey        string `json:"groupKey"`
+	TruncatedAlerts uint64 `json:"truncatedAlerts"`
+}
+
+func truncateAlerts(maxAlerts uint64, alerts []*types.Alert) ([]*types.Alert, uint64) {
+	if maxAlerts != 0 && uint64(len(alerts)) > maxAlerts {
+		return alerts[:maxAlerts], uint64(len(alerts)) - maxAlerts
+	}
+
+	return alerts, 0
+}
+
+// templateAlerts extracts custom templates from alert annotations, processes them,
+// and updates each alert's annotations with the rendered title and body
+// the idea is to send the templated annotations for title and body templates to the webhook endpoint.
+func (n *Notifier) templateAlerts(ctx context.Context, alerts []*types.Alert) error {
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.processor.ProcessAlertNotification(ctx, alertmanagertypes.NotificationProcessorInput{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: "",
+		DefaultBodyTemplate:  "",
+	}, alerts, markdownrenderer.MarkdownFormatNoop)
+	if err != nil {
+		return err
+	}
+
+	for i, alert := range alerts {
+		if alert.Annotations == nil {
+			continue
+		}
+		// Update title_template annotation with rendered title, only if key exists and result is non-blank
+		if _, ok := alert.Annotations[ruletypes.AnnotationTitleTemplate]; ok && result.Title != "" {
+			alert.Annotations[ruletypes.AnnotationTitleTemplate] = model.LabelValue(result.Title)
+		}
+		// Update body_template annotation with rendered body, only if key exists and result is non-blank
+		if _, ok := alert.Annotations[ruletypes.AnnotationBodyTemplate]; ok && i < len(result.Body) && result.Body[i] != "" {
+			alert.Annotations[ruletypes.AnnotationBodyTemplate] = model.LabelValue(result.Body[i])
+		}
+	}
+
+	return nil
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
+	alerts, numTruncated := truncateAlerts(n.conf.MaxAlerts, alerts)
+	// template alerts before preparing the notification data
+	if err := n.templateAlerts(ctx, alerts); err != nil {
+		return false, err
+	}
+	data := notify.GetTemplateData(ctx, n.tmpl, alerts, n.logger)
+
+	groupKey, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	logger := n.logger.With(slog.Any("group_key", groupKey))
+	logger.DebugContext(ctx, "extracted group key")
+
+	msg := &Message{
+		Version:         "4",
+		Data:            data,
+		GroupKey:        groupKey.String(),
+		TruncatedAlerts: numTruncated,
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return false, err
+	}
+
+	var url string
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.URL != "" {
+		url = tmpl(string(n.conf.URL))
+	} else {
+		content, err := os.ReadFile(n.conf.URLFile)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "read url_file")
+		}
+		url = tmpl(strings.TrimSpace(string(content)))
+	}
+
+	if tmplErr != nil {
+		return false, errors.NewInternalf(errors.CodeInternal, "failed to template webhook URL: %v", tmplErr)
+	}
+
+	if url == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "webhook URL is empty after templating")
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeTimeout, "configured webhook timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, url, &buf) //nolint:bodyclose
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.NewInternalf(errors.CodeInternal, "failed to post JSON to webhook: %v", context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return shouldRetry, err
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook_test.go

@@ -0,0 +1,342 @@
+// Copyright 2019 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertnotificationprocessor"
+	"github.com/SigNoz/signoz/pkg/emailing/templatestore/filetemplatestore"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func newTestProcessor(tmpl *template.Template) alertmanagertypes.NotificationProcessor {
+	logger := slog.Default()
+	templater := alertmanagertemplate.New(tmpl, logger)
+	renderer := markdownrenderer.NewMarkdownRenderer(logger)
+	return alertnotificationprocessor.New(templater, renderer, filetemplatestore.NewEmptyStore(), logger)
+}
+
+func TestWebhookRetry(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL("http://example.com"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	if err != nil {
+		require.NoError(t, err)
+	}
+
+	t.Run("test retry status code", func(t *testing.T) {
+		for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+			actual, _ := notifier.retrier.Check(statusCode, nil)
+			require.Equal(t, expected, actual, "error on status %d", statusCode)
+		}
+	})
+
+	t.Run("test retry error details", func(t *testing.T) {
+		for _, tc := range []struct {
+			status int
+			body   io.Reader
+
+			exp string
+		}{
+			{
+				status: http.StatusBadRequest,
+				body: bytes.NewBuffer([]byte(
+					`{"status":"invalid event"}`,
+				)),
+
+				exp: fmt.Sprintf(`unexpected status code %d: {"status":"invalid event"}`, http.StatusBadRequest),
+			},
+			{
+				status: http.StatusBadRequest,
+
+				exp: fmt.Sprintf(`unexpected status code %d`, http.StatusBadRequest),
+			},
+		} {
+			t.Run("", func(t *testing.T) {
+				_, err = notifier.retrier.Check(tc.status, tc.body)
+				require.Equal(t, tc.exp, err.Error())
+			})
+		}
+	})
+}
+
+func TestWebhookTruncateAlerts(t *testing.T) {
+	alerts := make([]*types.Alert, 10)
+
+	truncatedAlerts, numTruncated := truncateAlerts(0, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(4, alerts)
+	require.Len(t, truncatedAlerts, 4)
+	require.EqualValues(t, 6, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(100, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+}
+
+func TestWebhookRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	secret := "secret"
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL(u.String()),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, secret)
+}
+
+func TestWebhookReadingURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "webhook_url")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	tmpl := test.CreateTmpl(t)
+	notifier, err := New(
+		&config.WebhookConfig{
+			URLFile:    f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		promslog.NewNopLogger(),
+		newTestProcessor(tmpl),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestWebhookURLTemplating(t *testing.T) {
+	var calledURL string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calledURL = r.URL.Path
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	tests := []struct {
+		name           string
+		url            string
+		groupLabels    model.LabelSet
+		alertLabels    model.LabelSet
+		expectError    bool
+		expectedErrMsg string
+		expectedPath   string
+	}{
+		{
+			name:         "templating with alert labels",
+			url:          srv.URL + "/{{ .GroupLabels.alertname }}/{{ .CommonLabels.severity }}",
+			groupLabels:  model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:  model.LabelSet{"alertname": "TestAlert", "severity": "critical"},
+			expectError:  false,
+			expectedPath: "/TestAlert/critical",
+		},
+		{
+			name:           "invalid template field",
+			url:            srv.URL + "/{{ .InvalidField }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "failed to template webhook URL",
+		},
+		{
+			name:           "template renders to empty string",
+			url:            "{{ if .CommonLabels.nonexistent }}http://example.com{{ end }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "webhook URL is empty after templating",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			calledURL = "" // Reset for each test
+
+			tmpl := test.CreateTmpl(t)
+			notifier, err := New(
+				&config.WebhookConfig{
+					URL:        config.SecretTemplateURL(tc.url),
+					HTTPConfig: &commoncfg.HTTPClientConfig{},
+				},
+				tmpl,
+				promslog.NewNopLogger(),
+				newTestProcessor(tmpl),
+			)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "test-group")
+			if tc.groupLabels != nil {
+				ctx = notify.WithGroupLabels(ctx, tc.groupLabels)
+			}
+
+			alerts := []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels:   tc.alertLabels,
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}
+
+			_, err = notifier.Notify(ctx, alerts...)
+
+			if tc.expectError {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expectedErrMsg)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedPath, calledURL)
+			}
+		})
+	}
+}
+
+func TestTemplateAlerts(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
+	proc := newTestProcessor(tmpl)
+
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL("http://example.com"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		slog.Default(),
+		proc,
+	)
+	require.NoError(t, err)
+
+	t.Run("annotations are updated with custom title and body templates", func(t *testing.T) {
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"alertname": "TestAlert",
+						"severity":  "critical",
+					},
+					Annotations: model.LabelSet{
+						ruletypes.AnnotationTitleTemplate: "Alert: $labels.alertname",
+						ruletypes.AnnotationBodyTemplate:  "Severity is $labels.severity",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"alertname": "TestAlert",
+						"severity":  "warning",
+					},
+					Annotations: model.LabelSet{
+						ruletypes.AnnotationTitleTemplate: "Alert: $labels.alertname",
+						ruletypes.AnnotationBodyTemplate:  "Severity is $labels.severity",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+		}
+
+		ctx := context.Background()
+		err := notifier.templateAlerts(ctx, alerts)
+		require.NoError(t, err)
+
+		// Both alerts should have their title_template updated to the rendered title
+		require.Equal(t, model.LabelValue("Alert: TestAlert"), alerts[0].Annotations[ruletypes.AnnotationTitleTemplate])
+		require.Equal(t, model.LabelValue("Alert: TestAlert"), alerts[1].Annotations[ruletypes.AnnotationTitleTemplate])
+		// Each alert should have its own body_template based on its labels
+		require.Equal(t, model.LabelValue("Severity is critical"), alerts[0].Annotations[ruletypes.AnnotationBodyTemplate])
+		require.Equal(t, model.LabelValue("Severity is warning"), alerts[1].Annotations[ruletypes.AnnotationBodyTemplate])
+	})
+
+	t.Run("annotations not updated when template keys are absent", func(t *testing.T) {
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"alertname": "NoTemplateAlert",
+					},
+					Annotations: model.LabelSet{
+						"summary":     "keep this",
+						"description": "keep this too",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+		}
+
+		ctx := context.Background()
+		err := notifier.templateAlerts(ctx, alerts)
+		require.NoError(t, err)
+
+		// title_template and body_template keys should NOT be added
+		_, hasTitleTemplate := alerts[0].Annotations[ruletypes.AnnotationTitleTemplate]
+		_, hasBodyTemplate := alerts[0].Annotations[ruletypes.AnnotationBodyTemplate]
+		require.False(t, hasTitleTemplate, "title_template should not be added when absent")
+		require.False(t, hasBodyTemplate, "body_template should not be added when absent")
+
+		// Existing annotations should remain untouched
+		require.Equal(t, model.LabelValue("keep this"), alerts[0].Annotations["summary"])
+		require.Equal(t, model.LabelValue("keep this too"), alerts[0].Annotations["description"])
+	})
+}

pkg/alertmanager/alertmanagerserver/config.go

@@ -28,6 +28,9 @@ type Config struct {
 
 	// Configuration for the notification log.
 	NFLog NFLogConfig `mapstructure:"nflog"`
+
+	// EmailTemplatesDirectory is the directory containing email layout templates (.gotmpl files).
+	EmailTemplatesDirectory string `mapstructure:"email_templates_directory"`
 }
 
 type AlertsConfig struct {
@@ -100,5 +103,6 @@ func NewConfig() Config {
 			MaintenanceInterval: 15 * time.Minute,
 			Retention:           120 * time.Hour,
 		},
+		EmailTemplatesDirectory: "/root/templates/email",
 	}
 }

pkg/alertmanager/alertmanagerserver/server.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prometheus/alertmanager/types"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/prometheus/alertmanager/dispatch"
 	"github.com/prometheus/alertmanager/featurecontrol"
 	"github.com/prometheus/alertmanager/inhibit"
@@ -23,9 +24,13 @@ import (
 	"github.com/prometheus/common/model"
 
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertnotificationprocessor"
 	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager"
-	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/emailing/templatestore/filetemplatestore"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
 )
 
 var (
@@ -66,6 +71,8 @@ type Server struct {
 	pipelineBuilder     *notify.PipelineBuilder
 	marker              *alertmanagertypes.MemMarker
 	tmpl                *template.Template
+	processor           alertmanagertypes.NotificationProcessor
+	emailTemplateStore  emailtypes.TemplateStore
 	wg                  sync.WaitGroup
 	stopc               chan struct{}
 	notificationManager nfmanager.NotificationManager
@@ -198,6 +205,12 @@ func New(ctx context.Context, logger *slog.Logger, registry prometheus.Registere
 
 	server.pipelineBuilder = notify.NewPipelineBuilder(signozRegisterer, featurecontrol.NoopFlags{})
 	server.dispatcherMetrics = NewDispatcherMetrics(false, signozRegisterer)
+	emailTemplateStore, storeErr := filetemplatestore.NewStore(ctx, srvConfig.EmailTemplatesDirectory, emailtypes.Templates, server.logger)
+	if storeErr != nil {
+		server.logger.ErrorContext(ctx, "failed to create alert email template store, using empty store", errors.Attr(storeErr))
+		emailTemplateStore = filetemplatestore.NewEmptyStore()
+	}
+	server.emailTemplateStore = emailTemplateStore
 
 	return server, nil
 }
@@ -234,6 +247,11 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 
 	server.tmpl.ExternalURL = server.srvConfig.ExternalURL
 
+	// Construct the alert notification processor
+	templater := alertmanagertemplate.New(server.tmpl, server.logger)
+	renderer := markdownrenderer.NewMarkdownRenderer(server.logger)
+	server.processor = alertnotificationprocessor.New(templater, renderer, server.emailTemplateStore, server.logger)
+
 	// Build the routing tree and record which receivers are used.
 	routes := dispatch.NewRoute(config.Route, nil)
 	activeReceivers := make(map[string]struct{})
@@ -250,7 +268,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 			server.logger.InfoContext(ctx, "skipping creation of receiver not referenced by any route", slog.String("receiver", rcv.Name))
 			continue
 		}
-		integrations, err := alertmanagernotify.NewReceiverIntegrations(rcv, server.tmpl, server.logger)
+		integrations, err := alertmanagernotify.NewReceiverIntegrations(rcv, server.tmpl, server.logger, server.processor)
 		if err != nil {
 			return err
 		}
@@ -326,7 +344,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 
 func (server *Server) TestReceiver(ctx context.Context, receiver alertmanagertypes.Receiver) error {
 	testAlert := alertmanagertypes.NewTestAlert(receiver, time.Now(), time.Now())
-	return alertmanagertypes.TestReceiver(ctx, receiver, alertmanagernotify.NewReceiverIntegrations, server.alertmanagerConfig, server.tmpl, server.logger, testAlert.Labels, testAlert)
+	return alertmanagertypes.TestReceiver(ctx, receiver, alertmanagernotify.NewReceiverIntegrations, server.alertmanagerConfig, server.tmpl, server.logger, server.processor, testAlert.Labels, testAlert)
 }
 
 func (server *Server) TestAlert(ctx context.Context, receiversMap map[*alertmanagertypes.PostableAlert][]string, config *alertmanagertypes.NotificationConfig) error {
@@ -409,6 +427,7 @@ func (server *Server) TestAlert(ctx context.Context, receiversMap map[*alertmana
 					server.alertmanagerConfig,
 					server.tmpl,
 					server.logger,
+					server.processor,
 					group.groupLabels,
 					group.alerts...,
 				)

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate.go

@@ -0,0 +1,297 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"sort"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+)
+
+// AlertManagerTemplater processes alert notification templates.
+type AlertManagerTemplater interface {
+	// ProcessTemplates expands the title and body templates from input
+	// against the provided alerts and returns the expanded templates.
+	ProcessTemplates(ctx context.Context, input TemplateInput, alerts []*types.Alert) (*ExpandedTemplates, error)
+	// BuildNotificationTemplateData builds the NotificationTemplateData from context and alerts.
+	// This exposes the structured alert data that gets used in the notification templates.
+	BuildNotificationTemplateData(ctx context.Context, alerts []*types.Alert) *NotificationTemplateData
+}
+
+type alertManagerTemplater struct {
+	tmpl   *template.Template
+	logger *slog.Logger
+}
+
+func New(tmpl *template.Template, logger *slog.Logger) AlertManagerTemplater {
+	return &alertManagerTemplater{tmpl: tmpl, logger: logger}
+}
+
+// ProcessTemplates expands the title and body templates from input
+// against the provided alerts and returns the expanded templates.
+func (at *alertManagerTemplater) ProcessTemplates(
+	ctx context.Context,
+	input TemplateInput,
+	alerts []*types.Alert,
+) (*ExpandedTemplates, error) {
+	ntd := at.buildNotificationTemplateData(ctx, alerts)
+	missingVars := make(map[string]bool)
+
+	title, titleMissingVars, err := at.expandTitle(input.TitleTemplate, ntd)
+	if err != nil {
+		return nil, err
+	}
+	// if title template results in empty string, use default template
+	// this happens for old alerts and API users who've not configured custom title annotation
+	if title == "" && input.DefaultTitleTemplate != "" {
+		title, err = at.expandDefaultTemplate(ctx, input.DefaultTitleTemplate, alerts)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		mergeMissingVars(missingVars, titleMissingVars)
+	}
+
+	// isDefaultTemplated tracks whether the body is templated using default templates
+	isDefaultTemplated := false
+	body, bodyMissingVars, err := at.expandBody(input.BodyTemplate, ntd)
+	if err != nil {
+		return nil, err
+	}
+	// if body template results in nil, use default template
+	// this happens for old alerts and API users who've not configured custom body annotation
+	if body == nil {
+		isDefaultTemplated = true
+		defaultBody, err := at.expandDefaultTemplate(ctx, input.DefaultBodyTemplate, alerts)
+		if err != nil {
+			return nil, err
+		}
+		body = []string{defaultBody} // default template result is combined for all alerts
+	} else {
+		mergeMissingVars(missingVars, bodyMissingVars)
+	}
+
+	// convert the internal map to a sorted slice for returning missing variables
+	missingVarsList := make([]string, 0, len(missingVars))
+	for k := range missingVars {
+		missingVarsList = append(missingVarsList, k)
+	}
+	sort.Strings(missingVarsList)
+
+	return &ExpandedTemplates{
+		Title:                  title,
+		Body:                   body,
+		MissingVars:            missingVarsList,
+		IsDefaultTemplatedBody: isDefaultTemplated,
+	}, nil
+}
+
+// BuildNotificationTemplateData builds the NotificationTemplateData from context and alerts.
+func (at *alertManagerTemplater) BuildNotificationTemplateData(
+	ctx context.Context,
+	alerts []*types.Alert,
+) *NotificationTemplateData {
+	return at.buildNotificationTemplateData(ctx, alerts)
+}
+
+// expandDefaultTemplate uses go-template to expand the default template.
+func (at *alertManagerTemplater) expandDefaultTemplate(
+	ctx context.Context,
+	tmplStr string,
+	alerts []*types.Alert,
+) (string, error) {
+	// if even the default template is empty, return empty string
+	// this is possible if user added channel with blank template
+	if tmplStr == "" {
+		at.logger.WarnContext(ctx, "default template is empty")
+		return "", nil
+	}
+	data := notify.GetTemplateData(ctx, at.tmpl, alerts, at.logger)
+	result, err := at.tmpl.ExecuteTextString(tmplStr, data)
+	if err != nil {
+		return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to execute default template: %s", err.Error())
+	}
+	return result, nil
+}
+
+// mergeMissingVars adds all keys from src into dst.
+func mergeMissingVars(dst, src map[string]bool) {
+	for k := range src {
+		dst[k] = true
+	}
+}
+
+// expandTitle expands the title template. Returns empty string if the template is empty.
+func (at *alertManagerTemplater) expandTitle(
+	titleTemplate string,
+	ntd *NotificationTemplateData,
+) (string, map[string]bool, error) {
+	if titleTemplate == "" {
+		return "", nil, nil
+	}
+	processRes, err := PreProcessTemplateAndData(titleTemplate, ntd)
+	if err != nil {
+		return "", nil, err
+	}
+	result, err := at.tmpl.ExecuteTextString(processRes.Template, processRes.Data)
+	if err != nil {
+		return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to execute custom title template: %s", err.Error())
+	}
+	return strings.TrimSpace(result), processRes.UnknownVars, nil
+}
+
+// expandBody expands the body template for each individual alert. Returns nil if the template is empty.
+func (at *alertManagerTemplater) expandBody(
+	bodyTemplate string,
+	ntd *NotificationTemplateData,
+) ([]string, map[string]bool, error) {
+	if bodyTemplate == "" {
+		return nil, nil, nil
+	}
+	var sb []string
+	missingVars := make(map[string]bool)
+	for i := range ntd.Alerts {
+		processRes, err := PreProcessTemplateAndData(bodyTemplate, &ntd.Alerts[i])
+		if err != nil {
+			return nil, nil, err
+		}
+		part, err := at.tmpl.ExecuteTextString(processRes.Template, processRes.Data)
+		if err != nil {
+			return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to execute custom body template: %s", err.Error())
+		}
+		// add unknown variables and templated text to the result
+		for k := range processRes.UnknownVars {
+			missingVars[k] = true
+		}
+		if strings.TrimSpace(part) != "" {
+			sb = append(sb, strings.TrimSpace(part))
+		}
+	}
+	return sb, missingVars, nil
+}
+
+// buildNotificationTemplateData creates the NotificationTemplateData using
+// info from context and the raw alerts.
+func (at *alertManagerTemplater) buildNotificationTemplateData(
+	ctx context.Context,
+	alerts []*types.Alert,
+) *NotificationTemplateData {
+	// extract the required data from the context
+	receiver, ok := notify.ReceiverName(ctx)
+	if !ok {
+		at.logger.WarnContext(ctx, "missing receiver name in context")
+	}
+
+	groupLabels, ok := notify.GroupLabels(ctx)
+	if !ok {
+		at.logger.WarnContext(ctx, "missing group labels in context")
+	}
+
+	// extract the external URL from the template
+	externalURL := ""
+	if at.tmpl.ExternalURL != nil {
+		externalURL = at.tmpl.ExternalURL.String()
+	}
+
+	commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+	commonLabels := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
+
+	// aggregate labels and annotations from all alerts
+	labels := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
+	annotations := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+
+	// build the alert data slice
+	alertDataSlice := make([]AlertData, 0, len(alerts))
+	for _, a := range alerts {
+		ad := buildAlertData(a, receiver)
+		alertDataSlice = append(alertDataSlice, ad)
+	}
+
+	// count the number of firing and resolved alerts
+	var firing, resolved int
+	for _, ad := range alertDataSlice {
+		if ad.IsFiring {
+			firing++
+		} else if ad.IsResolved {
+			resolved++
+		}
+	}
+
+	// extract the rule-level convenience fields from common labels
+	alertName := commonLabels[ruletypes.LabelAlertName]
+	ruleID := commonLabels[ruletypes.LabelRuleId]
+	ruleLink := commonLabels[ruletypes.LabelRuleSource]
+
+	// build the group labels
+	gl := make(template.KV, len(groupLabels))
+	for k, v := range groupLabels {
+		gl[string(k)] = string(v)
+	}
+
+	// build the notification template data
+	return &NotificationTemplateData{
+		Receiver:          receiver,
+		Status:            string(types.Alerts(alerts...).Status()),
+		AlertName:         alertName,
+		RuleID:            ruleID,
+		RuleLink:          ruleLink,
+		TotalFiring:       firing,
+		TotalResolved:     resolved,
+		Alerts:            alertDataSlice,
+		GroupLabels:       gl,
+		CommonLabels:      commonLabels,
+		CommonAnnotations: commonAnnotations,
+		ExternalURL:       externalURL,
+		Labels:            labels,
+		Annotations:       annotations,
+	}
+}
+
+// buildAlertData converts a single *types.Alert into an AlertData.
+func buildAlertData(a *types.Alert, receiver string) AlertData {
+	labels := make(template.KV, len(a.Labels))
+	for k, v := range a.Labels {
+		labels[string(k)] = string(v)
+	}
+
+	annotations := make(template.KV, len(a.Annotations))
+	for k, v := range a.Annotations {
+		annotations[string(k)] = string(v)
+	}
+
+	status := string(a.Status())
+	isFiring := a.Status() == model.AlertFiring
+	isResolved := a.Status() == model.AlertResolved
+	isMissingData := labels[ruletypes.LabelNoData] == "true"
+
+	return AlertData{
+		Receiver:      receiver,
+		Status:        status,
+		Labels:        labels,
+		Annotations:   annotations,
+		StartsAt:      a.StartsAt,
+		EndsAt:        a.EndsAt,
+		GeneratorURL:  a.GeneratorURL,
+		Fingerprint:   a.Fingerprint().String(),
+		AlertName:     labels[ruletypes.LabelAlertName],
+		RuleID:        labels[ruletypes.LabelRuleId],
+		RuleLink:      labels[ruletypes.LabelRuleSource],
+		Severity:      labels[ruletypes.LabelSeverityName],
+		LogLink:       annotations[ruletypes.AnnotationRelatedLogs],
+		TraceLink:     annotations[ruletypes.AnnotationRelatedTraces],
+		Value:         annotations[ruletypes.AnnotationValue],
+		Threshold:     annotations[ruletypes.AnnotationThresholdValue],
+		CompareOp:     annotations[ruletypes.AnnotationCompareOp],
+		MatchType:     annotations[ruletypes.AnnotationMatchType],
+		IsFiring:      isFiring,
+		IsResolved:    isResolved,
+		IsMissingData: isMissingData,
+		IsRecovering:  labels[ruletypes.LabelIsRecovering] == "true",
+	}
+}

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate_test.go

@@ -0,0 +1,283 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"sort"
+	"testing"
+	"time"
+
+	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// testSetup returns an AlertTemplater and a context pre-populated with group key,
+// receiver name, and group labels for use in tests.
+func testSetup(t *testing.T) (AlertManagerTemplater, context.Context) {
+	t.Helper()
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "slack")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": "TestAlert", "severity": "critical"})
+	logger := slog.New(slog.DiscardHandler)
+	return New(tmpl, logger), ctx
+}
+
+func createAlert(labels, annotations map[string]string, isFiring bool) *types.Alert {
+	ls := model.LabelSet{}
+	for k, v := range labels {
+		ls[model.LabelName(k)] = model.LabelValue(v)
+	}
+	ann := model.LabelSet{}
+	for k, v := range annotations {
+		ann[model.LabelName(k)] = model.LabelValue(v)
+	}
+	startsAt := time.Now()
+	var endsAt time.Time
+	if isFiring {
+		endsAt = startsAt.Add(time.Hour)
+	} else {
+		startsAt = startsAt.Add(-2 * time.Hour)
+		endsAt = startsAt.Add(-time.Hour)
+	}
+	return &types.Alert{Alert: model.Alert{Labels: ls, Annotations: ann, StartsAt: startsAt, EndsAt: endsAt}}
+}
+
+func TestExpandTemplates(t *testing.T) {
+	at, ctx := testSetup(t)
+
+	tests := []struct {
+		name              string
+		alerts            []*types.Alert
+		input             TemplateInput
+		wantTitle         string
+		wantBody          []string
+		wantMissingVars   []string
+		errorContains     string
+		wantIsDefaultBody bool
+	}{
+		{
+			// High request throughput on a service — service is a custom label.
+			// $labels.service extracts the label value; $annotations.description pulls the annotation.
+			name: "new template: high request throughput for a service",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "HighRequestThroughput",
+						ruletypes.LabelSeverityName: "warning",
+						"service":                   "payment-service",
+					},
+					map[string]string{"description": "Request rate exceeded 10k/s"},
+					true,
+				),
+			},
+			input: TemplateInput{
+				TitleTemplate: "High request throughput for $service",
+				BodyTemplate: `The service $service is getting high request. Please investigate.
+Severity: $severity
+Status: $status
+Service: $service
+Description: $description`,
+			},
+			wantTitle: "High request throughput for payment-service",
+			wantBody: []string{`The service payment-service is getting high request. Please investigate.
+Severity: warning
+Status: firing
+Service: payment-service
+Description: Request rate exceeded 10k/s`},
+			wantIsDefaultBody: false,
+		},
+		{
+			// Disk usage alert using old Go template syntax throughout.
+			// No custom templates — both title and body use the default fallback path.
+			name: "old template: disk usage high on database host",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "DiskUsageHigh",
+						ruletypes.LabelSeverityName: "critical",
+						"instance":                  "db-primary-01",
+					},
+					map[string]string{
+						"summary":        "Disk usage high on database host",
+						"description":    "Disk usage is high on the database host",
+						"related_logs":   "https://logs.example.com/search?q=DiskUsageHigh",
+						"related_traces": "https://traces.example.com/search?q=DiskUsageHigh",
+					},
+					true,
+				),
+			},
+			input: TemplateInput{
+				DefaultTitleTemplate: `[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
+     {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
+       {{" "}}(
+       {{- with .CommonLabels.Remove .GroupLabels.Names }}
+         {{- range $index, $label := .SortedPairs -}}
+           {{ if $index }}, {{ end }}
+           {{- $label.Name }}="{{ $label.Value -}}"
+         {{- end }}
+       {{- end -}}
+       )
+     {{- end }}`,
+				DefaultBodyTemplate: `{{ range .Alerts -}}
+     *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}
+
+     *Summary:* {{ .Annotations.summary }}
+     *Description:* {{ .Annotations.description }}
+     *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}
+     *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}
+
+     *Details:*
+       {{ range .Labels.SortedPairs }} • *{{ .Name }}:* {{ .Value }}
+       {{ end }}
+     {{ end }}`,
+			},
+			wantTitle: "[FIRING:1] DiskUsageHigh for  (instance=\"db-primary-01\")",
+			wantBody: []string{`*Alert:* DiskUsageHigh - critical
+
+     *Summary:* Disk usage high on database host
+     *Description:* Disk usage is high on the database host
+     *RelatedLogs:* View in <https://logs.example.com/search?q=DiskUsageHigh|logs explorer>
+     *RelatedTraces:* View in <https://traces.example.com/search?q=DiskUsageHigh|traces explorer>
+
+     *Details:*
+        • *alertname:* DiskUsageHigh
+        • *instance:* db-primary-01
+        • *severity:* critical
+       
+     `},
+			wantIsDefaultBody: true,
+		},
+		{
+			// Pod crash loop on multiple pods — body is expanded once per alert
+			// and joined with "\n\n", with the pod name pulled from labels.
+			name: "new template: pod crash loop on multiple pods, body per-alert",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-1"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-2"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-3"}, nil, true),
+			},
+			input: TemplateInput{
+				TitleTemplate: "$rule_name: $total_firing pods affected",
+				BodyTemplate:  "$labels.pod is crash looping",
+			},
+			wantTitle:         "PodCrashLoop: 3 pods affected",
+			wantBody:          []string{"api-worker-1 is crash looping", "api-worker-2 is crash looping", "api-worker-3 is crash looping"},
+			wantIsDefaultBody: false,
+		},
+		{
+			// Incident partially resolved — one service still down, one recovered.
+			// Title shows the aggregate counts; body shows per-service status.
+			name: "new template: service degradation with mixed firing and resolved alerts",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "ServiceDown", "service": "auth-service"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "ServiceDown", "service": "payment-service"}, nil, false),
+			},
+			input: TemplateInput{
+				TitleTemplate: "$total_firing firing, $total_resolved resolved",
+				BodyTemplate:  "$labels.service ($status)",
+			},
+			wantTitle:         "1 firing, 1 resolved",
+			wantBody:          []string{"auth-service (firing)", "payment-service (resolved)"},
+			wantIsDefaultBody: false,
+		},
+		{
+			// $environment is not a known AlertData or NotificationTemplateData field,
+			// so it lands in MissingVars and renders as "<no value>" in the output.
+			name: "missing vars: unknown $environment variable in title",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: TemplateInput{
+				TitleTemplate: "[$environment] $rule_name",
+			},
+			wantTitle:         "[<no value>] HighCPU",
+			wantMissingVars:   []string{"environment"},
+			wantIsDefaultBody: true,
+		},
+		{
+			// $runbook_url is not a known field — someone tried to embed a runbook link
+			// directly as a variable instead of via annotations.
+			name: "missing vars: unknown $runbook_url variable in body",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodOOMKilled", ruletypes.LabelSeverityName: "warning"}, nil, true),
+			},
+			input: TemplateInput{
+				BodyTemplate: "$rule_name: see runbook at $runbook_url",
+			},
+			wantBody:        []string{"PodOOMKilled: see runbook at <no value>"},
+			wantMissingVars: []string{"runbook_url"},
+		},
+		{
+			// Both title and body use unknown variables; MissingVars is the union of both.
+			name: "missing vars: unknown variables in both title and body",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighMemory", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: TemplateInput{
+				TitleTemplate: "[$environment] $rule_name and [{{ $service }}]",
+				BodyTemplate:  "$rule_name: see runbook at $runbook_url",
+			},
+			wantTitle:       "[<no value>] HighMemory and [<no value>]",
+			wantBody:        []string{"HighMemory: see runbook at <no value>"},
+			wantMissingVars: []string{"environment", "runbook_url", "service"},
+		},
+		{
+			// Custom title template that expands to only whitespace triggers the fallback,
+			// so the default title template is used instead.
+			name: "fallback: whitespace-only custom title falls back to default",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, false),
+			},
+			input: TemplateInput{
+				TitleTemplate:        "   ",
+				DefaultTitleTemplate: "{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})",
+				DefaultBodyTemplate:  "Runbook: https://runbook.example.com",
+			},
+			wantTitle:         "HighCPU (RESOLVED)",
+			wantBody:          []string{"Runbook: https://runbook.example.com"},
+			wantIsDefaultBody: true,
+		},
+		{
+			name: "using non-existing function in template",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: TemplateInput{
+				TitleTemplate: "$rule_name ({{$severity | toUpperAndTrim}}) for $alertname",
+			},
+			errorContains: "function \"toUpperAndTrim\" not defined",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := at.ProcessTemplates(ctx, tc.input, tc.alerts)
+			if tc.errorContains != "" {
+				require.ErrorContains(t, err, tc.errorContains)
+				return
+			}
+			require.NoError(t, err)
+
+			if tc.wantTitle != "" {
+				require.Equal(t, tc.wantTitle, got.Title)
+			}
+			if tc.wantBody != nil {
+				require.Equal(t, tc.wantBody, got.Body)
+			}
+			require.Equal(t, tc.wantIsDefaultBody, got.IsDefaultTemplatedBody)
+
+			if len(tc.wantMissingVars) == 0 {
+				require.Empty(t, got.MissingVars)
+			} else {
+				sort.Strings(tc.wantMissingVars)
+				require.Equal(t, tc.wantMissingVars, got.MissingVars)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/preprocessor.go

@@ -0,0 +1,271 @@
+package alertmanagertemplate
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/go-viper/mapstructure/v2"
+)
+
+// fieldMapping represents a mapping from a JSON tag name to its struct field name.
+type fieldMapping struct {
+	VarName   string // JSON tag name (e.g., "receiver", "rule_name")
+	FieldName string // Struct field name (e.g., "Receiver", "AlertName")
+}
+
+// extractFieldMappings uses reflection to extract field mappings from a struct.
+func extractFieldMappings(data any) []fieldMapping {
+	val := reflect.ValueOf(data)
+	// Handle pointer types
+	if val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return nil
+		}
+		val = val.Elem()
+	}
+	// return nil if the given data is not a struct
+	if val.Kind() != reflect.Struct {
+		return nil
+	}
+	typ := val.Type()
+
+	var mappings []fieldMapping
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		// Skip unexported fields
+		if !field.IsExported() {
+			continue
+		}
+		// Get JSON tag name
+		jsonTag := field.Tag.Get("json")
+		if jsonTag == "" || jsonTag == "-" {
+			continue
+		}
+		// Extract the name part (before any comma options like omitempty)
+		varName := strings.Split(jsonTag, ",")[0]
+		if varName == "" {
+			continue
+		}
+		varFieldName := field.Tag.Get("mapstructure")
+		if varFieldName == "" {
+			varFieldName = field.Name
+		}
+		// Skip complex types: slices and interfaces
+		kind := field.Type.Kind()
+		if kind == reflect.Slice || kind == reflect.Interface {
+			continue
+		}
+		// For struct types, we skip all but with few exceptions like time.Time
+		if kind == reflect.Struct {
+			// Allow time.Time which is commonly used
+			if field.Type.String() != "time.Time" {
+				continue
+			}
+		}
+
+		mappings = append(mappings, fieldMapping{
+			VarName:   varName,
+			FieldName: varFieldName,
+		})
+	}
+
+	return mappings
+}
+
+// prepareVariableName prepares the variable name to be used in go-text-template
+// it replaces every unwanted character like dots, spaces, etc. with an underscore
+// for example, "http.request.method" becomes "http_request_method"
+func prepareVariableName(key string) string {
+	var b strings.Builder
+	b.Grow(len(key))
+
+	for i, r := range key {
+		switch {
+		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r == '_': // valid variable name characters
+			b.WriteRune(r)
+		case r >= '0' && r <= '9':
+			if b.Len() == 0 {
+				// leading digit — replace with underscore
+				b.WriteByte('_')
+			} else {
+				b.WriteRune(r)
+			}
+		default:
+			// dots, hyphens, spaces, etc. → underscore
+			b.WriteByte('_')
+		}
+		_ = i
+	}
+	return b.String()
+}
+
+// extractNestedFieldsDefinitions adds the labels and annotations keys from the data struct to the template variable definitions
+// it takes the known data struct and extracts the labels and annotations maps and adds their keys to template variable definitions to be used in the template
+func extractNestedFieldsDefinitions(data any) map[string]string {
+	variables := make(map[string]string)
+
+	addLabelsAndAnnotations := func(labels, annotations map[string]string) {
+		for k := range annotations {
+			variables[prepareVariableName(k)] = fmt.Sprintf("index .annotations \"%s\"", k)
+		}
+		for k := range labels {
+			variables[prepareVariableName(k)] = fmt.Sprintf("index .labels \"%s\"", k)
+		}
+	}
+
+	switch data := data.(type) {
+	case *NotificationTemplateData:
+		addLabelsAndAnnotations(data.Labels, data.Annotations)
+	case *AlertData:
+		addLabelsAndAnnotations(data.Labels, data.Annotations)
+	default:
+		return variables
+	}
+
+	return variables
+}
+
+// prepareDataForTemplating prepares the data for templating by adding the labels and annotations values to the resulting map
+// so they can be accessed directly from root level, the predefined values takes precedence over the labels and annotations values
+// for example, if labels has a value called rule_name, which collides with the rule_name field in the data struct, the value from the data struct will take precedence
+func prepareDataForTemplating(data any) (map[string]interface{}, error) {
+	var result map[string]interface{}
+	if err := mapstructure.Decode(data, &result); err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to prepare data for templating")
+	}
+
+	addLabelsAndAnnotationsValues := func(labels, annotations map[string]string) {
+		for k, v := range labels {
+			k = prepareVariableName(k)
+			if _, ok := result[k]; !ok {
+				result[k] = v
+			}
+		}
+		for k, v := range annotations {
+			k = prepareVariableName(k)
+			if _, ok := result[k]; !ok {
+				result[k] = v
+			}
+		}
+	}
+
+	switch data := data.(type) {
+	case *NotificationTemplateData:
+		addLabelsAndAnnotationsValues(data.Labels, data.Annotations)
+	case *AlertData:
+		addLabelsAndAnnotationsValues(data.Labels, data.Annotations)
+	default:
+		return result, nil
+	}
+
+	return result, nil
+}
+
+// generateVariableDefinitions creates `{{ $varname := "" }}` declarations for each variable name.
+func generateVariableDefinitions(varNames map[string]string) string {
+	if len(varNames) == 0 {
+		return ""
+	}
+	var sb strings.Builder
+	for name := range varNames {
+		fmt.Fprintf(&sb, `{{ $%s := %s }}`, name, varNames[name])
+	}
+	return sb.String()
+}
+
+// buildVariableDefinitions constructs the full variable definition preamble for a template.
+// containing all known and unknown variables, the reason to include unknown variables is to
+// populate them with "<no value>" in template so go-text-template don't throw errors
+// when these variables are used in the template.
+func buildVariableDefinitions(tmpl string, data any) (string, map[string]bool, error) {
+	// Extract the initial fields from the data struct and add to the definitions
+	mappings := extractFieldMappings(data)
+
+	// Add variables from struct root level fields to the definitions
+	variables := make(map[string]string)
+	for _, m := range mappings {
+		variables[m.VarName] = fmt.Sprintf(".%s", m.FieldName)
+	}
+
+	// Extract the nested fields definitions from the data struct, like labels, annotations, etc.
+	// once extracted we add them to the variables map along with the field address
+	nestedVariables := extractNestedFieldsDefinitions(data)
+	for k, v := range nestedVariables {
+		variables[k] = v
+	}
+
+	// variables that are used throughout the template
+	usedVars, err := ExtractUsedVariables(tmpl)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// Compute unknown variables: used in template but not covered by a field mapping
+	probableUnknownVars := make(map[string]bool)
+	for name := range usedVars {
+		_, ok := variables[name]
+		if !ok {
+			probableUnknownVars[name] = true
+		}
+	}
+
+	// Add missing variables to the definitions with "<no value>"
+	// missingkey=zero is used to replace the missing value with "<no value>"
+	// but it only works when getting map values like {{ .keyfrommap }} from map and in struct this breaks
+	// with missing variable errors, we add missing variables in map so when directly variables
+	// are accessed directly in template block like {{ $variable }} it's handled and doesn't throw errors.
+	for name := range probableUnknownVars {
+		variables[name] = `"<no value>"`
+	}
+
+	return generateVariableDefinitions(variables), probableUnknownVars, nil
+}
+
+type ProcessingResult struct {
+	Template string
+	Data     map[string]interface{}
+	// UnknownVars is the set of possible unknown variables exptracted using regex
+	UnknownVars map[string]bool
+}
+
+// PreProcessTemplateAndData prepares a template string and struct data for Go template execution.
+//
+//	Input:  "$receiver has $rule_name in $status state"
+//	Output: "{{ $receiver := .Receiver }}...{{ $receiver }} has {{ $rule_name }} in {{ $status }} state"
+func PreProcessTemplateAndData(tmpl string, data any) (*ProcessingResult, error) {
+	// Handle empty template
+	unknownVars := make(map[string]bool)
+	if tmpl == "" {
+		result, err := prepareDataForTemplating(data)
+		if err != nil {
+			return nil, err
+		}
+		return &ProcessingResult{Data: result, UnknownVars: unknownVars}, nil
+	}
+
+	// Build variable definitions: known struct fields + fallback empty-string declarations
+	definitions, unknownVars, err := buildVariableDefinitions(tmpl, data)
+	if err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to build template definitions")
+	}
+
+	// Attach definitions prefix so WrapDollarVariables can parse the AST without "undefined variable" errors.
+	finalTmpl := definitions + tmpl
+
+	// Call WrapDollarVariables to transform bare $variable references to go-text-template format
+	// with {{ $variable }} syntax from $variable syntax
+	wrappedTmpl, err := WrapDollarVariables(finalTmpl)
+	if err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to prepare template for templating")
+	}
+
+	// Convert struct to map using mapstructure to be used for template execution
+	result, err := prepareDataForTemplating(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ProcessingResult{Template: wrappedTmpl, Data: result, UnknownVars: unknownVars}, nil
+}

pkg/alertmanager/alertmanagertemplate/preprocessor_test.go

@@ -0,0 +1,327 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/alertmanager/template"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExtractFieldMappings(t *testing.T) {
+	// Struct with various field types to test extraction logic
+	type TestStruct struct {
+		Name            string    `json:"name"`
+		Status          string    `json:"status"`
+		ActiveUserCount int       `json:"user_count" mapstructure:"active_user_count"`
+		IsActive        bool      `json:"is_active"`
+		CreatedAt       time.Time `json:"created_at"` // time.Time allowed
+		Items           []string  `json:"items"`      // slice skipped
+		unexported      string    // unexported skipped (no tag needed)
+		NoTag           string    // no json tag skipped
+		SkippedTag      string    `json:"-"` // json:"-" skipped
+	}
+
+	testCases := []struct {
+		name     string
+		data     any
+		expected []fieldMapping
+	}{
+		{
+			name: "struct with mixed field types",
+			data: TestStruct{Name: "test", ActiveUserCount: 5, unexported: ""},
+			expected: []fieldMapping{
+				{VarName: "name", FieldName: "Name"},
+				{VarName: "status", FieldName: "Status"},
+				{VarName: "user_count", FieldName: "active_user_count"},
+				{VarName: "is_active", FieldName: "IsActive"},
+				{VarName: "created_at", FieldName: "CreatedAt"},
+			},
+		},
+		{
+			name:     "nil data",
+			data:     nil,
+			expected: nil,
+		},
+		{
+			name:     "non-struct type",
+			data:     "string",
+			expected: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := extractFieldMappings(tc.data)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestBuildVariableDefinitions(t *testing.T) {
+	testCases := []struct {
+		name         string
+		tmpl         string
+		data         any
+		expectedVars []string // substrings that must appear in result
+		expectError  bool
+	}{
+		{
+			name: "empty template still returns struct field definitions",
+			tmpl: "",
+			data: &NotificationTemplateData{Receiver: "test"},
+			expectedVars: []string{
+				"{{ $receiver := .receiver }}",
+				"{{ $status := .status }}",
+			},
+		},
+		{
+			name: "mix of known and unknown vars",
+			tmpl: "$rule_name: $custom_label",
+			data: &AlertData{AlertName: "test", Status: "ok", Severity: "critical"},
+			expectedVars: []string{
+				"{{ $rule_name := .rule_name }}",
+				"{{ $status := .status }}",
+				"{{ $severity := .severity }}",
+				`{{ $custom_label := "<no value>" }}`,
+			},
+		},
+		{
+			name: "nested fields definitions coming from NotificationTemplateData",
+			tmpl: "$severity for $service",
+			data: &NotificationTemplateData{Labels: template.KV{
+				"severity":              "critical",
+				"service":               "test",
+				"cloud.region.instance": "ap-south-1",
+			}},
+			expectedVars: []string{
+				"{{ $severity := index .labels \"severity\" }}",
+				"{{ $service := index .labels \"service\" }}",
+				"{{ $cloud_region_instance := index .labels \"cloud.region.instance\" }}",
+			},
+		},
+		{
+			name: "nested fields definitions coming from AlertData",
+			tmpl: "$severity for $service",
+			data: &AlertData{Labels: template.KV{
+				"severity": "critical",
+				"service":  "test",
+			}},
+			expectedVars: []string{
+				"{{ $severity := index .labels \"severity\" }}",
+				"{{ $service := index .labels \"service\" }}",
+			},
+		},
+		{
+			name:        "invalid template syntax returns error",
+			tmpl:        "{{invalid",
+			data:        &NotificationTemplateData{},
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, _, err := buildVariableDefinitions(tc.tmpl, tc.data)
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			if len(tc.expectedVars) == 0 {
+				require.Empty(t, result)
+				return
+			}
+			for _, expected := range tc.expectedVars {
+				require.Contains(t, result, expected)
+			}
+		})
+	}
+}
+
+func TestPreProcessTemplateAndData(t *testing.T) {
+	testCases := []struct {
+		name                     string
+		tmpl                     string
+		data                     any
+		expectedTemplateContains []string
+		expectedData             map[string]any
+		expectedUnknownVars      map[string]bool
+		expectError              bool
+	}{
+		{
+			name: "NotificationTemplateData with dollar variables and labels with dots and hyphens",
+			tmpl: "[$status] $rule_name (ID: $rule_id) - Firing: $total_firing, Resolved: $total_resolved, Severity: $severity\nHTTP method is: $http_request_method\nRequest path is: $http_request_path",
+			data: &NotificationTemplateData{
+				Receiver:  "pagerduty",
+				Status:    "firing",
+				AlertName: "HighMemory",
+				RuleID:    "rule-123",
+				Labels: template.KV{
+					"severity":            "critical",
+					"http.request.method": "GET",
+					"http-request-path":   "/api/v1/metrics",
+				},
+				TotalFiring:   3,
+				TotalResolved: 1,
+			},
+			expectedTemplateContains: []string{
+				"{{$status := .status}}",
+				"{{$rule_name := .rule_name}}",
+				"{{$rule_id := .rule_id}}",
+				"{{$total_firing := .total_firing}}",
+				"{{$total_resolved := .total_resolved}}",
+				"{{$severity := index .labels \"severity\"}}",
+				"[{{ .status }}] {{ .rule_name }} (ID: {{ .rule_id }}) - Firing: {{ .total_firing }}, Resolved: {{ .total_resolved }}",
+				"{{$http_request_method := index .labels \"http.request.method\"}}",
+				"{{$http_request_path := index .labels \"http-request-path\"}}",
+			},
+			expectedData: map[string]any{
+				"status":              "firing",
+				"rule_name":           "HighMemory",
+				"rule_id":             "rule-123",
+				"total_firing":        3,
+				"total_resolved":      1,
+				"severity":            "critical",
+				"http_request_method": "GET",
+				"http_request_path":   "/api/v1/metrics",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "AlertData with dollar variables",
+			tmpl: "$rule_name: Value $value exceeded $threshold (Status: $status, Severity: $severity, Description: $description)",
+			data: &AlertData{
+				Receiver:  "webhook",
+				Status:    "resolved",
+				AlertName: "DiskFull",
+				RuleID:    "disk-001",
+				Severity:  "warning",
+				Annotations: template.KV{
+					"description": "Disk full and cannot be written to",
+				},
+				Value:      "85%",
+				Threshold:  "80%",
+				IsFiring:   false,
+				IsResolved: true,
+			},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				"{{$value := .value}}",
+				"{{$threshold := .threshold}}",
+				"{{$status := .status}}",
+				"{{$severity := .severity}}",
+				"{{$description := index .annotations \"description\"}}",
+				"{{ .rule_name }}: Value {{ .value }} exceeded {{ .threshold }} (Status: {{ .status }}, Severity: {{ .severity }}, Description: {{ .description }})",
+			},
+			expectedData: map[string]any{
+				"status":      "resolved",
+				"rule_name":   "DiskFull",
+				"rule_id":     "disk-001",
+				"severity":    "warning",
+				"value":       "85%",
+				"threshold":   "80%",
+				"description": "Disk full and cannot be written to",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "mixed dollar and dot notation with both labels and annotations",
+			tmpl: "Alert $rule_name has {{.total_firing}} firing alerts",
+			data: &NotificationTemplateData{
+				AlertName:   "HighCPU",
+				TotalFiring: 5,
+				Labels: template.KV{
+					"value":      "<MASKED VALUE>",
+					"cpu.number": "10",
+				},
+				Annotations: template.KV{
+					"value": "85%",
+				},
+			},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				"{{$value := index .labels \"value\"}}",
+				"Alert {{ .rule_name }} has {{.total_firing}} firing alerts",
+				"{{$cpu_number := index .labels \"cpu.number\"}}",
+			},
+			expectedData: map[string]any{
+				"rule_name":    "HighCPU",
+				"total_firing": 5,
+				"value":        "<MASKED VALUE>",
+				"cpu_number":   "10",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "empty template",
+			tmpl: "",
+			data: &NotificationTemplateData{Receiver: "slack"},
+		},
+		{
+			name:        "invalid template syntax",
+			tmpl:        "{{invalid",
+			data:        &NotificationTemplateData{},
+			expectError: true,
+		},
+		{
+			name: "unknown dollar var in text renders empty",
+			tmpl: "alert $custom_note fired",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				`{{$custom_note := "<no value>"}}`,
+				"alert {{ .custom_note }} fired",
+			},
+			expectedUnknownVars: map[string]bool{"custom_note": true},
+		},
+		{
+			name: "unknown dollar var in action block renders empty",
+			tmpl: "alert {{ $custom_note }} fired",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				`{{$custom_note := "<no value>"}}`,
+				`alert {{$custom_note}} fired`,
+			},
+			expectedUnknownVars: map[string]bool{"custom_note": true},
+		},
+		{
+			name: "mix of known and unknown vars",
+			tmpl: "$rule_name: $custom_label",
+			data: &NotificationTemplateData{AlertName: "HighCPU"},
+			expectedTemplateContains: []string{
+				"{{$rule_name := .rule_name}}",
+				`{{$custom_label := "<no value>"}}`,
+				"{{ .rule_name }}: {{ .custom_label }}",
+			},
+			expectedData:        map[string]any{"rule_name": "HighCPU"},
+			expectedUnknownVars: map[string]bool{"custom_label": true},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := PreProcessTemplateAndData(tc.tmpl, tc.data)
+
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			if tc.tmpl == "" {
+				require.Equal(t, "", result.Template)
+				return
+			}
+
+			for _, substr := range tc.expectedTemplateContains {
+				require.Contains(t, result.Template, substr)
+			}
+			for k, v := range tc.expectedData {
+				require.Equal(t, v, result.Data[k])
+			}
+			if tc.expectedUnknownVars != nil {
+				require.Equal(t, tc.expectedUnknownVars, result.UnknownVars)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/types.go

@@ -0,0 +1,90 @@
+package alertmanagertemplate
+
+import (
+	"time"
+
+	"github.com/prometheus/alertmanager/template"
+)
+
+// TemplateInput carries the title/body templates
+// and their defaults to apply in case the custom templates
+// are result in empty strings.
+type TemplateInput struct {
+	TitleTemplate        string
+	BodyTemplate         string
+	DefaultTitleTemplate string
+	DefaultBodyTemplate  string
+}
+
+// ExpandedTemplates is the result of ExpandTemplates.
+type ExpandedTemplates struct {
+	Title string
+	// Body is notification array of body for each alert
+	Body []string
+	// IsDefaultTemplatedBody is true if the body templates are templated using
+	// default templates, false when custom templates were used for templating.
+	IsDefaultTemplatedBody bool
+	MissingVars            []string // union of unknown vars from title + body templates
+}
+
+// AlertData holds per-alert data used when expanding body templates
+type AlertData struct {
+	Receiver     string      `json:"receiver" mapstructure:"receiver"`
+	Status       string      `json:"status" mapstructure:"status"`
+	Labels       template.KV `json:"labels" mapstructure:"labels"`
+	Annotations  template.KV `json:"annotations" mapstructure:"annotations"`
+	StartsAt     time.Time   `json:"starts_at" mapstructure:"starts_at"`
+	EndsAt       time.Time   `json:"ends_at" mapstructure:"ends_at"`
+	GeneratorURL string      `json:"generator_url" mapstructure:"generator_url"`
+	Fingerprint  string      `json:"fingerprint" mapstructure:"fingerprint"`
+
+	// Convenience fields extracted from well-known labels/annotations.
+	AlertName string `json:"rule_name" mapstructure:"rule_name"`
+	RuleID    string `json:"rule_id" mapstructure:"rule_id"`
+	RuleLink  string `json:"rule_link" mapstructure:"rule_link"`
+	Severity  string `json:"severity" mapstructure:"severity"`
+
+	// Alert internal data fields
+	Value     string `json:"value" mapstructure:"value"`
+	Threshold string `json:"threshold" mapstructure:"threshold"`
+	CompareOp string `json:"compare_op" mapstructure:"compare_op"`
+	MatchType string `json:"match_type" mapstructure:"match_type"`
+
+	// Link annotations added by the rule evaluator.
+	LogLink   string `json:"log_link" mapstructure:"log_link"`
+	TraceLink string `json:"trace_link" mapstructure:"trace_link"`
+
+	// Status booleans for easy conditional templating.
+	IsFiring      bool `json:"is_firing" mapstructure:"is_firing"`
+	IsResolved    bool `json:"is_resolved" mapstructure:"is_resolved"`
+	IsMissingData bool `json:"is_missing_data" mapstructure:"is_missing_data"`
+	IsRecovering  bool `json:"is_recovering" mapstructure:"is_recovering"`
+}
+
+// NotificationTemplateData is the top-level data struct provided to custom templates.
+type NotificationTemplateData struct {
+	Receiver string `json:"receiver" mapstructure:"receiver"`
+	Status   string `json:"status" mapstructure:"status"`
+
+	// Convenience fields for title templates.
+	AlertName     string `json:"rule_name" mapstructure:"rule_name"`
+	RuleID        string `json:"rule_id" mapstructure:"rule_id"`
+	RuleLink      string `json:"rule_link" mapstructure:"rule_link"`
+	TotalFiring   int    `json:"total_firing" mapstructure:"total_firing"`
+	TotalResolved int    `json:"total_resolved" mapstructure:"total_resolved"`
+
+	// Per-alert data, also available as filtered sub-slices.
+	Alerts []AlertData `json:"-" mapstructure:"-"`
+
+	// Cross-alert aggregates, computed as intersection across all alerts.
+	GroupLabels       template.KV `json:"group_labels" mapstructure:"group_labels"`
+	CommonLabels      template.KV `json:"common_labels" mapstructure:"common_labels"`
+	CommonAnnotations template.KV `json:"common_annotations" mapstructure:"common_annotations"`
+	ExternalURL       string      `json:"external_url" mapstructure:"external_url"`
+	// Labels and Annotations that are collection of labels
+	// and annotations from all alerts, it includes only the common labels and annotations
+	// and for non-common labels and annotations, it picks some first few labels/annotations
+	// and joins them with ", " to avoid blank values in the template
+	Labels      template.KV `json:"labels" mapstructure:"labels"`
+	Annotations template.KV `json:"annotations" mapstructure:"annotations"`
+}

pkg/alertmanager/alertmanagertemplate/utils.go

@@ -0,0 +1,231 @@
+package alertmanagertemplate
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+	"text/template/parse"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+)
+
+// maxAggregatedValues is the maximum number of unique values to include
+// when aggregating non-common label/annotation values across alerts.
+const maxAggregatedValues = 5
+
+// bareVariableRegex matches bare $variable references including dotted paths like $service.name.
+var bareVariableRegex = regexp.MustCompile(`\$(\w+(?:\.\w+)*)`)
+
+// bareVariableRegexFirstSeg matches only the base $variable name, stopping before any dotted path.
+// e.g. "$labels.severity" matches "$labels", "$name" matches "$name".
+var bareVariableRegexFirstSeg = regexp.MustCompile(`\$\w+`)
+
+// ExtractTemplatesFromAnnotations computes the common annotations across all alerts
+// and returns the values for the title_template and body_template annotation keys as title and body templates.
+func ExtractTemplatesFromAnnotations(alerts []*types.Alert) (titleTemplate, bodyTemplate string) {
+	if len(alerts) == 0 {
+		return "", ""
+	}
+
+	commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+	return commonAnnotations[ruletypes.AnnotationTitleTemplate], commonAnnotations[ruletypes.AnnotationBodyTemplate]
+}
+
+// WrapDollarVariables wraps bare $variable references in Go template syntax.
+// Example transformations:
+//   - "$name is $status" -> "{{ $name }} is {{ $status }}"
+//   - "$labels.severity" -> "{{ index .labels \"severity\" }}"
+//   - "$labels.http.status" -> "{{ index .labels \"http.status\" }}"
+//   - "$annotations.summary" -> "{{ index .annotations \"summary\" }}"
+//   - "$service.name" -> "{{ index . \"service.name\" }}"
+//   - "$name is {{ .Status }}" -> "{{ $name }} is {{ .Status }}"
+func WrapDollarVariables(src string) (string, error) {
+	if src == "" {
+		return src, nil
+	}
+
+	// Create a new parse.Tree directly
+	tree := parse.New("template")
+	tree.Mode = parse.SkipFuncCheck
+
+	// Parse the template
+	_, err := tree.Parse(src, "{{", "}}", make(map[string]*parse.Tree), nil)
+	if err != nil {
+		return "", err
+	}
+
+	// Walk the AST and transform TextNodes
+	walkAndWrapTextNodes(tree.Root)
+
+	// Return the reassembled template
+	return tree.Root.String(), nil
+}
+
+// walkAndWrapTextNodes recursively walks the parse tree trying to find a text node
+// once text node is found it wraps the bare $variable and changes it to index based
+// element access form datamap like .key or .key.subkey
+func walkAndWrapTextNodes(node parse.Node) {
+	if reflect.ValueOf(node).IsNil() {
+		return
+	}
+
+	switch n := node.(type) {
+	// `$name is {{.Status}}` is a list node with one text and one action node
+	case *parse.ListNode:
+		// Recurse into all child nodes
+		if n.Nodes != nil {
+			for _, child := range n.Nodes {
+				walkAndWrapTextNodes(child)
+			}
+		}
+
+	// `$name is ` is a text node with plain text in root
+	// we try to find the $name variable and wrap it with template block
+	// like `{{ .name }}`, for labels and annotations we use the index to access the value
+	// so `$labels.service` becomes `{{ index .labels "service" }}`
+	case *parse.TextNode:
+		// Transform $variable based on its pattern
+		n.Text = bareVariableRegex.ReplaceAllFunc(n.Text, func(match []byte) []byte {
+			// Extract variable name without the $
+			varName := string(match[1:])
+
+			// Check if variable contains dots
+			if strings.Contains(varName, ".") {
+				// Check for reserved prefixes: labels.* or annotations.*
+				if strings.HasPrefix(varName, "labels.") {
+					key := strings.TrimPrefix(varName, "labels.")
+					return []byte(fmt.Sprintf(`{{ index .labels "%s" }}`, key))
+				}
+				if strings.HasPrefix(varName, "annotations.") {
+					key := strings.TrimPrefix(varName, "annotations.")
+					return []byte(fmt.Sprintf(`{{ index .annotations "%s" }}`, key))
+				}
+				// Other dotted variables: index into root context
+				return []byte(fmt.Sprintf(`{{ index . "%s" }}`, varName))
+			}
+
+			// Simple variables: use dot notation to directly access the field
+			// without raising any error due to missing variables
+			return []byte(fmt.Sprintf("{{ .%s }}", varName))
+		})
+
+	// `{{if pipeline}} T1 {{else}} T0 {{end}}` is a if node with T1 part of List and T0 part of ElseList
+	case *parse.IfNode:
+		// Recurse into both branches
+		walkAndWrapTextNodes(n.List)
+		walkAndWrapTextNodes(n.ElseList)
+
+	// `{{range pipeline}} T1 {{else}} T0 {{end}}` is a range node with T1 part of List and T0 part of ElseList
+	case *parse.RangeNode:
+		// Recurse into both branches
+		walkAndWrapTextNodes(n.List)
+		walkAndWrapTextNodes(n.ElseList)
+
+		// All other node types (ActionNode, PipeNode, VariableNode, etc.) are already
+		// inside {{ }} action blocks and don't need transformation
+
+		// Support for `with` can be added later when we start supporting it in editor block
+	}
+}
+
+// ExtractUsedVariables returns the set of all $variable referenced in template
+// — text nodes, action blocks, branch conditions, and loop declarations — regardless of scope.
+// After finding all variables we find the ones which are not part of our alert data and handle them so
+// Go-text-template parser doesn't rejects undefined $variables
+func ExtractUsedVariables(src string) (map[string]bool, error) {
+	if src == "" {
+		return map[string]bool{}, nil
+	}
+
+	// Regex-scan raw template string to collect all $var base names.
+	// bareVariableRegexFirstSeg stops before dots, so "$labels.severity" yields "$labels".
+	used := make(map[string]bool)
+	for _, m := range bareVariableRegexFirstSeg.FindAll([]byte(src), -1) {
+		used[string(m[1:])] = true // strip leading "$"
+	}
+
+	// Build a preamble that pre-declares every found variable.
+	// This prevents "undefined variable" parse errors for $vars used in action
+	// blocks while still letting genuine syntax errors propagate.
+	var preamble strings.Builder
+	for name := range used {
+		fmt.Fprintf(&preamble, `{{$%s := ""}}`, name)
+	}
+
+	// Validate template syntax.
+	tree := parse.New("template")
+	tree.Mode = parse.SkipFuncCheck
+	if _, err := tree.Parse(preamble.String()+src, "{{", "}}", make(map[string]*parse.Tree), nil); err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInternal, "failed to extract used variables")
+	}
+
+	return used, nil
+}
+
+// aggregateKV aggregates key-value pairs (labels or annotations) from all alerts into a single template.KV
+// the result is used to populate the labels and annotations in the notification template data.
+// this is done to avoid blank values in the template when labels and annotations used are not common throughout the alerts
+func aggregateKV(alerts []*types.Alert, extractFn func(*types.Alert) model.LabelSet) template.KV {
+	// track unique values per key in order of first appearance
+	valuesPerKey := make(map[string][]string)
+	// track which values have been seen for deduplication
+	seenValues := make(map[string]map[string]bool)
+
+	for _, alert := range alerts {
+		kvPairs := extractFn(alert)
+		for k, v := range kvPairs {
+			key := string(k)
+			value := string(v)
+
+			if seenValues[key] == nil {
+				seenValues[key] = make(map[string]bool)
+			}
+
+			// only add if not already seen and under the limit of maxAggregatedValues
+			if !seenValues[key][value] && len(valuesPerKey[key]) < maxAggregatedValues {
+				seenValues[key][value] = true
+				valuesPerKey[key] = append(valuesPerKey[key], value)
+			}
+		}
+	}
+
+	// build the result by joining values
+	result := make(template.KV, len(valuesPerKey))
+	for key, values := range valuesPerKey {
+		result[key] = strings.Join(values, ", ")
+	}
+
+	return result
+}
+
+// extractCommonKV returns the intersection of key-value pairs across all alerts.
+// A key/value pair is included only if it appears identically on every alert.
+func extractCommonKV(alerts []*types.Alert, extractFn func(*types.Alert) model.LabelSet) template.KV {
+	if len(alerts) == 0 {
+		return template.KV{}
+	}
+
+	common := make(template.KV, len(extractFn(alerts[0])))
+	for k, v := range extractFn(alerts[0]) {
+		common[string(k)] = string(v)
+	}
+
+	for _, a := range alerts[1:] {
+		kv := extractFn(a)
+		for k := range common {
+			if string(kv[model.LabelName(k)]) != common[k] {
+				delete(common, k)
+			}
+		}
+		if len(common) == 0 {
+			break
+		}
+	}
+
+	return common
+}

pkg/alertmanager/alertmanagertemplate/utils_test.go

@@ -0,0 +1,348 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWrapBareVars(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    string
+		expectError bool
+	}{
+		{
+			name:     "mixed variables with actions",
+			input:    "$name is {{.Status}}",
+			expected: "{{ .name }} is {{.Status}}",
+		},
+		{
+			name: "nested variables in range",
+			input: `{{range .items}}
+				$title
+				{{end}}`,
+			expected: `{{range .items}}
+				{{ .title }}
+				{{end}}`,
+		},
+		{
+			name:     "nested variables in if else",
+			input:    "{{if .ok}}$a{{else}}$b{{end}}",
+			expected: "{{if .ok}}{{ .a }}{{else}}{{ .b }}{{end}}",
+		},
+		// Labels prefix: index into .labels map
+		{
+			name:     "labels variables prefix simple",
+			input:    "$labels.service",
+			expected: `{{ index .labels "service" }}`,
+		},
+		{
+			name:     "labels variables prefix nested with multiple dots",
+			input:    "$labels.http.status",
+			expected: `{{ index .labels "http.status" }}`,
+		},
+		{
+			name:     "multiple labels variables simple and nested",
+			input:    "$labels.service and $labels.instance.id",
+			expected: `{{ index .labels "service" }} and {{ index .labels "instance.id" }}`,
+		},
+		// Annotations prefix: index into .annotations map
+		{
+			name:     "annotations variables prefix simple",
+			input:    "$annotations.summary",
+			expected: `{{ index .annotations "summary" }}`,
+		},
+		{
+			name:     "annotations variables prefix nested with multiple dots",
+			input:    "$annotations.alert.url",
+			expected: `{{ index .annotations "alert.url" }}`,
+		},
+		// Other dotted paths: index into root context
+		{
+			name:     "other variables with multiple dots",
+			input:    "$service.name",
+			expected: `{{ index . "service.name" }}`,
+		},
+		{
+			name:     "other variables with multiple dots nested",
+			input:    "$http.status.code",
+			expected: `{{ index . "http.status.code" }}`,
+		},
+		// Hybrid: all types combined
+		{
+			name:     "hybrid - all variables types",
+			input:    "Alert: $alert_name Labels: $labels.severity Annotations: $annotations.desc Service: $service.name Count: $error_count",
+			expected: `Alert: {{ .alert_name }} Labels: {{ index .labels "severity" }} Annotations: {{ index .annotations "desc" }} Service: {{ index . "service.name" }} Count: {{ .error_count }}`,
+		},
+		{
+			name:     "already wrapped should not be changed",
+			input:    "{{$status := .status}}{{.name}} is {{$status | toUpper}}",
+			expected: "{{$status := .status}}{{.name}} is {{$status | toUpper}}",
+		},
+		{
+			name:     "no variables should not be changed",
+			input:    "Hello world",
+			expected: "Hello world",
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "deeply nested",
+			input:    "{{range .items}}{{if .ok}}$deep{{end}}{{end}}",
+			expected: "{{range .items}}{{if .ok}}{{ .deep }}{{end}}{{end}}",
+		},
+		{
+			name: "complex example",
+			input: `Hello $name, your score is $score.
+				{{if .isAdmin}}
+					Welcome back $name, you have {{.unreadCount}} messages.
+				{{end}}`,
+			expected: `Hello {{ .name }}, your score is {{ .score }}.
+				{{if .isAdmin}}
+					Welcome back {{ .name }}, you have {{.unreadCount}} messages.
+				{{end}}`,
+		},
+		{
+			name:     "with custom function",
+			input:    "$name triggered at {{urlescape .url}}",
+			expected: "{{ .name }} triggered at {{urlescape .url}}",
+		},
+		{
+			name:        "invalid template",
+			input:       "{{invalid",
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := WrapDollarVariables(tc.input)
+
+			if tc.expectError {
+				require.Error(t, err, "should error on invalid template syntax")
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestExtractUsedVariables(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    map[string]bool
+		expectError bool
+	}{
+		{
+			name:     "simple usage in text",
+			input:    "$name is $status",
+			expected: map[string]bool{"name": true, "status": true},
+		},
+		{
+			name:     "declared in action block",
+			input:    "{{ $name := .name }}",
+			expected: map[string]bool{"name": true},
+		},
+		{
+			name:     "range loop vars",
+			input:    "{{ range $i, $v := .items }}{{ end }}",
+			expected: map[string]bool{"i": true, "v": true},
+		},
+		{
+			name:     "mixed text and action",
+			input:    "$x and {{ $y }}",
+			expected: map[string]bool{"x": true, "y": true},
+		},
+		{
+			name:     "dotted path in text extracts base only",
+			input:    "$labels.severity",
+			expected: map[string]bool{"labels": true},
+		},
+		{
+			name:     "nested if else",
+			input:    "{{ if .ok }}{{ $a }}{{ else }}{{ $b }}{{ end }}",
+			expected: map[string]bool{"a": true, "b": true},
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: map[string]bool{},
+		},
+		{
+			name:     "no variables",
+			input:    "Hello world",
+			expected: map[string]bool{},
+		},
+		{
+			name:        "invalid template returns error",
+			input:       "{{invalid",
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := ExtractUsedVariables(tc.input)
+
+			if tc.expectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestAggregateKV(t *testing.T) {
+	extractLabels := func(a *types.Alert) model.LabelSet { return a.Labels }
+
+	testCases := []struct {
+		name      string
+		alerts    []*types.Alert
+		extractFn func(*types.Alert) model.LabelSet
+		expected  template.KV
+	}{
+		{
+			name:      "empty alerts slice",
+			alerts:    []*types.Alert{},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "single alert",
+			alerts: []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels: model.LabelSet{
+							"env":     "production",
+							"service": "backend",
+						},
+					},
+				},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"env":     "production",
+				"service": "backend",
+			},
+		},
+		{
+			name: "varying values with duplicates deduped",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "backend"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "frontend"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "api"}}},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"env":     "production",
+				"service": "backend, api, frontend",
+			},
+		},
+		{
+			name: "more than 5 unique values truncates to 5",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc1"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc2"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc3"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc4"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc5"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc6"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc7"}}},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"service": "svc1, svc2, svc3, svc4, svc5",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := aggregateKV(tc.alerts, tc.extractFn)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestExtractCommonKV(t *testing.T) {
+	extractLabels := func(a *types.Alert) model.LabelSet { return a.Labels }
+	extractAnnotations := func(a *types.Alert) model.LabelSet { return a.Annotations }
+
+	testCases := []struct {
+		name      string
+		alerts    []*types.Alert
+		extractFn func(*types.Alert) model.LabelSet
+		expected  template.KV
+	}{
+		{
+			name:      "empty alerts slice",
+			alerts:    []*types.Alert{},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "single alert returns all labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "api"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod", "service": "api"},
+		},
+		{
+			name: "multiple alerts with fully common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "region": "us-east"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "region": "us-east"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod", "region": "us-east"},
+		},
+		{
+			name: "multiple alerts with partially common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "worker"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod"},
+		},
+		{
+			name: "multiple alerts with no common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "worker"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "annotations extract common annotations",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Annotations: model.LabelSet{"summary": "high cpu", "runbook": "http://x"}}},
+				{Alert: model.Alert{Annotations: model.LabelSet{"summary": "high cpu", "runbook": "http://y"}}},
+			},
+			extractFn: extractAnnotations,
+			expected:  template.KV{"summary": "high cpu"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := extractCommonKV(tc.alerts, tc.extractFn)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}

pkg/alertmanager/alertnotificationprocessor/alertnotificationprocessor.go

@@ -0,0 +1,106 @@
+package alertnotificationprocessor
+
+import (
+	"bytes"
+	"context"
+	htmltemplate "html/template"
+	"log/slog"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
+	"github.com/prometheus/alertmanager/types"
+)
+
+type alertNotificationProcessor struct {
+	templater     alertmanagertemplate.AlertManagerTemplater
+	renderer      markdownrenderer.MarkdownRenderer
+	logger        *slog.Logger
+	templateStore emailtypes.TemplateStore
+}
+
+func New(templater alertmanagertemplate.AlertManagerTemplater, renderer markdownrenderer.MarkdownRenderer, templateStore emailtypes.TemplateStore, logger *slog.Logger) alertmanagertypes.NotificationProcessor {
+	return &alertNotificationProcessor{
+		templater:     templater,
+		renderer:      renderer,
+		logger:        logger,
+		templateStore: templateStore,
+	}
+}
+
+// emailNotificationTemplateData is the data passed to the email HTML layout template.
+// It embeds NotificationTemplateData so all its fields are directly accessible in the template.
+type emailNotificationTemplateData struct {
+	alertmanagertemplate.NotificationTemplateData
+	Title  string
+	Bodies []htmltemplate.HTML
+}
+
+func (p *alertNotificationProcessor) ProcessAlertNotification(ctx context.Context, input alertmanagertypes.NotificationProcessorInput, alerts []*types.Alert, rendererFormat markdownrenderer.OutputFormat) (*alertmanagertypes.NotificationProcessorResult, error) {
+	// delegate to templater
+	expanded, err := p.templater.ProcessTemplates(ctx, alertmanagertemplate.TemplateInput{
+		TitleTemplate:        input.TitleTemplate,
+		BodyTemplate:         input.BodyTemplate,
+		DefaultTitleTemplate: input.DefaultTitleTemplate,
+		DefaultBodyTemplate:  input.DefaultBodyTemplate,
+	}, alerts)
+	if err != nil {
+		return nil, err
+	}
+
+	// apply rendering to body based on the format
+	var renderedBodies []string
+	if expanded.IsDefaultTemplatedBody {
+		// default templates already produce format-appropriate output
+		renderedBodies = expanded.Body
+	} else {
+		// render each body string using the renderer
+		for _, body := range expanded.Body {
+			rendered, err := p.renderer.Render(ctx, body, rendererFormat)
+			if err != nil {
+				return nil, err
+			}
+			renderedBodies = append(renderedBodies, rendered)
+		}
+	}
+
+	return &alertmanagertypes.NotificationProcessorResult{
+		Title:                  expanded.Title,
+		Body:                   renderedBodies,
+		IsDefaultTemplatedBody: expanded.IsDefaultTemplatedBody,
+		MissingVars:            expanded.MissingVars,
+	}, nil
+}
+
+func (p *alertNotificationProcessor) RenderEmailNotification(
+	ctx context.Context,
+	templateName emailtypes.TemplateName,
+	result *alertmanagertypes.NotificationProcessorResult,
+	alerts []*types.Alert,
+) (string, error) {
+	layoutTmpl, err := p.templateStore.Get(ctx, templateName)
+	if err != nil {
+		return "", err
+	}
+
+	ntd := p.templater.BuildNotificationTemplateData(ctx, alerts)
+
+	bodies := make([]htmltemplate.HTML, 0, len(result.Body))
+	for _, b := range result.Body {
+		bodies = append(bodies, htmltemplate.HTML(b))
+	}
+
+	data := emailNotificationTemplateData{
+		NotificationTemplateData: *ntd,
+		Title:                    result.Title,
+		Bodies:                   bodies,
+	}
+
+	var buf bytes.Buffer
+	if err := layoutTmpl.Execute(&buf, data); err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
+}

pkg/alertmanager/alertnotificationprocessor/alertnotificationprocessor_test.go

@@ -0,0 +1,343 @@
+package alertnotificationprocessor
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+	"time"
+
+	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/emailing/templatestore/filetemplatestore"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+)
+
+func testSetup(t *testing.T) (alertmanagertypes.NotificationProcessor, context.Context) {
+	t.Helper()
+	tmpl := test.CreateTmpl(t)
+	logger := slog.New(slog.DiscardHandler)
+	templater := alertmanagertemplate.New(tmpl, logger)
+	renderer := markdownrenderer.NewMarkdownRenderer(logger)
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "slack")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{
+		"alertname": "TestAlert",
+		"severity":  "critical",
+	})
+	return New(templater, renderer, filetemplatestore.NewEmptyStore(), logger), ctx
+}
+
+func createAlert(labels, annotations map[string]string, isFiring bool) *types.Alert {
+	ls := model.LabelSet{}
+	for k, v := range labels {
+		ls[model.LabelName(k)] = model.LabelValue(v)
+	}
+	ann := model.LabelSet{}
+	for k, v := range annotations {
+		ann[model.LabelName(k)] = model.LabelValue(v)
+	}
+	startsAt := time.Now()
+	var endsAt time.Time
+	if isFiring {
+		endsAt = startsAt.Add(time.Hour)
+	} else {
+		startsAt = startsAt.Add(-2 * time.Hour)
+		endsAt = startsAt.Add(-time.Hour)
+	}
+	return &types.Alert{Alert: model.Alert{Labels: ls, Annotations: ann, StartsAt: startsAt, EndsAt: endsAt}}
+}
+
+func TestProcessAlertNotification(t *testing.T) {
+	processor, ctx := testSetup(t)
+
+	tests := []struct {
+		name              string
+		alerts            []*types.Alert
+		input             alertmanagertypes.NotificationProcessorInput
+		wantTitle         string
+		wantBody          []string
+		wantIsDefaultBody bool
+		wantMissingVars   []string
+		RendererFormat    markdownrenderer.OutputFormat
+	}{
+		{
+			name: "custom title and body rendered as HTML",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "HighCPU",
+						ruletypes.LabelSeverityName: "critical",
+						"service":                   "api-server",
+					},
+					map[string]string{"description": "CPU usage exceeded 95%"},
+					true,
+				),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				TitleTemplate: "Alert: $rule_name on $service",
+				BodyTemplate:  "**Service:** $service\n\n**Description:** $description",
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatHTML,
+			wantTitle:         "Alert: HighCPU on api-server",
+			wantBody:          []string{"<p><strong>Service:</strong> api-server</p><p></p><p><strong>Description:</strong> CPU usage exceeded 95%</p><p></p>"},
+			wantIsDefaultBody: false,
+		},
+		{
+			name: "custom title and body rendered as SlackBlockKit",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "HighMemory",
+						ruletypes.LabelSeverityName: "warning",
+					},
+					map[string]string{"description": "Memory usage high"},
+					true,
+				),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				TitleTemplate: "$rule_name - $severity",
+				BodyTemplate:  "Memory alert: $description",
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatSlackBlockKit,
+			wantTitle:         "HighMemory - warning",
+			wantBody:          []string{`[{"type":"section","text":{"type":"mrkdwn","text":"Memory alert: Memory usage high"}}]`},
+			wantIsDefaultBody: false,
+		},
+		{
+			name: "custom title and body with Noop format passes through as-is",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "DiskFull",
+						ruletypes.LabelSeverityName: "critical",
+						"host":                      "db-01",
+					},
+					nil,
+					true,
+				),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				TitleTemplate: "$rule_name on $host",
+				BodyTemplate:  "**Host:** $labels.host is full",
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatNoop,
+			wantTitle:         "DiskFull on db-01",
+			wantBody:          []string{"**Host:** db-01 is full"},
+			wantIsDefaultBody: false,
+		},
+		{
+			name: "default fallback when custom templates are empty",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "TestAlert",
+						ruletypes.LabelSeverityName: "critical",
+					},
+					map[string]string{"description": "Something broke"},
+					true,
+				),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				DefaultTitleTemplate: `{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})`,
+				DefaultBodyTemplate:  `{{ range .Alerts }}{{ .Annotations.description }}{{ end }}`,
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatHTML,
+			wantTitle:         "TestAlert (FIRING)",
+			wantBody:          []string{"Something broke"},
+			wantIsDefaultBody: true,
+		},
+		{
+			name: "missing vars pass through to result",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "TestAlert"},
+					nil,
+					true,
+				),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				TitleTemplate: "[$environment] $rule_name",
+				BodyTemplate:  "See runbook: $runbook_url",
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatNoop,
+			wantTitle:         "[<no value>] TestAlert",
+			wantBody:          []string{"See runbook: <no value>"},
+			wantIsDefaultBody: false,
+			wantMissingVars:   []string{"environment", "runbook_url"},
+		},
+		{
+			name: "slack mrkdwn renders bold and italic correctly along with missing variables",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "HighCPU",
+						ruletypes.LabelSeverityName: "critical",
+						"service":                   "api-server",
+					},
+					map[string]string{"description": "CPU usage exceeded 95%"},
+					true,
+				),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				TitleTemplate: "Alert: $rule_name",
+				BodyTemplate:  "**Service:** $service\n\n*Description:* $description $http_request_method",
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatSlackMrkdwn,
+			wantTitle:         "Alert: HighCPU",
+			wantBody:          []string{"*Service:* api-server\n\n_Description:_ CPU usage exceeded 95% <no value>\n\n"},
+			wantMissingVars:   []string{"http_request_method"},
+			wantIsDefaultBody: false,
+		},
+		{
+			name: "slack mrkdwn with multiple alerts produces per-alert bodies",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "SvcDown", "service": "auth"},
+					map[string]string{"description": "Auth service **down**"},
+					true,
+				),
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "SvcDown", "service": "payments"},
+					map[string]string{"description": "Payments service **degraded**"},
+					false,
+				),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				TitleTemplate: "$rule_name: $total_firing firing, $total_resolved resolved",
+				BodyTemplate:  "**$service** ($status): $description",
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatSlackMrkdwn,
+			wantTitle:         "SvcDown: 1 firing, 1 resolved",
+			wantBody:          []string{"*auth* (firing): Auth service *down*\n\n", "*payments* (resolved): Payments service *degraded*\n\n"},
+			wantIsDefaultBody: false,
+		},
+		{
+			name: "slack mrkdwn skips rendering for default templates",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "TestAlert",
+						ruletypes.LabelSeverityName: "critical",
+					},
+					map[string]string{"description": "Something broke"},
+					true,
+				),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				DefaultTitleTemplate: `{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})`,
+				DefaultBodyTemplate:  `{{ range .Alerts }}**Bold** *italic* ~~strike~~ {{ .Annotations.description }}{{ end }}`,
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatSlackMrkdwn,
+			wantTitle:         "TestAlert (FIRING)",
+			wantBody:          []string{"**Bold** *italic* ~~strike~~ Something broke"},
+			wantIsDefaultBody: true,
+		},
+		{
+			name: "multiple alerts produce one body entry per alert",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrash", "pod": "worker-1"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrash", "pod": "worker-2"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrash", "pod": "worker-3"}, nil, false),
+			},
+			input: alertmanagertypes.NotificationProcessorInput{
+				TitleTemplate: "$rule_name: $total_firing firing",
+				BodyTemplate:  "$labels.pod ($status)",
+			},
+			RendererFormat:    markdownrenderer.MarkdownFormatNoop,
+			wantTitle:         "PodCrash: 2 firing",
+			wantBody:          []string{"worker-1 (firing)", "worker-2 (firing)", "worker-3 (resolved)"},
+			wantIsDefaultBody: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := processor.ProcessAlertNotification(ctx, tc.input, tc.alerts, tc.RendererFormat)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.wantTitle, result.Title)
+			require.Equal(t, tc.wantBody, result.Body)
+			require.Equal(t, tc.wantIsDefaultBody, result.IsDefaultTemplatedBody)
+
+			if len(tc.wantMissingVars) == 0 {
+				require.Empty(t, result.MissingVars)
+			} else {
+				sort.Strings(tc.wantMissingVars)
+				require.Equal(t, tc.wantMissingVars, result.MissingVars)
+			}
+		})
+	}
+}
+
+func TestRenderEmailNotification_TemplateNotFound(t *testing.T) {
+	processor, ctx := testSetup(t)
+
+	result := &alertmanagertypes.NotificationProcessorResult{
+		Title: "Test Alert",
+		Body:  []string{"alert body"},
+	}
+	alerts := []*types.Alert{
+		createAlert(map[string]string{ruletypes.LabelAlertName: "TestAlert"}, nil, true),
+	}
+
+	_, err := processor.RenderEmailNotification(ctx, emailtypes.TemplateNameAlertEmailNotification, result, alerts)
+	require.Error(t, err)
+	require.True(t, errors.Ast(err, errors.TypeNotFound))
+}
+
+func TestRenderEmailNotification_RendersTemplate(t *testing.T) {
+	// Create a temp dir with a test template
+	tmpDir := t.TempDir()
+	tmplContent := `<!DOCTYPE html><html><body><h1>{{.Title}}</h1><p>Status: {{.Status}}</p><p>Firing: {{.TotalFiring}}</p>{{range .Bodies}}<div>{{.}}</div>{{end}}{{range .Alerts}}<p>{{.AlertName}}</p>{{end}}</body></html>`
+	err := os.WriteFile(filepath.Join(tmpDir, "alert_email_notification.gotmpl"), []byte(tmplContent), 0644)
+	require.NoError(t, err)
+
+	tmpl := test.CreateTmpl(t)
+	logger := slog.New(slog.DiscardHandler)
+	templater := alertmanagertemplate.New(tmpl, logger)
+	renderer := markdownrenderer.NewMarkdownRenderer(logger)
+	store, err := filetemplatestore.NewStore(context.Background(), tmpDir, emailtypes.Templates, logger)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "email")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{
+		"alertname": "HighCPU",
+		"severity":  "critical",
+	})
+
+	processor := New(templater, renderer, store, logger)
+
+	result := &alertmanagertypes.NotificationProcessorResult{
+		Title:                  "HighCPU Alert",
+		Body:                   []string{"<strong>CPU is high</strong>", "<strong>CPU is low</strong>"},
+		IsDefaultTemplatedBody: false,
+	}
+	alerts := []*types.Alert{
+		createAlert(
+			map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"},
+			nil,
+			true,
+		),
+	}
+
+	html, err := processor.RenderEmailNotification(ctx, emailtypes.TemplateNameAlertEmailNotification, result, alerts)
+	require.NoError(t, err)
+	require.NotEmpty(t, html)
+	// the html template should be filled with go text templating
+	require.Equal(t, "<!DOCTYPE html><html><body><h1>HighCPU Alert</h1><p>Status: firing</p><p>Firing: 1</p><div><strong>CPU is high</strong></div><div><strong>CPU is low</strong></div><p>HighCPU</p></body></html>", html)
+}

pkg/errors/errors.go

@@ -172,7 +172,7 @@ func Ast(cause error, typ typ) bool {
 	return t == typ
 }
 
-// Ast checks if the provided error matches the specified custom error code.
+// Asc checks if the provided error matches the specified custom error code.
 func Asc(cause error, code Code) bool {
 	_, c, _, _, _, _ := Unwrapb(cause)
 

pkg/query-service/rules/prom_rule.go

@@ -252,8 +252,23 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 
 		annotations := make(qslabels.Labels, 0, len(r.annotations.Map()))
 		for name, value := range r.annotations.Map() {
+			// no need to expand custom templating annotations — they get expanded in the notifier layer
+			if ruletypes.IsCustomTemplatingAnnotation(name) {
+				annotations = append(annotations, qslabels.Label{Name: name, Value: value})
+				continue
+			}
 			annotations = append(annotations, qslabels.Label{Name: name, Value: expand(value)})
 		}
+		// Add values to be used in notifier layer for notification templates
+		annotations = append(annotations, qslabels.Label{Name: ruletypes.AnnotationValue, Value: valueFormatter.Format(result.V, r.Unit())})
+		annotations = append(annotations, qslabels.Label{Name: ruletypes.AnnotationThresholdValue, Value: threshold})
+		annotations = append(annotations, qslabels.Label{Name: ruletypes.AnnotationCompareOp, Value: result.CompareOp.String()})
+		annotations = append(annotations, qslabels.Label{Name: ruletypes.AnnotationMatchType, Value: result.MatchType.String()})
+
+		if result.IsRecovering {
+			lb.Set(ruletypes.LabelIsRecovering, "true")
+		}
+
 		if result.IsMissing {
 			lb.Set(qslabels.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(qslabels.NoDataLabel, "true")

pkg/query-service/rules/threshold_rule.go

@@ -658,8 +658,23 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 
 		annotations := make(labels.Labels, 0, len(r.annotations.Map()))
 		for name, value := range r.annotations.Map() {
+			// no need to expand custom templating annotations — they get expanded in the notifier layer
+			if ruletypes.IsCustomTemplatingAnnotation(name) {
+				annotations = append(annotations, labels.Label{Name: name, Value: value})
+				continue
+			}
 			annotations = append(annotations, labels.Label{Name: name, Value: expand(value)})
 		}
+		// Add values to be used in notifier layer for notification templates
+		annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationValue, Value: value})
+		annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationThresholdValue, Value: threshold})
+		annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationCompareOp, Value: smpl.CompareOp.String()})
+		annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationMatchType, Value: smpl.MatchType.String()})
+
+		if smpl.IsRecovering {
+			lb.Set(ruletypes.LabelIsRecovering, "true")
+		}
+
 		if smpl.IsMissing {
 			lb.Set(labels.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(labels.NoDataLabel, "true")
@@ -673,13 +688,13 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 			link := r.prepareLinksToTraces(ctx, ts, smpl.Metric)
 			if link != "" && r.hostFromSource() != "" {
 				r.logger.InfoContext(ctx, "adding traces link to annotations", "link", fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link))
-				annotations = append(annotations, labels.Label{Name: "related_traces", Value: fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)})
+				annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationRelatedTraces, Value: fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)})
 			}
 		case ruletypes.AlertTypeLogs:
 			link := r.prepareLinksToLogs(ctx, ts, smpl.Metric)
 			if link != "" && r.hostFromSource() != "" {
 				r.logger.InfoContext(ctx, "adding logs link to annotations", "link", fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link))
-				annotations = append(annotations, labels.Label{Name: "related_logs", Value: fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)})
+				annotations = append(annotations, labels.Label{Name: ruletypes.AnnotationRelatedLogs, Value: fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)})
 			}
 		}
 

pkg/templating/markdownrenderer/html.go

@@ -0,0 +1,28 @@
+package markdownrenderer
+
+import (
+	"bytes"
+	"context"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+)
+
+// SoftLineBreakHTML is a HTML tag that is used to represent a soft line break.
+const SoftLineBreakHTML = `<p></p>`
+
+func (r *markdownRenderer) renderHTML(_ context.Context, markdown string) (string, error) {
+	var buf bytes.Buffer
+	if err := r.htmlRenderer.Convert([]byte(markdown), &buf); err != nil {
+		return "", errors.WrapInternalf(err, errors.CodeInternal, "failed to convert markdown to HTML")
+	}
+
+	// return buf.String(), nil
+
+	// TODO: check if there is another way to handle soft line breaks in HTML
+	// the idea with paragraph tags is that it will start the content in new
+	// line without using a line break tag, this works well in variety of cases
+	// but not all, for example, in case of code block, the paragraph tags will be added
+	// to the code block where newline is present.
+	return strings.ReplaceAll(buf.String(), "\n", SoftLineBreakHTML), nil
+}

pkg/templating/markdownrenderer/html_test.go

@@ -0,0 +1,182 @@
+package markdownrenderer
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testMarkdown = `# 🔥 FIRING: High CPU Usage on api-gateway
+
+https://signoz.example.com/alerts/123
+https://runbooks.example.com/cpu-high
+
+## Alert Details
+
+**Status:** **FIRING** | *api-gateway* service is experiencing high CPU usage. ~~resolved~~ previously.
+
+Alert triggered because ` + "`cpu_usage_percent`" + ` exceeded threshold ` + "`90`" + `.
+
+[View Alert in SigNoz](https://signoz.example.com/alerts/123) | [View Logs](https://signoz.example.com/logs?service=api-gateway) | [View Traces](https://signoz.example.com/traces?service=api-gateway)
+
+![critical](https://signoz.example.com/badges/critical.svg "Critical Alert")
+
+## Alert Labels
+
+| Label    | Value       |
+| -------- | ----------- |
+| service  | api-gateway |
+| instance | pod-5a8b3c  |
+| severity | critical    |
+| region   | us-east-1   |
+
+## Remediation Steps
+
+1. Check current CPU usage on the pod
+2. Review recent deployments for regressions
+3. Scale horizontally if load-related
+    1. Increase replica count
+    2. Verify HPA configuration
+
+## Affected Services
+
+* api-gateway
+* auth-service
+* payment-service
+    * payment-processor
+    * payment-validator
+
+## Incident Checklist
+
+- [x] Alert acknowledged
+- [x] On-call notified
+- [ ] Root cause identified
+- [ ] Fix deployed
+
+## Alert Rule Description
+
+> This alert fires when CPU usage exceeds 90% for more than 5 minutes on any pod in the api-gateway service.
+>
+>> For capacity planning guidelines, see the infrastructure runbook section on horizontal pod autoscaling.
+
+## Triggered Query
+` + "```promql\navg(rate(container_cpu_usage_seconds_total{service=\"api-gateway\"}[5m])) by (pod) > 0.9\n```" + `
+
+## Inline Details
+
+This alert was generated by SigNoz using ` + "`alertmanager`" + ` rules engine.
+`
+)
+
+func newTestRenderer() MarkdownRenderer {
+	return NewMarkdownRenderer(slog.New(slog.NewTextHandler(os.Stdout, nil)))
+}
+
+func TestRenderHTML_Composite(t *testing.T) {
+	renderer := newTestRenderer()
+
+	html, err := renderer.Render(context.Background(), testMarkdown, MarkdownFormatHTML)
+	require.NoError(t, err)
+
+	// Full expected output for exact match
+	expected := `<h1>🔥 FIRING: High CPU Usage on api-gateway</h1><p></p>` +
+		`<p><a href="https://signoz.example.com/alerts/123">https://signoz.example.com/alerts/123</a><p></p><a href="https://runbooks.example.com/cpu-high">https://runbooks.example.com/cpu-high</a></p><p></p>` +
+		`<h2>Alert Details</h2><p></p>` +
+		`<p><strong>Status:</strong> <strong>FIRING</strong> | <em>api-gateway</em> service is experiencing high CPU usage. <del>resolved</del> previously.</p><p></p>` +
+		`<p>Alert triggered because <code>cpu_usage_percent</code> exceeded threshold <code>90</code>.</p><p></p>` +
+		`<p><a href="https://signoz.example.com/alerts/123">View Alert in SigNoz</a> | <a href="https://signoz.example.com/logs?service=api-gateway">View Logs</a> | <a href="https://signoz.example.com/traces?service=api-gateway">View Traces</a></p><p></p>` +
+		`<p><img src="https://signoz.example.com/badges/critical.svg" alt="critical" title="Critical Alert"></p><p></p>` +
+		`<h2>Alert Labels</h2><p></p>` +
+		`<table><p></p><thead><p></p><tr><p></p><th>Label</th><p></p><th>Value</th><p></p></tr><p></p></thead><p></p>` +
+		`<tbody><p></p><tr><p></p><td>service</td><p></p><td>api-gateway</td><p></p></tr><p></p>` +
+		`<tr><p></p><td>instance</td><p></p><td>pod-5a8b3c</td><p></p></tr><p></p>` +
+		`<tr><p></p><td>severity</td><p></p><td>critical</td><p></p></tr><p></p>` +
+		`<tr><p></p><td>region</td><p></p><td>us-east-1</td><p></p></tr><p></p></tbody><p></p></table><p></p>` +
+		`<h2>Remediation Steps</h2><p></p>` +
+		`<ol><p></p><li>Check current CPU usage on the pod</li><p></p><li>Review recent deployments for regressions</li><p></p><li>Scale horizontally if load-related<p></p>` +
+		`<ol><p></p><li>Increase replica count</li><p></p><li>Verify HPA configuration</li><p></p></ol><p></p></li><p></p></ol><p></p>` +
+		`<h2>Affected Services</h2><p></p>` +
+		`<ul><p></p><li>api-gateway</li><p></p><li>auth-service</li><p></p><li>payment-service<p></p>` +
+		`<ul><p></p><li>payment-processor</li><p></p><li>payment-validator</li><p></p></ul><p></p></li><p></p></ul><p></p>` +
+		`<h2>Incident Checklist</h2><p></p>` +
+		`<ul><p></p><li><input checked="" disabled="" type="checkbox"> Alert acknowledged</li><p></p>` +
+		`<li><input checked="" disabled="" type="checkbox"> On-call notified</li><p></p>` +
+		`<li><input disabled="" type="checkbox"> Root cause identified</li><p></p>` +
+		`<li><input disabled="" type="checkbox"> Fix deployed</li><p></p></ul><p></p>` +
+		`<h2>Alert Rule Description</h2><p></p>` +
+		`<blockquote><p></p><p>This alert fires when CPU usage exceeds 90% for more than 5 minutes on any pod in the api-gateway service.</p><p></p>` +
+		`<blockquote><p></p><p>For capacity planning guidelines, see the infrastructure runbook section on horizontal pod autoscaling.</p><p></p></blockquote><p></p></blockquote><p></p>` +
+		`<h2>Triggered Query</h2><p></p>` +
+		`<pre><code class="language-promql">avg(rate(container_cpu_usage_seconds_total{service=&quot;api-gateway&quot;}[5m])) by (pod) &gt; 0.9<p></p></code></pre><p></p>` +
+		`<h2>Inline Details</h2><p></p>` +
+		`<p>This alert was generated by SigNoz using <code>alertmanager</code> rules engine.</p><p></p>`
+
+	assert.Equal(t, expected, html)
+}
+
+func TestRenderHTML_InlineFormatting(t *testing.T) {
+	renderer := newTestRenderer()
+
+	input := `# 🔥 FIRING: High CPU on api-gateway
+## Alert Status
+
+**FIRING** alert for *api-gateway* service — ~~resolved~~ previously.
+
+Metric ` + "`cpu_usage_percent`" + ` exceeded threshold. [View in SigNoz](https://signoz.example.com/alerts/123)
+
+![critical](https://signoz.example.com/badges/critical.svg "Critical Alert")`
+
+	html, err := renderer.Render(context.Background(), input, MarkdownFormatHTML)
+	require.NoError(t, err)
+
+	expected := `<h1>🔥 FIRING: High CPU on api-gateway</h1><p></p><h2>Alert Status</h2><p></p>` +
+		`<p><strong>FIRING</strong> alert for <em>api-gateway</em> service — <del>resolved</del> previously.</p><p></p>` +
+		`<p>Metric <code>cpu_usage_percent</code> exceeded threshold. <a href="https://signoz.example.com/alerts/123">View in SigNoz</a></p><p></p>` +
+		`<p><img src="https://signoz.example.com/badges/critical.svg" alt="critical" title="Critical Alert"></p><p></p>`
+
+	assert.Equal(t, expected, html)
+}
+
+func TestRenderHTML_BlockElements(t *testing.T) {
+	renderer := newTestRenderer()
+
+	input := `1. Check CPU usage on the pod
+2. Review recent deployments
+3. Scale horizontally if needed
+
+* api-gateway
+* auth-service
+* payment-service
+
+- [x] Alert acknowledged
+- [ ] Root cause identified
+
+> This alert fires when CPU usage exceeds 90% for more than 5 minutes.
+
+| Label    | Value       |
+| -------- | ----------- |
+| service  | api-gateway |
+| severity | <no value>    |
+
+` + "```promql\navg(rate(container_cpu_usage_seconds_total{service=\"api-gateway\"}[5m])) by (pod) > 0.9\n```"
+
+	html, err := renderer.Render(context.Background(), input, MarkdownFormatHTML)
+	require.NoError(t, err)
+
+	expected := `<ol><p></p><li>Check CPU usage on the pod</li><p></p><li>Review recent deployments</li><p></p><li>Scale horizontally if needed</li><p></p></ol><p></p>` +
+		`<ul><p></p><li>api-gateway</li><p></p><li>auth-service</li><p></p><li>payment-service</li><p></p></ul><p></p>` +
+		`<ul><p></p><li><input checked="" disabled="" type="checkbox"> Alert acknowledged</li><p></p>` +
+		`<li><input disabled="" type="checkbox"> Root cause identified</li><p></p></ul><p></p>` +
+		`<blockquote><p></p><p>This alert fires when CPU usage exceeds 90% for more than 5 minutes.</p><p></p></blockquote><p></p>` +
+		`<table><p></p><thead><p></p><tr><p></p><th>Label</th><p></p><th>Value</th><p></p></tr><p></p></thead><p></p>` +
+		`<tbody><p></p><tr><p></p><td>service</td><p></p><td>api-gateway</td><p></p></tr><p></p>` +
+		`<tr><p></p><td>severity</td><p></p><td>&lt;no value&gt;</td><p></p></tr><p></p></tbody><p></p></table><p></p>` +
+		`<pre><code class="language-promql">avg(rate(container_cpu_usage_seconds_total{service=&quot;api-gateway&quot;}[5m])) by (pod) &gt; 0.9<p></p></code></pre><p></p>`
+
+	assert.Equal(t, expected, html)
+}

pkg/templating/markdownrenderer/markdownrenderer.go

@@ -0,0 +1,70 @@
+package markdownrenderer
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/slackblockkitrenderer"
+	"github.com/SigNoz/signoz/pkg/templating/slackmrkdwnrenderer"
+	"github.com/SigNoz/signoz/pkg/templating/templatingextensions"
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+)
+
+type OutputFormat int
+
+const (
+	MarkdownFormatHTML OutputFormat = iota
+	MarkdownFormatSlackBlockKit
+	MarkdownFormatSlackMrkdwn
+	MarkdownFormatNoop
+)
+
+// MarkdownRenderer is the interface for rendering markdown to different formats.
+type MarkdownRenderer interface {
+	// Render renders the markdown to the given output format.
+	Render(ctx context.Context, markdown string, outputFormat OutputFormat) (string, error)
+}
+
+type markdownRenderer struct {
+	logger                *slog.Logger
+	htmlRenderer          goldmark.Markdown
+	slackBlockKitRenderer goldmark.Markdown
+	slackMrkdwnRenderer   goldmark.Markdown
+}
+
+func NewMarkdownRenderer(logger *slog.Logger) MarkdownRenderer {
+	htmlRenderer := goldmark.New(
+		// basic GitHub Flavored Markdown extensions
+		goldmark.WithExtensions(extension.GFM),
+		goldmark.WithExtensions(templatingextensions.EscapeNoValue),
+	)
+	slackBlockKitRenderer := goldmark.New(
+		goldmark.WithExtensions(slackblockkitrenderer.BlockKitV2),
+	)
+	slackMrkdwnRenderer := goldmark.New(
+		goldmark.WithExtensions(slackmrkdwnrenderer.SlackMrkdwn),
+	)
+	return &markdownRenderer{
+		logger:                logger,
+		htmlRenderer:          htmlRenderer,
+		slackBlockKitRenderer: slackBlockKitRenderer,
+		slackMrkdwnRenderer:   slackMrkdwnRenderer,
+	}
+}
+
+func (r *markdownRenderer) Render(ctx context.Context, markdown string, outputFormat OutputFormat) (string, error) {
+	switch outputFormat {
+	case MarkdownFormatHTML:
+		return r.renderHTML(ctx, markdown)
+	case MarkdownFormatSlackBlockKit:
+		return r.renderSlackBlockKit(ctx, markdown)
+	case MarkdownFormatSlackMrkdwn:
+		return r.renderSlackMrkdwn(ctx, markdown)
+	case MarkdownFormatNoop:
+		return markdown, nil
+	default:
+		return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unknown output format: %v", outputFormat)
+	}
+}

pkg/templating/markdownrenderer/noop_test.go

@@ -0,0 +1,17 @@
+package markdownrenderer
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRenderNoop(t *testing.T) {
+	renderer := newTestRenderer()
+
+	output, err := renderer.Render(context.Background(), testMarkdown, MarkdownFormatNoop)
+	require.NoError(t, err)
+	assert.Equal(t, testMarkdown, output)
+}

pkg/templating/markdownrenderer/slack.go

@@ -0,0 +1,24 @@
+package markdownrenderer
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+)
+
+func (r *markdownRenderer) renderSlackBlockKit(_ context.Context, markdown string) (string, error) {
+	var buf bytes.Buffer
+	if err := r.slackBlockKitRenderer.Convert([]byte(markdown), &buf); err != nil {
+		return "", errors.WrapInternalf(err, errors.CodeInternal, "failed to convert markdown to Slack Block Kit")
+	}
+	return buf.String(), nil
+}
+
+func (r *markdownRenderer) renderSlackMrkdwn(_ context.Context, markdown string) (string, error) {
+	var buf bytes.Buffer
+	if err := r.slackMrkdwnRenderer.Convert([]byte(markdown), &buf); err != nil {
+		return "", errors.WrapInternalf(err, errors.CodeInternal, "failed to convert markdown to Slack Mrkdwn")
+	}
+	return buf.String(), nil
+}

pkg/templating/markdownrenderer/slack_test.go

@@ -0,0 +1,152 @@
+package markdownrenderer
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"testing"
+)
+
+func jsonEqual(a, b string) bool {
+	var va, vb any
+	if err := json.Unmarshal([]byte(a), &va); err != nil {
+		return false
+	}
+	if err := json.Unmarshal([]byte(b), &vb); err != nil {
+		return false
+	}
+	ja, _ := json.Marshal(va)
+	jb, _ := json.Marshal(vb)
+	return string(ja) == string(jb)
+}
+
+func prettyJSON(s string) string {
+	var v any
+	if err := json.Unmarshal([]byte(s), &v); err != nil {
+		return s
+	}
+	b, _ := json.MarshalIndent(v, "", "  ")
+	return string(b)
+}
+
+func TestRenderSlackBlockKit(t *testing.T) {
+	renderer := NewMarkdownRenderer(slog.Default())
+
+	tests := []struct {
+		name     string
+		markdown string
+		expected string
+	}{
+		{
+			name:     "simple paragraph",
+			markdown: "Hello world",
+			expected: `[
+				{
+					"type": "section",
+					"text": { "type": "mrkdwn", "text": "Hello world" }
+				}
+			]`,
+		},
+		{
+			name: "alert-themed with heading, list, and code block",
+			markdown: `# Alert Triggered
+
+- Service: **checkout-api**
+- Status: _critical_
+
+` + "```" + `
+error: connection timeout after 30s
+` + "```",
+			expected: `[
+				{
+					"type": "section",
+					"text": { "type": "mrkdwn", "text": "*Alert Triggered*" }
+				},
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [
+									{ "type": "text", "text": "Service: " },
+									{ "type": "text", "text": "checkout-api", "style": { "bold": true } }
+								]},
+								{ "type": "rich_text_section", "elements": [
+									{ "type": "text", "text": "Status: " },
+									{ "type": "text", "text": "critical", "style": { "italic": true } }
+								]}
+							]
+						}
+					]
+				},
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_preformatted",
+							"border": 0,
+							"elements": [
+								{ "type": "text", "text": "error: connection timeout after 30s" }
+							]
+						}
+					]
+				}
+			]`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := renderer.Render(context.Background(), tt.markdown, MarkdownFormatSlackBlockKit)
+			if err != nil {
+				t.Fatalf("Render error: %v", err)
+			}
+
+			// Verify output is valid JSON
+			if !json.Valid([]byte(got)) {
+				t.Fatalf("output is not valid JSON:\n%s", got)
+			}
+
+			if !jsonEqual(got, tt.expected) {
+				t.Errorf("JSON mismatch\n\nMarkdown:\n%s\n\nExpected:\n%s\n\nGot:\n%s",
+					tt.markdown, prettyJSON(tt.expected), prettyJSON(got))
+			}
+		})
+	}
+}
+
+func TestRenderSlackMrkdwn(t *testing.T) {
+	renderer := NewMarkdownRenderer(slog.Default())
+
+	markdown := `# Alert Triggered
+
+- Service: **checkout-api**
+- Status: _critical_
+- Dashboard: [View Dashboard](https://example.com/dashboard)
+
+| Metric | Value | Threshold |
+| --- | --- | --- |
+| Latency | 250ms | 100ms |
+| Error Rate | 5.2% | 1% |
+
+` + "```" + `
+error: connection timeout after 30s
+` + "```"
+
+	expected := "*Alert Triggered*\n\n" +
+		"• Service: *checkout-api*\n" +
+		"• Status: _critical_\n" +
+		"• Dashboard: <https://example.com/dashboard|View Dashboard>\n\n" +
+		"```\nMetric     | Value | Threshold\n-----------|-------|----------\nLatency    | 250ms | 100ms    \nError Rate | 5.2%  | 1%       \n```\n\n" +
+		"```\nerror: connection timeout after 30s\n```\n\n"
+
+	got, err := renderer.Render(context.Background(), markdown, MarkdownFormatSlackMrkdwn)
+	if err != nil {
+		t.Fatalf("Render error: %v", err)
+	}
+
+	if got != expected {
+		t.Errorf("mrkdwn mismatch\n\nExpected:\n%q\n\nGot:\n%q", expected, got)
+	}
+}

pkg/templating/slackblockkitrenderer/extender.go

@@ -0,0 +1,23 @@
+package slackblockkitrenderer
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/renderer"
+	"github.com/yuin/goldmark/util"
+)
+
+type blockKitV2 struct{}
+
+// BlockKitV2 is a goldmark.Extender that configures the Slack Block Kit v2 renderer.
+var BlockKitV2 = &blockKitV2{}
+
+// Extend implements goldmark.Extender.
+func (e *blockKitV2) Extend(m goldmark.Markdown) {
+	extension.Table.Extend(m)
+	extension.Strikethrough.Extend(m)
+	extension.TaskList.Extend(m)
+	m.Renderer().AddOptions(
+		renderer.WithNodeRenderers(util.Prioritized(NewRenderer(), 1)),
+	)
+}

pkg/templating/slackblockkitrenderer/renderer_test.go

@@ -0,0 +1,542 @@
+package slackblockkitrenderer
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"github.com/yuin/goldmark"
+)
+
+func jsonEqual(a, b string) bool {
+	var va, vb interface{}
+	if err := json.Unmarshal([]byte(a), &va); err != nil {
+		return false
+	}
+	if err := json.Unmarshal([]byte(b), &vb); err != nil {
+		return false
+	}
+	ja, _ := json.Marshal(va)
+	jb, _ := json.Marshal(vb)
+	return string(ja) == string(jb)
+}
+
+func prettyJSON(s string) string {
+	var v interface{}
+	if err := json.Unmarshal([]byte(s), &v); err != nil {
+		return s
+	}
+	b, _ := json.MarshalIndent(v, "", "  ")
+	return string(b)
+}
+
+func TestRenderer(t *testing.T) {
+	tests := []struct {
+		name     string
+		markdown string
+		expected string
+	}{
+		{
+			name:     "empty input",
+			markdown: "",
+			expected: `[]`,
+		},
+		{
+			name:     "simple paragraph",
+			markdown: "Hello world",
+			expected: `[
+				{
+					"type": "section",
+					"text": { "type": "mrkdwn", "text": "Hello world" }
+				}
+			]`,
+		},
+		{
+			name:     "heading",
+			markdown: "# My Heading",
+			expected: `[
+				{
+					"type": "section",
+					"text": { "type": "mrkdwn", "text": "*My Heading*" }
+				}
+			]`,
+		},
+		{
+			name:     "multiple paragraphs",
+			markdown: "First paragraph\n\nSecond paragraph",
+			expected: `[
+				{
+					"type": "section",
+					"text": { "type": "mrkdwn", "text": "First paragraph\nSecond paragraph" }
+				}
+			]`,
+		},
+		{
+			name:     "todo list ",
+			markdown: "- [ ] item 1\n- [x] item 2",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+					{ 
+						"border": 0, 
+						"elements": [ 
+							{ "elements": [ { "text": "[ ] ", "type": "text" }, { "text": "item 1", "type": "text" } ], "type": "rich_text_section" }, 
+							{ "elements": [ { "text": "[x] ", "type": "text" }, { "text": "item 2", "type": "text" } ], "type": "rich_text_section" } 
+						],
+						"indent": 0, 
+						"style": "bullet", 
+						"type": "rich_text_list" 
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "thematic break between paragraphs",
+			markdown: "Before\n\n---\n\nAfter",
+			expected: `[
+				{ "type": "section", "text": { "type": "mrkdwn", "text": "Before" } },
+				{ "type": "divider" },
+				{ "type": "section", "text": { "type": "mrkdwn", "text": "After" } }
+			]`,
+		},
+		{
+			name:     "fenced code block with language",
+			markdown: "```go\nfmt.Println(\"hello\")\n```",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_preformatted",
+							"border": 0,
+							"language": "go",
+							"elements": [
+								{ "type": "text", "text": "fmt.Println(\"hello\")" }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "indented code block",
+			markdown: "    code line 1\n    code line 2",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_preformatted",
+							"border": 0,
+							"elements": [
+								{ "type": "text", "text": "code line 1\ncode line 2" }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "empty fenced code block",
+			markdown: "```\n```",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_preformatted",
+							"border": 0,
+							"elements": [
+								{ "type": "text", "text": " " }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "simple bullet list",
+			markdown: "- item 1\n- item 2\n- item 3",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "item 1" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "item 2" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "item 3" }] }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "simple ordered list",
+			markdown: "1. first\n2. second\n3. third",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "first" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "second" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "third" }] }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "nested bullet list (2 levels)",
+			markdown: "- item 1\n- item 2\n  - sub a\n  - sub b\n- item 3",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "item 1" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "item 2" }] }
+							]
+						},
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 1, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "sub a" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "sub b" }] }
+							]
+						},
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "item 3" }] }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "nested ordered list with offset",
+			markdown: "1. first\n   1. nested-a\n   2. nested-b\n2. second\n3. third",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "first" }] }
+							]
+						},
+						{
+							"type": "rich_text_list", "style": "ordered", "indent": 1, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "nested-a" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "nested-b" }] }
+							]
+						},
+						{
+							"type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0, "offset": 1,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "second" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "third" }] }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "mixed ordered/bullet nesting",
+			markdown: "1. ordered\n   - bullet child\n2. ordered again",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "ordered" }] }
+							]
+						},
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 1, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "bullet child" }] }
+							]
+						},
+						{
+							"type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0, "offset": 1,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "ordered again" }] }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "list items with bold/italic/link/code",
+			markdown: "- **bold item**\n- _italic item_\n- [link](http://example.com)\n- `code item`",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "bold item", "style": { "bold": true } }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "italic item", "style": { "italic": true } }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "link", "url": "http://example.com", "text": "link" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "code item", "style": { "code": true } }] }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "table with header and body",
+			markdown: "| Name | Age |\n|------|-----|\n| Alice | 30 |",
+			expected: `[
+				{
+					"type": "table",
+					"rows": [
+						[
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "Name", "style": { "bold": true } }] }
+							]},
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "Age", "style": { "bold": true } }] }
+							]}
+						],
+						[
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "Alice" }] }
+							]},
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "30" }] }
+							]}
+						]
+					]
+				}
+			]`,
+		},
+		{
+			name:     "blockquote",
+			markdown: "> quoted text",
+			expected: `[
+				{
+					"type": "section",
+					"text": { "type": "mrkdwn", "text": "> quoted text" }
+				}
+			]`,
+		},
+		{
+			name:     "blockquote with nested list",
+			markdown: "> item 1\n> > item 2\n> > item 3",
+			expected: `[
+				{
+					"text": {
+					"text": "> item 1\n> > item 2\n> > item 3",
+					"type": "mrkdwn"
+					},
+					"type": "section"
+				}
+			]`,
+		},
+		{
+			name:     "inline formatting in paragraph",
+			markdown: "This is **bold** and _italic_ and ~strike~ and `code`",
+			expected: `[
+				{
+					"type": "section",
+					"text": { "type": "mrkdwn", "text": "This is *bold* and _italic_ and ~strike~ and ` + "`code`" + `" }
+				}
+			]`,
+		},
+		{
+			name:     "link in paragraph",
+			markdown: "Visit [Google](http://google.com)",
+			expected: `[
+				{
+					"type": "section",
+					"text": { "type": "mrkdwn", "text": "Visit <http://google.com|Google>" }
+				}
+			]`,
+		},
+		{
+			name:     "image is skipped",
+			markdown: "![alt](http://example.com/image.png)",
+			// For image skip the block and return empty array
+			expected: `[]`,
+		},
+		{
+			name:     "paragraph then list then paragraph",
+			markdown: "Before\n\n- item\n\nAfter",
+			expected: `[
+				{ "type": "section", "text": { "type": "mrkdwn", "text": "Before" } },
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "item" }] }
+							]
+						}
+					]
+				},
+				{ "type": "section", "text": { "type": "mrkdwn", "text": "After" } }
+			]`,
+		},
+		{
+			name:     "ordered list with start > 1",
+			markdown: "5. fifth\n6. sixth",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0, "offset": 4,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "fifth" }] },
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "sixth" }] }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "deeply nested ordered list (3 levels) with offsets",
+			markdown: "1. Some things\n\t1. are best left\n2. to the fate\n\t1. of the world\n\t\t1. and then\n\t\t2. this is how\n3. it turns out to be",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{ "type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0,
+						  "elements": [{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "Some things" }] }] },
+
+						{ "type": "rich_text_list", "style": "ordered", "indent": 1, "border": 0,
+						  "elements": [{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "are best left" }] }] },
+
+						{ "type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0, "offset": 1,
+						  "elements": [{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "to the fate" }] }] },
+
+						{ "type": "rich_text_list", "style": "ordered", "indent": 1, "border": 0,
+						  "elements": [{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "of the world" }] }] },
+
+						{ "type": "rich_text_list", "style": "ordered", "indent": 2, "border": 0,
+						  "elements": [
+						    { "type": "rich_text_section", "elements": [{ "type": "text", "text": "and then" }] },
+						    { "type": "rich_text_section", "elements": [{ "type": "text", "text": "this is how" }] }
+						  ]
+						},
+
+						{ "type": "rich_text_list", "style": "ordered", "indent": 0, "border": 0, "offset": 2,
+						  "elements": [{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "it turns out to be" }] }] }
+					]
+				}
+			]`,
+		},
+		{
+			name:     "link with bold label in list item",
+			markdown: "- [**docs**](http://example.com)",
+			expected: `[
+				{
+					"type": "rich_text",
+					"elements": [
+						{
+							"type": "rich_text_list", "style": "bullet", "indent": 0, "border": 0,
+							"elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "link", "url": "http://example.com", "text": "docs" }] }
+							]
+						}
+					]
+				}
+			]`,
+		},
+		{
+			name:     "table with empty cell",
+			markdown: "| A | B |\n|---|---|\n| 1 | |",
+			expected: `[
+				{
+					"type": "table",
+					"rows": [
+						[
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "A", "style": { "bold": true } }] }
+							]},
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "B", "style": { "bold": true } }] }
+							]}
+						],
+						[
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "1" }] }
+							]},
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": " " }] }
+							]}
+						]
+					]
+				}
+			]`,
+		},
+		{
+			name:     "table with missing column in row",
+			markdown: "| A | B |\n|---|---|\n| 1 |",
+			expected: `[
+				{
+					"type": "table",
+					"rows": [
+						[
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "A", "style": { "bold": true } }] }
+							]},
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "B", "style": { "bold": true } }] }
+							]}
+						],
+						[
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": "1" }] }
+							]},
+							{ "type": "rich_text", "elements": [
+								{ "type": "rich_text_section", "elements": [{ "type": "text", "text": " " }] }
+							]}
+						]
+					]
+				}
+			]`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			md := goldmark.New(
+				goldmark.WithExtensions(BlockKitV2),
+			)
+			var buf bytes.Buffer
+			if err := md.Convert([]byte(tt.markdown), &buf); err != nil {
+				t.Fatalf("convert error: %v", err)
+			}
+			got := buf.String()
+			if !jsonEqual(got, tt.expected) {
+				t.Errorf("JSON mismatch\n\nMarkdown:\n%s\n\nExpected:\n%s\n\nGot:\n%s",
+					tt.markdown, prettyJSON(tt.expected), prettyJSON(got))
+			}
+		})
+	}
+}

pkg/templating/slackblockkitrenderer/slackblockkitrenderer.go

@@ -0,0 +1,737 @@
+package slackblockkitrenderer
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+
+	"github.com/yuin/goldmark/ast"
+	extensionast "github.com/yuin/goldmark/extension/ast"
+	"github.com/yuin/goldmark/renderer"
+	"github.com/yuin/goldmark/util"
+)
+
+// listFrame tracks state for a single level of list nesting.
+type listFrame struct {
+	style     string // "bullet" or "ordered"
+	indent    int
+	itemCount int
+}
+
+// listContext holds all state while processing a list tree.
+type listContext struct {
+	result             []RichTextList
+	stack              []listFrame
+	current            *RichTextList
+	currentItemInlines []interface{}
+}
+
+// tableContext holds state while processing a table.
+type tableContext struct {
+	rows               [][]TableCell
+	currentRow         []TableCell
+	currentCellInlines []interface{}
+	isHeader           bool
+}
+
+// Renderer converts Markdown AST to Slack Block Kit JSON.
+type Renderer struct {
+	blocks []interface{}
+	mrkdwn strings.Builder
+	// holds active styles for the current rich text element
+	styleStack []RichTextStyle
+	// holds the current list context while processing a list tree.
+	listCtx *listContext
+	// holds the current table context while processing a table.
+	tableCtx *tableContext
+	// stores the current blockquote depth while processing a blockquote.
+	// so blockquote with nested list can be rendered correctly.
+	blockquoteDepth int
+}
+
+// NewRenderer returns a new block kit renderer.
+func NewRenderer() renderer.NodeRenderer {
+	return &Renderer{}
+}
+
+// RegisterFuncs registers node rendering functions.
+func (r *Renderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
+	// Blocks
+	reg.Register(ast.KindDocument, r.renderDocument)
+	reg.Register(ast.KindHeading, r.renderHeading)
+	reg.Register(ast.KindParagraph, r.renderParagraph)
+	reg.Register(ast.KindThematicBreak, r.renderThematicBreak)
+	reg.Register(ast.KindCodeBlock, r.renderCodeBlock)
+	reg.Register(ast.KindFencedCodeBlock, r.renderFencedCodeBlock)
+	reg.Register(ast.KindBlockquote, r.renderBlockquote)
+	reg.Register(ast.KindList, r.renderList)
+	reg.Register(ast.KindListItem, r.renderListItem)
+	reg.Register(ast.KindImage, r.renderImage)
+
+	// Inlines
+	reg.Register(ast.KindText, r.renderText)
+	reg.Register(ast.KindEmphasis, r.renderEmphasis)
+	reg.Register(ast.KindCodeSpan, r.renderCodeSpan)
+	reg.Register(ast.KindLink, r.renderLink)
+
+	// Extensions
+	reg.Register(extensionast.KindStrikethrough, r.renderStrikethrough)
+	reg.Register(extensionast.KindTable, r.renderTable)
+	reg.Register(extensionast.KindTableHeader, r.renderTableHeader)
+	reg.Register(extensionast.KindTableRow, r.renderTableRow)
+	reg.Register(extensionast.KindTableCell, r.renderTableCell)
+	reg.Register(extensionast.KindTaskCheckBox, r.renderTaskCheckBox)
+}
+
+// inRichTextMode returns true when we're inside a list or table context
+// in slack blockkit list and table items are rendered as rich_text elements
+// if more cases are found in future those needs to be added here.
+func (r *Renderer) inRichTextMode() bool {
+	return r.listCtx != nil || r.tableCtx != nil
+}
+
+// currentStyle merges the stored style stack into RichTextStyle
+// which can be applied on rich text elements.
+func (r *Renderer) currentStyle() *RichTextStyle {
+	s := RichTextStyle{}
+	for _, f := range r.styleStack {
+		s.Bold = s.Bold || f.Bold
+		s.Italic = s.Italic || f.Italic
+		s.Strike = s.Strike || f.Strike
+		s.Code = s.Code || f.Code
+	}
+	if s == (RichTextStyle{}) {
+		return nil
+	}
+	return &s
+}
+
+// flushMrkdwn collects markdown text and adds it as a SectionBlock with mrkdwn text
+// whenever starting a new block we flush markdown to render it as a separate block.
+func (r *Renderer) flushMrkdwn() {
+	text := strings.TrimSpace(r.mrkdwn.String())
+	if text != "" {
+		r.blocks = append(r.blocks, SectionBlock{
+			Type: "section",
+			Text: &TextObject{
+				Type: "mrkdwn",
+				Text: text,
+			},
+		})
+	}
+	r.mrkdwn.Reset()
+}
+
+// addInline adds an inline element to the appropriate context.
+func (r *Renderer) addInline(el interface{}) {
+	if r.listCtx != nil {
+		r.listCtx.currentItemInlines = append(r.listCtx.currentItemInlines, el)
+	} else if r.tableCtx != nil {
+		r.tableCtx.currentCellInlines = append(r.tableCtx.currentCellInlines, el)
+	}
+}
+
+// --- Document ---
+
+func (r *Renderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.blocks = nil
+		r.mrkdwn.Reset()
+		r.styleStack = nil
+		r.listCtx = nil
+		r.tableCtx = nil
+		r.blockquoteDepth = 0
+	} else {
+		// on exiting the document node write the json for the collected blocks.
+		r.flushMrkdwn()
+		var data []byte
+		var err error
+		if len(r.blocks) > 0 {
+			data, err = json.Marshal(r.blocks)
+			if err != nil {
+				return ast.WalkStop, err
+			}
+		} else {
+			// if no blocks are collected, write an empty array.
+			data = []byte("[]")
+		}
+		_, err = w.Write(data)
+		if err != nil {
+			return ast.WalkStop, err
+		}
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- Heading ---
+
+func (r *Renderer) renderHeading(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.mrkdwn.WriteString("*")
+	} else {
+		r.mrkdwn.WriteString("*\n")
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- Paragraph ---
+
+func (r *Renderer) renderParagraph(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		if r.mrkdwn.Len() > 0 {
+			text := r.mrkdwn.String()
+			if !strings.HasSuffix(text, "\n") {
+				r.mrkdwn.WriteString("\n")
+			}
+		}
+		// handling of nested blockquotes
+		if r.blockquoteDepth > 0 {
+			r.mrkdwn.WriteString(strings.Repeat("> ", r.blockquoteDepth))
+		}
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- ThematicBreak ---
+
+func (r *Renderer) renderThematicBreak(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.flushMrkdwn()
+		r.blocks = append(r.blocks, DividerBlock{Type: "divider"})
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- CodeBlock (indented) ---
+
+func (r *Renderer) renderCodeBlock(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	r.flushMrkdwn()
+
+	var buf bytes.Buffer
+	lines := node.Lines()
+	for i := 0; i < lines.Len(); i++ {
+		line := lines.At(i)
+		buf.Write(line.Value(source))
+	}
+
+	text := buf.String()
+	// Remove trailing newline
+	text = strings.TrimRight(text, "\n")
+	// Slack API rejects empty text in rich_text_preformatted elements
+	if text == "" {
+		text = " "
+	}
+
+	elements := []interface{}{
+		RichTextInline{Type: "text", Text: text},
+	}
+
+	r.blocks = append(r.blocks, RichTextBlock{
+		Type: "rich_text",
+		Elements: []interface{}{
+			RichTextPreformatted{
+				Type:     "rich_text_preformatted",
+				Elements: elements,
+				Border:   0,
+			},
+		},
+	})
+	return ast.WalkContinue, nil
+}
+
+// --- FencedCodeBlock ---
+
+func (r *Renderer) renderFencedCodeBlock(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	r.flushMrkdwn()
+
+	n := node.(*ast.FencedCodeBlock)
+	var buf bytes.Buffer
+	lines := node.Lines()
+	for i := 0; i < lines.Len(); i++ {
+		line := lines.At(i)
+		buf.Write(line.Value(source))
+	}
+
+	text := buf.String()
+	text = strings.TrimRight(text, "\n")
+	// Slack API rejects empty text in rich_text_preformatted elements
+	if text == "" {
+		text = " "
+	}
+
+	elements := []interface{}{
+		RichTextInline{Type: "text", Text: text},
+	}
+
+	// If language is specified, collect it.
+	var language string
+	lang := n.Language(source)
+	if len(lang) > 0 {
+		language = string(lang)
+	}
+	// Add the preformatted block to the blocks slice with the collected language.
+	r.blocks = append(r.blocks, RichTextBlock{
+		Type: "rich_text",
+		Elements: []interface{}{
+			RichTextPreformatted{
+				Type:     "rich_text_preformatted",
+				Elements: elements,
+				Border:   0,
+				Language: language,
+			},
+		},
+	})
+
+	return ast.WalkSkipChildren, nil
+}
+
+// --- Blockquote ---
+
+func (r *Renderer) renderBlockquote(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.blockquoteDepth++
+	} else {
+		r.blockquoteDepth--
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- List ---
+
+func (r *Renderer) renderList(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	list := node.(*ast.List)
+
+	if entering {
+		style := "bullet"
+		if list.IsOrdered() {
+			style = "ordered"
+		}
+
+		if r.listCtx == nil {
+			// Top-level list: flush mrkdwn and create context
+			r.flushMrkdwn()
+			r.listCtx = &listContext{}
+		} else {
+			// Nested list: check if we already have some collected list items that needs to be flushed.
+			// in slack blockkit, list items with different levels of indentation are added as different rich_text_list blocks.
+			if len(r.listCtx.currentItemInlines) > 0 {
+				sec := RichTextBlock{
+					Type:     "rich_text_section",
+					Elements: r.listCtx.currentItemInlines,
+				}
+				if r.listCtx.current != nil {
+					r.listCtx.current.Elements = append(r.listCtx.current.Elements, sec)
+				}
+				r.listCtx.currentItemInlines = nil
+				// Increment parent's itemCount
+				if len(r.listCtx.stack) > 0 {
+					r.listCtx.stack[len(r.listCtx.stack)-1].itemCount++
+				}
+			}
+			// Finalize current list to result only if items were collected
+			if r.listCtx.current != nil && len(r.listCtx.current.Elements) > 0 {
+				r.listCtx.result = append(r.listCtx.result, *r.listCtx.current)
+			}
+		}
+
+		// the stack accumulated till this level derives hte indentation
+		// the stack get's collected as we go in more nested levels of list
+		// and as we get our of the nesting we remove the items from the slack
+		indent := len(r.listCtx.stack)
+		r.listCtx.stack = append(r.listCtx.stack, listFrame{
+			style:     style,
+			indent:    indent,
+			itemCount: 0,
+		})
+
+		newList := &RichTextList{
+			Type:     "rich_text_list",
+			Style:    style,
+			Indent:   indent,
+			Border:   0,
+			Elements: []interface{}{},
+		}
+
+		// Handle ordered list with start > 1
+		if list.IsOrdered() && list.Start > 1 {
+			newList.Offset = list.Start - 1
+		}
+
+		r.listCtx.current = newList
+
+	} else {
+		// Leaving list: finalize current list
+		if r.listCtx.current != nil && len(r.listCtx.current.Elements) > 0 {
+			r.listCtx.result = append(r.listCtx.result, *r.listCtx.current)
+		}
+
+		// Pop stack to so upcoming indentations can be handled correctly.
+		r.listCtx.stack = r.listCtx.stack[:len(r.listCtx.stack)-1]
+
+		if len(r.listCtx.stack) > 0 {
+			// Resume parent: start a new list segment at parent indent/style
+			parent := &r.listCtx.stack[len(r.listCtx.stack)-1]
+			newList := &RichTextList{
+				Type:     "rich_text_list",
+				Style:    parent.style,
+				Indent:   parent.indent,
+				Border:   0,
+				Elements: []interface{}{},
+			}
+			// Set offset for ordered parent continuation
+			if parent.style == "ordered" && parent.itemCount > 0 {
+				newList.Offset = parent.itemCount
+			}
+			r.listCtx.current = newList
+		} else {
+			// Top-level list is done since all stack are popped: build RichTextBlock if non-empty
+			if len(r.listCtx.result) > 0 {
+				elements := make([]interface{}, len(r.listCtx.result))
+				for i, l := range r.listCtx.result {
+					elements[i] = l
+				}
+				r.blocks = append(r.blocks, RichTextBlock{
+					Type:     "rich_text",
+					Elements: elements,
+				})
+			}
+			r.listCtx = nil
+		}
+	}
+
+	return ast.WalkContinue, nil
+}
+
+// --- ListItem ---
+
+func (r *Renderer) renderListItem(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.listCtx.currentItemInlines = nil
+	} else {
+		// Only add if there are inlines (might be empty after nested list consumed them)
+		if len(r.listCtx.currentItemInlines) > 0 {
+			sec := RichTextBlock{
+				Type:     "rich_text_section",
+				Elements: r.listCtx.currentItemInlines,
+			}
+			if r.listCtx.current != nil {
+				r.listCtx.current.Elements = append(r.listCtx.current.Elements, sec)
+			}
+			r.listCtx.currentItemInlines = nil
+			// Increment parent frame's itemCount
+			if len(r.listCtx.stack) > 0 {
+				r.listCtx.stack[len(r.listCtx.stack)-1].itemCount++
+			}
+		}
+	}
+
+	return ast.WalkContinue, nil
+}
+
+// --- Table ---
+// when table is encountered, we flush the markdown and create a table context.
+// when header row is encountered, we set the isHeader flag to true
+// when each row ends in renderTableRow we add that row to rows array of table context.
+// when table cell is encountered, we apply header related styles to the collected inline items,
+// all inline items are parsed as separate AST items like list item, links, text, etc. are collected
+// using the addInline function and wrapped in a rich_text_section block.
+
+func (r *Renderer) renderTable(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.flushMrkdwn()
+		r.tableCtx = &tableContext{}
+	} else {
+		// Pad short rows to match header column count for valid Block Kit payload
+		// without this slack blockkit attachment is invalid and the API fails
+		rows := r.tableCtx.rows
+		if len(rows) > 0 {
+			maxCols := len(rows[0])
+			for i, row := range rows {
+				for len(row) < maxCols {
+					emptySec := RichTextBlock{
+						Type:     "rich_text_section",
+						Elements: []interface{}{RichTextInline{Type: "text", Text: " "}},
+					}
+					row = append(row, TableCell{
+						Type:     "rich_text",
+						Elements: []interface{}{emptySec},
+					})
+				}
+				rows[i] = row
+			}
+		}
+		r.blocks = append(r.blocks, TableBlock{
+			Type: "table",
+			Rows: rows,
+		})
+		r.tableCtx = nil
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderTableHeader(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.tableCtx.isHeader = true
+		r.tableCtx.currentRow = nil
+	} else {
+		r.tableCtx.rows = append(r.tableCtx.rows, r.tableCtx.currentRow)
+		r.tableCtx.currentRow = nil
+		r.tableCtx.isHeader = false
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderTableRow(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.tableCtx.currentRow = nil
+	} else {
+		r.tableCtx.rows = append(r.tableCtx.rows, r.tableCtx.currentRow)
+		r.tableCtx.currentRow = nil
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderTableCell(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.tableCtx.currentCellInlines = nil
+	} else {
+		// If header, make text bold for the collected inline items.
+		if r.tableCtx.isHeader {
+			for i, el := range r.tableCtx.currentCellInlines {
+				if inline, ok := el.(RichTextInline); ok {
+					if inline.Style == nil {
+						inline.Style = &RichTextStyle{Bold: true}
+					} else {
+						inline.Style.Bold = true
+					}
+					r.tableCtx.currentCellInlines[i] = inline
+				}
+			}
+		}
+		// Ensure cell has at least one element for valid Block Kit payload
+		if len(r.tableCtx.currentCellInlines) == 0 {
+			r.tableCtx.currentCellInlines = []interface{}{
+				RichTextInline{Type: "text", Text: " "},
+			}
+		}
+		// All inline items that are collected for a table cell are wrapped in a rich_text_section block.
+		sec := RichTextBlock{
+			Type:     "rich_text_section",
+			Elements: r.tableCtx.currentCellInlines,
+		}
+		// The rich_text_section block is wrapped in a rich_text block.
+		cell := TableCell{
+			Type:     "rich_text",
+			Elements: []interface{}{sec},
+		}
+		r.tableCtx.currentRow = append(r.tableCtx.currentRow, cell)
+		r.tableCtx.currentCellInlines = nil
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- TaskCheckBox ---
+
+func (r *Renderer) renderTaskCheckBox(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	n := node.(*extensionast.TaskCheckBox)
+	text := "[ ] "
+	if n.IsChecked {
+		text = "[x] "
+	}
+	if r.inRichTextMode() {
+		r.addInline(RichTextInline{Type: "text", Text: text})
+	} else {
+		r.mrkdwn.WriteString(text)
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- Inline: Text ---
+
+func (r *Renderer) renderText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	n := node.(*ast.Text)
+	value := string(n.Segment.Value(source))
+
+	if r.inRichTextMode() {
+		r.addInline(RichTextInline{
+			Type:  "text",
+			Text:  value,
+			Style: r.currentStyle(),
+		})
+		if n.HardLineBreak() || n.SoftLineBreak() {
+			r.addInline(RichTextInline{Type: "text", Text: "\n"})
+		}
+	} else {
+		r.mrkdwn.WriteString(value)
+		if n.HardLineBreak() || n.SoftLineBreak() {
+			r.mrkdwn.WriteString("\n")
+			if r.blockquoteDepth > 0 {
+				r.mrkdwn.WriteString(strings.Repeat("> ", r.blockquoteDepth))
+			}
+		}
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- Inline: Emphasis ---
+
+func (r *Renderer) renderEmphasis(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	n := node.(*ast.Emphasis)
+	if r.inRichTextMode() {
+		if entering {
+			s := RichTextStyle{}
+			if n.Level == 1 {
+				s.Italic = true
+			} else {
+				s.Bold = true
+			}
+			r.styleStack = append(r.styleStack, s)
+		} else {
+			// the collected style gets used by the rich text element using currentStyle()
+			// so we remove this style from the stack.
+			if len(r.styleStack) > 0 {
+				r.styleStack = r.styleStack[:len(r.styleStack)-1]
+			}
+		}
+	} else {
+		if n.Level == 1 {
+			r.mrkdwn.WriteString("_")
+		} else {
+			r.mrkdwn.WriteString("*")
+		}
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- Inline: Strikethrough ---
+
+func (r *Renderer) renderStrikethrough(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if r.inRichTextMode() {
+		if entering {
+			r.styleStack = append(r.styleStack, RichTextStyle{Strike: true})
+		} else {
+			// the collected style gets used by the rich text element using currentStyle()
+			// so we remove this style from the stack.
+			if len(r.styleStack) > 0 {
+				r.styleStack = r.styleStack[:len(r.styleStack)-1]
+			}
+		}
+	} else {
+		r.mrkdwn.WriteString("~")
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- Inline: CodeSpan ---
+
+func (r *Renderer) renderCodeSpan(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	if r.inRichTextMode() {
+		// Collect all child text
+		var buf bytes.Buffer
+		for c := node.FirstChild(); c != nil; c = c.NextSibling() {
+			if t, ok := c.(*ast.Text); ok {
+				v := t.Segment.Value(source)
+				if bytes.HasSuffix(v, []byte("\n")) {
+					buf.Write(v[:len(v)-1])
+					buf.WriteByte(' ')
+				} else {
+					buf.Write(v)
+				}
+			} else if s, ok := c.(*ast.String); ok {
+				buf.Write(s.Value)
+			}
+		}
+		style := r.currentStyle()
+		if style == nil {
+			style = &RichTextStyle{Code: true}
+		} else {
+			style.Code = true
+		}
+		r.addInline(RichTextInline{
+			Type:  "text",
+			Text:  buf.String(),
+			Style: style,
+		})
+		return ast.WalkSkipChildren, nil
+	}
+	// mrkdwn mode
+	r.mrkdwn.WriteByte('`')
+	for c := node.FirstChild(); c != nil; c = c.NextSibling() {
+		if t, ok := c.(*ast.Text); ok {
+			v := t.Segment.Value(source)
+			if bytes.HasSuffix(v, []byte("\n")) {
+				r.mrkdwn.Write(v[:len(v)-1])
+				r.mrkdwn.WriteByte(' ')
+			} else {
+				r.mrkdwn.Write(v)
+			}
+		} else if s, ok := c.(*ast.String); ok {
+			r.mrkdwn.Write(s.Value)
+		}
+	}
+	r.mrkdwn.WriteByte('`')
+	return ast.WalkSkipChildren, nil
+}
+
+// --- Inline: Link ---
+
+func (r *Renderer) renderLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	n := node.(*ast.Link)
+	if r.inRichTextMode() {
+		if entering {
+			// Walk the entire subtree to collect text from all descendants,
+			// including nested inline nodes like emphasis, strong, code spans, etc.
+			var buf bytes.Buffer
+			_ = ast.Walk(node, func(child ast.Node, entering bool) (ast.WalkStatus, error) {
+				if !entering || child == node {
+					return ast.WalkContinue, nil
+				}
+				if t, ok := child.(*ast.Text); ok {
+					buf.Write(t.Segment.Value(source))
+				} else if s, ok := child.(*ast.String); ok {
+					buf.Write(s.Value)
+				}
+				return ast.WalkContinue, nil
+			})
+			// Once we've collected the text for the link (given it was present)
+			// let's add the link to the rich text block.
+			r.addInline(RichTextLink{
+				Type:  "link",
+				URL:   string(n.Destination),
+				Text:  buf.String(),
+				Style: r.currentStyle(),
+			})
+			return ast.WalkSkipChildren, nil
+		}
+	} else {
+		if entering {
+			r.mrkdwn.WriteString("<")
+			r.mrkdwn.Write(n.Destination)
+			r.mrkdwn.WriteString("|")
+		} else {
+			r.mrkdwn.WriteString(">")
+		}
+	}
+	return ast.WalkContinue, nil
+}
+
+// --- Image (skip) ---
+
+func (r *Renderer) renderImage(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	return ast.WalkSkipChildren, nil
+}

pkg/templating/slackblockkitrenderer/types.go

@@ -0,0 +1,80 @@
+package slackblockkitrenderer
+
+// SectionBlock represents a Slack section block with mrkdwn text.
+type SectionBlock struct {
+	Type string      `json:"type"`
+	Text *TextObject `json:"text"`
+}
+
+// DividerBlock represents a Slack divider block.
+type DividerBlock struct {
+	Type string `json:"type"`
+}
+
+// RichTextBlock is a container for rich text elements (lists, code blocks, table and cell blocks).
+type RichTextBlock struct {
+	Type     string        `json:"type"`
+	Elements []interface{} `json:"elements"`
+}
+
+// TableBlock represents a Slack table rendered as a rich_text block with preformatted text.
+type TableBlock struct {
+	Type string        `json:"type"`
+	Rows [][]TableCell `json:"rows"`
+}
+
+// TableCell is a cell in a table block.
+type TableCell struct {
+	Type     string        `json:"type"`
+	Elements []interface{} `json:"elements"`
+}
+
+// TextObject is the text field inside a SectionBlock.
+type TextObject struct {
+	Type string `json:"type"`
+	Text string `json:"text"`
+}
+
+// RichTextList represents an ordered or unordered list
+type RichTextList struct {
+	Type     string        `json:"type"`
+	Style    string        `json:"style"`
+	Indent   int           `json:"indent"`
+	Border   int           `json:"border"`
+	Offset   int           `json:"offset,omitempty"`
+	Elements []interface{} `json:"elements"`
+}
+
+// RichTextPreformatted represents a code block
+type RichTextPreformatted struct {
+	Type     string        `json:"type"`
+	Elements []interface{} `json:"elements"`
+	Border   int           `json:"border"`
+	Language string        `json:"language,omitempty"`
+}
+
+// RichTextInline represents inline text with optional styling
+// ex: text inside list, table cell
+type RichTextInline struct {
+	Type  string         `json:"type"`
+	Text  string         `json:"text"`
+	Style *RichTextStyle `json:"style,omitempty"`
+}
+
+// RichTextLink represents a link inside rich text
+// ex: link inside list, table cell
+type RichTextLink struct {
+	Type  string         `json:"type"`
+	URL   string         `json:"url"`
+	Text  string         `json:"text,omitempty"`
+	Style *RichTextStyle `json:"style,omitempty"`
+}
+
+// RichTextStyle holds boolean style flags for inline elements
+// these bools can toggle different styles for a rich text element at once.
+type RichTextStyle struct {
+	Bold   bool `json:"bold,omitempty"`
+	Italic bool `json:"italic,omitempty"`
+	Strike bool `json:"strike,omitempty"`
+	Code   bool `json:"code,omitempty"`
+}

pkg/templating/slackmrkdwnrenderer/extender.go

@@ -0,0 +1,22 @@
+package slackmrkdwnrenderer
+
+import (
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/renderer"
+	"github.com/yuin/goldmark/util"
+)
+
+type slackMrkdwn struct{}
+
+// SlackMrkdwn is a goldmark.Extender that configures the Slack mrkdwn renderer.
+var SlackMrkdwn = &slackMrkdwn{}
+
+// Extend implements goldmark.Extender.
+func (e *slackMrkdwn) Extend(m goldmark.Markdown) {
+	extension.Table.Extend(m)
+	extension.Strikethrough.Extend(m)
+	m.Renderer().AddOptions(
+		renderer.WithNodeRenderers(util.Prioritized(NewRenderer(), 1)),
+	)
+}

pkg/templating/slackmrkdwnrenderer/slack.go

@@ -0,0 +1,383 @@
+package slackmrkdwnrenderer
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/yuin/goldmark/ast"
+	extensionast "github.com/yuin/goldmark/extension/ast"
+	"github.com/yuin/goldmark/renderer"
+	"github.com/yuin/goldmark/util"
+)
+
+// Renderer renders nodes as Slack mrkdwn.
+type Renderer struct {
+	prefixes []string
+}
+
+// NewRenderer returns a new Renderer with given options.
+func NewRenderer() renderer.NodeRenderer {
+	return &Renderer{}
+}
+
+// RegisterFuncs implements NodeRenderer.RegisterFuncs.
+func (r *Renderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
+	// Blocks
+	reg.Register(ast.KindDocument, r.renderDocument)
+	reg.Register(ast.KindHeading, r.renderHeading)
+	reg.Register(ast.KindBlockquote, r.renderBlockquote)
+	reg.Register(ast.KindCodeBlock, r.renderCodeBlock)
+	reg.Register(ast.KindFencedCodeBlock, r.renderCodeBlock)
+	reg.Register(ast.KindList, r.renderList)
+	reg.Register(ast.KindListItem, r.renderListItem)
+	reg.Register(ast.KindParagraph, r.renderParagraph)
+	reg.Register(ast.KindTextBlock, r.renderTextBlock)
+	reg.Register(ast.KindRawHTML, r.renderRawHTML)
+	reg.Register(ast.KindThematicBreak, r.renderThematicBreak)
+
+	// Inlines
+	reg.Register(ast.KindAutoLink, r.renderAutoLink)
+	reg.Register(ast.KindCodeSpan, r.renderCodeSpan)
+	reg.Register(ast.KindEmphasis, r.renderEmphasis)
+	reg.Register(ast.KindImage, r.renderImage)
+	reg.Register(ast.KindLink, r.renderLink)
+	reg.Register(ast.KindText, r.renderText)
+
+	// Extensions
+	reg.Register(extensionast.KindStrikethrough, r.renderStrikethrough)
+	reg.Register(extensionast.KindTable, r.renderTable)
+}
+
+func (r *Renderer) writePrefix(w util.BufWriter) {
+	for _, p := range r.prefixes {
+		_, _ = w.WriteString(p)
+	}
+}
+
+// writeLineSeparator writes a newline followed by the current prefix.
+// Used for tight separations (e.g., between list items or text blocks).
+func (r *Renderer) writeLineSeparator(w util.BufWriter) {
+	_ = w.WriteByte('\n')
+	r.writePrefix(w)
+}
+
+// writeBlockSeparator writes a blank line separator between block-level elements,
+// respecting any active prefixes for proper nesting (e.g., inside blockquotes).
+func (r *Renderer) writeBlockSeparator(w util.BufWriter) {
+	r.writeLineSeparator(w)
+	r.writeLineSeparator(w)
+}
+
+// separateFromPrevious writes a block separator if the node has a previous sibling.
+func (r *Renderer) separateFromPrevious(w util.BufWriter, n ast.Node) {
+	if n.PreviousSibling() != nil {
+		r.writeBlockSeparator(w)
+	}
+}
+
+func (r *Renderer) renderDocument(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		_, _ = w.WriteString("\n\n")
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderHeading(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.separateFromPrevious(w, node)
+	}
+	_, _ = w.WriteString("*")
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderBlockquote(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.separateFromPrevious(w, n)
+		r.prefixes = append(r.prefixes, "> ")
+		_, _ = w.WriteString("> ")
+	} else {
+		r.prefixes = r.prefixes[:len(r.prefixes)-1]
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderCodeBlock(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.separateFromPrevious(w, n)
+		// start code block and write code line by line
+		_, _ = w.WriteString("```\n")
+		l := n.Lines().Len()
+		for i := 0; i < l; i++ {
+			line := n.Lines().At(i)
+			v := line.Value(source)
+			_, _ = w.Write(v)
+		}
+	} else {
+		_, _ = w.WriteString("```")
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderList(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		if node.PreviousSibling() != nil {
+			r.writeLineSeparator(w)
+			// another line break if not a nested list item and starting another block
+			if node.Parent() == nil || node.Parent().Kind() != ast.KindListItem {
+				r.writeLineSeparator(w)
+			}
+		}
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderListItem(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		if n.PreviousSibling() != nil {
+			r.writeLineSeparator(w)
+		}
+		parent := n.Parent().(*ast.List)
+		// compute and write the prefix based on list type and index
+		var prefixStr string
+		if parent.IsOrdered() {
+			index := parent.Start
+			for c := parent.FirstChild(); c != nil && c != n; c = c.NextSibling() {
+				index++
+			}
+			prefixStr = fmt.Sprintf("%d. ", index)
+		} else {
+			prefixStr = "• "
+		}
+		_, _ = w.WriteString(prefixStr)
+		r.prefixes = append(r.prefixes, "\t") // add tab for nested list items
+	} else {
+		r.prefixes = r.prefixes[:len(r.prefixes)-1]
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderParagraph(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.separateFromPrevious(w, n)
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderTextBlock(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering && n.PreviousSibling() != nil {
+		r.writeLineSeparator(w)
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderRawHTML(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		n := n.(*ast.RawHTML)
+		l := n.Segments.Len()
+		for i := 0; i < l; i++ {
+			segment := n.Segments.At(i)
+			_, _ = w.Write(segment.Value(source))
+		}
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderThematicBreak(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		r.separateFromPrevious(w, n)
+		_, _ = w.WriteString("---")
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderAutoLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	n := node.(*ast.AutoLink)
+	url := string(n.URL(source))
+	label := string(n.Label(source))
+
+	if n.AutoLinkType == ast.AutoLinkEmail && !strings.HasPrefix(strings.ToLower(url), "mailto:") {
+		url = "mailto:" + url
+	}
+
+	if url == label {
+		_, _ = fmt.Fprintf(w, "<%s>", url)
+	} else {
+		_, _ = fmt.Fprintf(w, "<%s|%s>", url, label)
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderCodeSpan(w util.BufWriter, source []byte, n ast.Node, entering bool) (ast.WalkStatus, error) {
+	if entering {
+		_ = w.WriteByte('`')
+		for c := n.FirstChild(); c != nil; c = c.NextSibling() {
+			segment := c.(*ast.Text).Segment
+			value := segment.Value(source)
+			if bytes.HasSuffix(value, []byte("\n")) { // replace newline with space
+				_, _ = w.Write(value[:len(value)-1])
+				_ = w.WriteByte(' ')
+			} else {
+				_, _ = w.Write(value)
+			}
+		}
+		return ast.WalkSkipChildren, nil
+	}
+	_ = w.WriteByte('`')
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderEmphasis(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	n := node.(*ast.Emphasis)
+	mark := "_"
+	if n.Level == 2 {
+		mark = "*"
+	}
+	_, _ = w.WriteString(mark)
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderLink(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	n := node.(*ast.Link)
+	if entering {
+		_, _ = w.WriteString("<")
+		_, _ = w.Write(util.URLEscape(n.Destination, true))
+		_, _ = w.WriteString("|")
+	} else {
+		_, _ = w.WriteString(">")
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderImage(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	n := node.(*ast.Image)
+	_, _ = w.WriteString("<")
+	_, _ = w.Write(util.URLEscape(n.Destination, true))
+	_, _ = w.WriteString("|")
+
+	// Write the alt text directly
+	var altBuf bytes.Buffer
+	for c := n.FirstChild(); c != nil; c = c.NextSibling() {
+		if textNode, ok := c.(*ast.Text); ok {
+			altBuf.Write(textNode.Segment.Value(source))
+		}
+	}
+	_, _ = w.Write(altBuf.Bytes())
+
+	_, _ = w.WriteString(">")
+	return ast.WalkSkipChildren, nil
+}
+
+func (r *Renderer) renderText(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+	n := node.(*ast.Text)
+	segment := n.Segment
+	value := segment.Value(source)
+	_, _ = w.Write(value)
+	if n.HardLineBreak() || n.SoftLineBreak() {
+		r.writeLineSeparator(w)
+	}
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderStrikethrough(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	_, _ = w.WriteString("~")
+	return ast.WalkContinue, nil
+}
+
+func (r *Renderer) renderTable(w util.BufWriter, source []byte, node ast.Node, entering bool) (ast.WalkStatus, error) {
+	if !entering {
+		return ast.WalkContinue, nil
+	}
+
+	r.separateFromPrevious(w, node)
+
+	// Collect cells and max widths
+	var rows [][]string
+	var colWidths []int
+
+	for c := node.FirstChild(); c != nil; c = c.NextSibling() {
+		if c.Kind() == extensionast.KindTableHeader || c.Kind() == extensionast.KindTableRow {
+			var row []string
+			colIdx := 0
+			for cc := c.FirstChild(); cc != nil; cc = cc.NextSibling() {
+				if cc.Kind() == extensionast.KindTableCell {
+					cellText := extractPlainText(cc, source)
+					row = append(row, cellText)
+					runeLen := utf8.RuneCountInString(cellText)
+					if colIdx >= len(colWidths) {
+						colWidths = append(colWidths, runeLen)
+					} else if runeLen > colWidths[colIdx] {
+						colWidths[colIdx] = runeLen
+					}
+					colIdx++
+				}
+			}
+			rows = append(rows, row)
+		}
+	}
+
+	// writing table in code block
+	_, _ = w.WriteString("```\n")
+	for i, row := range rows {
+		for colIdx, cellText := range row {
+			width := 0
+			if colIdx < len(colWidths) {
+				width = colWidths[colIdx]
+			}
+			runeLen := utf8.RuneCountInString(cellText)
+			padding := max(0, width-runeLen)
+
+			_, _ = w.WriteString(cellText)
+			_, _ = w.WriteString(strings.Repeat(" ", padding))
+			if colIdx < len(row)-1 {
+				_, _ = w.WriteString(" | ")
+			}
+		}
+		_ = w.WriteByte('\n')
+
+		// Print separator after header
+		if i == 0 {
+			for colIdx := range row {
+				width := 0
+				if colIdx < len(colWidths) {
+					width = colWidths[colIdx]
+				}
+				_, _ = w.WriteString(strings.Repeat("-", width))
+				if colIdx < len(row)-1 {
+					_, _ = w.WriteString("-|-")
+				}
+			}
+			_ = w.WriteByte('\n')
+		}
+	}
+	_, _ = w.WriteString("```")
+
+	return ast.WalkSkipChildren, nil
+}
+
+// extractPlainText extracts all the text content from the given node
+func extractPlainText(n ast.Node, source []byte) string {
+	var buf bytes.Buffer
+	_ = ast.Walk(n, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
+		if !entering {
+			return ast.WalkContinue, nil
+		}
+		if textNode, ok := node.(*ast.Text); ok {
+			buf.Write(textNode.Segment.Value(source))
+		} else if strNode, ok := node.(*ast.String); ok {
+			buf.Write(strNode.Value)
+		}
+		return ast.WalkContinue, nil
+	})
+	return strings.TrimSpace(buf.String())
+}

pkg/templating/slackmrkdwnrenderer/slack_test.go

@@ -0,0 +1,115 @@
+package slackmrkdwnrenderer
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/yuin/goldmark"
+)
+
+func TestRenderer(t *testing.T) {
+	tests := []struct {
+		name     string
+		markdown string
+		expected string
+	}{
+		{
+			name:     "Heading with Thematic Break",
+			markdown: "# Title 1\n# Hello World\n---\nthis is sometext",
+			expected: "*Title 1*\n\n*Hello World*\n\n---\n\nthis is sometext\n\n",
+		},
+		{
+			name:     "Blockquote",
+			markdown: "> This is a quote\n> It continues",
+			expected: "> This is a quote\n> It continues\n\n",
+		},
+		{
+			name:     "Fenced Code Block",
+			markdown: "```go\npackage main\nfunc main() {}\n```",
+			expected: "```\npackage main\nfunc main() {}\n```\n\n",
+		},
+		{
+			name:     "Unordered List",
+			markdown: "- item 1\n- item 2\n- item 3",
+			expected: "• item 1\n• item 2\n• item 3\n\n",
+		},
+		{
+			name:     "nested unordered list",
+			markdown: "- item 1\n- item 2\n\t- item 2.1\n\t\t- item 2.1.1\n\t\t- item 2.1.2\n\t- item 2.2\n- item 3",
+			expected: "• item 1\n• item 2\n\t• item 2.1\n\t\t• item 2.1.1\n\t\t• item 2.1.2\n\t• item 2.2\n• item 3\n\n",
+		},
+		{
+			name:     "Ordered List",
+			markdown: "1. item 1\n2. item 2\n3. item 3",
+			expected: "1. item 1\n2. item 2\n3. item 3\n\n",
+		},
+		{
+			name:     "nested ordered list",
+			markdown: "1. item 1\n2. item 2\n\t1. item 2.1\n\t\t1. item 2.1.1\n\t\t2. item 2.1.2\n\t2. item 2.2\n\t3. item 2.3\n3. item 3\n4. item 4",
+			expected: "1. item 1\n2. item 2\n\t1. item 2.1\n\t\t1. item 2.1.1\n\t\t2. item 2.1.2\n\t2. item 2.2\n\t3. item 2.3\n3. item 3\n4. item 4\n\n",
+		},
+		{
+			name:     "Links and AutoLinks",
+			markdown: "This is a [link](https://example.com) and an autolink <https://test.com>",
+			expected: "This is a <https://example.com|link> and an autolink <https://test.com>\n\n",
+		},
+		{
+			name:     "Images",
+			markdown: "An image ![alt text](https://example.com/image.png)",
+			expected: "An image <https://example.com/image.png|alt text>\n\n",
+		},
+		{
+			name:     "Emphasis",
+			markdown: "This is **bold** and *italic* and __bold__ and _italic_",
+			expected: "This is *bold* and _italic_ and *bold* and _italic_\n\n",
+		},
+		{
+			name:     "Strikethrough",
+			markdown: "This is ~~strike~~",
+			expected: "This is ~strike~\n\n",
+		},
+		{
+			name:     "Code Span",
+			markdown: "This is `inline code` embedded.",
+			expected: "This is `inline code` embedded.\n\n",
+		},
+		{
+			name:     "Table",
+			markdown: "Col 1 | Col 2 | Col 3\n--- | --- | ---\nVal 1 | Long Value 2 | 3\nShort | V | 1000",
+			expected: "```\nCol 1 | Col 2        | Col 3\n------|--------------|------\nVal 1 | Long Value 2 | 3    \nShort | V            | 1000 \n```\n\n",
+		},
+		{
+			name:     "Mixed Nested Lists",
+			markdown: "1. first\n\t- nested bullet\n\t- another bullet\n2. second",
+			expected: "1. first\n\t• nested bullet\n\t• another bullet\n2. second\n\n",
+		},
+		{
+			name:     "Email AutoLink",
+			markdown: "<user@example.com>",
+			expected: "<mailto:user@example.com|user@example.com>\n\n",
+		},
+		{
+			name:     "No value string parsed as is",
+			markdown: "Service: <no value>",
+			expected: "Service: <no value>\n\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			md := goldmark.New(goldmark.WithExtensions(SlackMrkdwn))
+
+			var buf bytes.Buffer
+			if err := md.Convert([]byte(tt.markdown), &buf); err != nil {
+				t.Fatalf("failed to convert: %v", err)
+			}
+
+			// Do exact string matching
+			actual := buf.String()
+			if actual != tt.expected {
+				t.Errorf("\nExpected:\n%q\nGot:\n%q\nRaw Expected:\n%s\nRaw Got:\n%s",
+					tt.expected, actual, tt.expected, actual)
+			}
+		})
+	}
+}

pkg/templating/templatingextensions/html.go

@@ -0,0 +1,66 @@
+package templatingextensions
+
+import (
+	"github.com/yuin/goldmark"
+	gast "github.com/yuin/goldmark/ast"
+	"github.com/yuin/goldmark/renderer"
+	"github.com/yuin/goldmark/renderer/html"
+	"github.com/yuin/goldmark/util"
+)
+
+// NoValueHTMLRenderer is a renderer.NodeRenderer implementation that
+// renders <no value> as escaped visible text instead of omitting it.
+type NoValueHTMLRenderer struct {
+	html.Config
+}
+
+// NewNoValueHTMLRenderer returns a new NoValueHTMLRenderer.
+func NewNoValueHTMLRenderer(opts ...html.Option) renderer.NodeRenderer {
+	r := &NoValueHTMLRenderer{
+		Config: html.NewConfig(),
+	}
+	for _, opt := range opts {
+		opt.SetHTMLOption(&r.Config)
+	}
+	return r
+}
+
+// RegisterFuncs implements renderer.NodeRenderer.RegisterFuncs.
+func (r *NoValueHTMLRenderer) RegisterFuncs(reg renderer.NodeRendererFuncRegisterer) {
+	reg.Register(gast.KindRawHTML, r.renderRawHTML)
+}
+
+func (r *NoValueHTMLRenderer) renderRawHTML(
+	w util.BufWriter, source []byte, node gast.Node, entering bool) (gast.WalkStatus, error) {
+	if !entering {
+		return gast.WalkSkipChildren, nil
+	}
+	if r.Unsafe {
+		n := node.(*gast.RawHTML)
+		for i := 0; i < n.Segments.Len(); i++ {
+			segment := n.Segments.At(i)
+			_, _ = w.Write(segment.Value(source))
+		}
+		return gast.WalkSkipChildren, nil
+	}
+	n := node.(*gast.RawHTML)
+	raw := string(n.Segments.Value(source))
+	if raw == "<no value>" {
+		_, _ = w.WriteString("&lt;no value&gt;")
+		return gast.WalkSkipChildren, nil
+	}
+	_, _ = w.WriteString("<!-- raw HTML omitted -->")
+	return gast.WalkSkipChildren, nil
+}
+
+type escapeNoValue struct{}
+
+// EscapeNoValue is an extension that renders <no value> as visible
+// escaped text instead of omitting it as raw HTML.
+var EscapeNoValue = &escapeNoValue{}
+
+func (e *escapeNoValue) Extend(m goldmark.Markdown) {
+	m.Renderer().AddOptions(renderer.WithNodeRenderers(
+		util.Prioritized(NewNoValueHTMLRenderer(), 500),
+	))
+}

pkg/templating/templatingextensions/html_test.go

@@ -0,0 +1,66 @@
+package templatingextensions
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/yuin/goldmark"
+	"github.com/yuin/goldmark/extension"
+)
+
+func TestEscapeNoValue(t *testing.T) {
+	tests := []struct {
+		name     string
+		markdown string
+		expected string
+	}{
+		{
+			name:     "plain text",
+			markdown: "Service: <no value>",
+			expected: "<p>Service: &lt;no value&gt;</p>\n",
+		},
+		{
+			name:     "inside strong",
+			markdown: "Service: **<no value>**",
+			expected: "<p>Service: <strong>&lt;no value&gt;</strong></p>\n",
+		},
+		{
+			name:     "inside emphasis",
+			markdown: "Service: *<no value>*",
+			expected: "<p>Service: <em>&lt;no value&gt;</em></p>\n",
+		},
+		{
+			name:     "inside strikethrough",
+			markdown: "Service: ~~<no value>~~",
+			expected: "<p>Service: <del>&lt;no value&gt;</del></p>\n",
+		},
+		{
+			name:     "real html still omitted",
+			markdown: "hello <div>world</div>",
+			expected: "<p>hello <!-- raw HTML omitted -->world<!-- raw HTML omitted --></p>\n",
+		},
+		{
+			name:     "inside heading",
+			markdown: "# Title <no value>",
+			expected: "<h1>Title &lt;no value&gt;</h1>\n",
+		},
+		{
+			name:     "inside list item",
+			markdown: "- item <no value>",
+			expected: "<ul>\n<li>item &lt;no value&gt;</li>\n</ul>\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gm := goldmark.New(goldmark.WithExtensions(EscapeNoValue, extension.Strikethrough))
+			var buf bytes.Buffer
+			if err := gm.Convert([]byte(tt.markdown), &buf); err != nil {
+				t.Fatal(err)
+			}
+			if buf.String() != tt.expected {
+				t.Errorf("expected:\n%s\ngot:\n%s", tt.expected, buf.String())
+			}
+		})
+	}
+}

pkg/types/alertmanagertypes/processor.go

@@ -0,0 +1,47 @@
+package alertmanagertypes
+
+import (
+	"context"
+
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// NotificationProcessor orchestrates template expansion and markdown rendering
+type NotificationProcessor interface {
+	ProcessAlertNotification(ctx context.Context, input NotificationProcessorInput, alerts []*types.Alert, rendererFormat markdownrenderer.OutputFormat) (*NotificationProcessorResult, error)
+	// RenderEmailNotification renders the given processor result into final HTML using
+	// the named layout template from the file template store.
+	// Returns an error if the template is not found.
+	RenderEmailNotification(ctx context.Context, templateName emailtypes.TemplateName, result *NotificationProcessorResult, alerts []*types.Alert) (string, error)
+}
+
+// NotificationProcessorInput carries the templates and rendering format for a notification
+type NotificationProcessorInput struct {
+	TitleTemplate        string
+	BodyTemplate         string
+	DefaultTitleTemplate string
+	DefaultBodyTemplate  string
+}
+
+// NotificationProcessorResult has the final expanded and rendered notification content
+type NotificationProcessorResult struct {
+	Title string
+	// Body contains per-alert rendered body strings.
+	Body []string
+	// IsDefaultTemplatedBody indicates the body came from default
+	// templates rather than custom annotation templates.
+	// Notifiers use this to decide presentation (e.g., Slack: single
+	// attachment vs. multiple BlockKit attachments).
+	IsDefaultTemplatedBody bool
+	// MissingVars is the union of unknown $variables found during
+	// custom template expansion.
+	MissingVars []string
+}
+
+// IsCustomTemplated returns true if the body came from custom annotation templates
+// rather than default templates.
+func (npr NotificationProcessorResult) IsCustomTemplated() bool {
+	return !npr.IsDefaultTemplatedBody
+}

pkg/types/alertmanagertypes/receiver.go

@@ -4,10 +4,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/prometheus/common/model"
 	"log/slog"
 	"time"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/template"
@@ -19,7 +20,7 @@ import (
 type (
 	// Receiver is the type for the receiver configuration.
 	Receiver                 = config.Receiver
-	ReceiverIntegrationsFunc = func(nc Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error)
+	ReceiverIntegrationsFunc = func(nc Receiver, tmpl *template.Template, logger *slog.Logger, processor NotificationProcessor) ([]notify.Integration, error)
 )
 
 // Creates a new receiver from a string. The input is initialized with the default values from the upstream alertmanager.
@@ -50,7 +51,7 @@ func NewReceiver(input string) (Receiver, error) {
 	return receiverWithDefaults, nil
 }
 
-func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFunc ReceiverIntegrationsFunc, config *Config, tmpl *template.Template, logger *slog.Logger, lSet model.LabelSet, alert ...*Alert) error {
+func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFunc ReceiverIntegrationsFunc, config *Config, tmpl *template.Template, logger *slog.Logger, processor NotificationProcessor, lSet model.LabelSet, alert ...*Alert) error {
 	ctx = notify.WithGroupKey(ctx, fmt.Sprintf("%s-%s-%d", receiver.Name, lSet.Fingerprint(), time.Now().Unix()))
 	ctx = notify.WithGroupLabels(ctx, lSet)
 	ctx = notify.WithReceiverName(ctx, receiver.Name)
@@ -72,7 +73,7 @@ func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFu
 		return err
 	}
 
-	integrations, err := receiverIntegrationsFunc(receiver, tmpl, logger)
+	integrations, err := receiverIntegrationsFunc(receiver, tmpl, logger, processor)
 	if err != nil {
 		return err
 	}

pkg/types/alertmanagertypes/template.go

@@ -11,19 +11,22 @@ import (
 	alertmanagertemplate "github.com/prometheus/alertmanager/template"
 )
 
+func AdditionalFuncMap() tmpltext.FuncMap {
+	return tmpltext.FuncMap{
+		// urlescape escapes the string for use in a URL query parameter.
+		// It returns tmplhtml.HTML to prevent the template engine from escaping the already escaped string.
+		// url.QueryEscape escapes spaces as "+", and html/template escapes "+" as "+" if tmplhtml.HTML is not used.
+		"urlescape": func(value string) tmplhtml.HTML {
+			return tmplhtml.HTML(url.QueryEscape(value))
+		},
+	}
+}
+
 // customTemplateOption returns an Option that adds custom functions to the template.
 func customTemplateOption() alertmanagertemplate.Option {
 	return func(text *tmpltext.Template, html *tmplhtml.Template) {
-		funcs := tmpltext.FuncMap{
-			// urlescape escapes the string for use in a URL query parameter.
-			// It returns tmplhtml.HTML to prevent the template engine from escaping the already escaped string.
-			// url.QueryEscape escapes spaces as "+", and html/template escapes "+" as "+" if tmplhtml.HTML is not used.
-			"urlescape": func(value string) tmplhtml.HTML {
-				return tmplhtml.HTML(url.QueryEscape(value))
-			},
-		}
-		text.Funcs(funcs)
-		html.Funcs(funcs)
+		text.Funcs(AdditionalFuncMap())
+		html.Funcs(AdditionalFuncMap())
 	}
 }
 

pkg/types/emailtypes/template.go

@@ -12,13 +12,14 @@ import (
 var (
 	// Templates is a list of all the templates that are supported by the emailing service.
 	// This list should be updated whenever a new template is added.
-	Templates = []TemplateName{TemplateNameInvitationEmail, TemplateNameResetPassword}
+	Templates = []TemplateName{TemplateNameInvitationEmail, TemplateNameResetPassword, TemplateNameAlertEmailNotification}
 )
 
 var (
 	TemplateNameInvitationEmail = TemplateName{valuer.NewString("invitation")}
 	TemplateNameResetPassword   = TemplateName{valuer.NewString("reset_password")}
-	TemplateNameAPIKeyEvent     = TemplateName{valuer.NewString("api_key_event")}
+	TemplateNameAPIKeyEvent              = TemplateName{valuer.NewString("api_key_event")}
+	TemplateNameAlertEmailNotification = TemplateName{valuer.NewString("alert_email_notification")}
 )
 
 type TemplateName struct{ valuer.String }
@@ -31,6 +32,8 @@ func NewTemplateName(name string) (TemplateName, error) {
 		return TemplateNameResetPassword, nil
 	case TemplateNameAPIKeyEvent.StringValue():
 		return TemplateNameAPIKeyEvent, nil
+	case TemplateNameAlertEmailNotification.StringValue():
+		return TemplateNameAlertEmailNotification, nil
 	default:
 		return TemplateName{}, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid template name: %s", name)
 	}

pkg/types/ruletypes/alerting.go

@@ -94,6 +94,29 @@ const (
 	ValueOutsideBounds CompareOp = "7"
 )
 
+// String returns a human-readable string for the comparison operator.
+// TODO: replace with String literals once the Refactor PR is merged
+func (c CompareOp) String() string {
+	switch c {
+	case ValueIsAbove:
+		return "above"
+	case ValueIsBelow:
+		return "below"
+	case ValueIsEq:
+		return "equal to"
+	case ValueIsNotEq:
+		return "not equal to"
+	case ValueAboveOrEq:
+		return "above or equal to"
+	case ValueBelowOrEq:
+		return "below or equal to"
+	case ValueOutsideBounds:
+		return "outside bounds"
+	default:
+		return string(c)
+	}
+}
+
 type MatchType string
 
 const (
@@ -105,6 +128,25 @@ const (
 	Last          MatchType = "5"
 )
 
+// String returns a human-readable string for the match type.
+// TODO: replace with String literals once the Refactor PR is merged
+func (m MatchType) String() string {
+	switch m {
+	case AtleastOnce:
+		return "at least once"
+	case AllTheTimes:
+		return "all the times"
+	case OnAverage:
+		return "on average"
+	case InTotal:
+		return "in total"
+	case Last:
+		return "last"
+	default:
+		return string(m)
+	}
+}
+
 type RuleCondition struct {
 	CompositeQuery    *v3.CompositeQuery `json:"compositeQuery,omitempty"`
 	CompareOp         CompareOp          `json:"op,omitempty"`

pkg/types/ruletypes/constants.go

@@ -9,4 +9,20 @@ const (
 	LabelSeverityName     = "severity"
 	LabelLastSeen         = "lastSeen"
 	LabelRuleId           = "ruleId"
+	LabelRuleSource       = "ruleSource"
+	LabelNoData           = "nodata"
+	LabelTestAlert        = "testalert"
+	LabelAlertName        = "alertname"
+	LabelIsRecovering     = "is_recovering"
+)
+
+const (
+	AnnotationRelatedLogs    = "related_logs"
+	AnnotationRelatedTraces  = "related_traces"
+	AnnotationTitleTemplate  = "title_template"
+	AnnotationBodyTemplate   = "body_template"
+	AnnotationValue          = "value"
+	AnnotationThresholdValue = "threshold.value"
+	AnnotationCompareOp      = "compare_op"
+	AnnotationMatchType      = "match_type"
 )

pkg/types/ruletypes/result_types.go

@@ -26,6 +26,11 @@ type Sample struct {
 	RecoveryTarget *float64
 
 	TargetUnit string
+
+	// CompareOp and MatchType carry the threshold evaluation context
+	// so the rule layer can set them as annotations for the notifier.
+	CompareOp CompareOp
+	MatchType MatchType
 }
 
 func (s Sample) String() string {

pkg/types/ruletypes/templating.go

@@ -0,0 +1,17 @@
+package ruletypes
+
+var CustomTemplatingAnnotations = []string{
+	AnnotationTitleTemplate,
+	AnnotationBodyTemplate,
+}
+
+// IsCustomTemplatingAnnotation checks if the given annotation is a custom templating annotation
+// in order to avoid expanding them in the rule manager layer.
+func IsCustomTemplatingAnnotation(name string) bool {
+	for _, annotation := range CustomTemplatingAnnotations {
+		if annotation == name {
+			return true
+		}
+	}
+	return false
+}

pkg/types/ruletypes/threshold.go

@@ -134,6 +134,8 @@ func (r BasicRuleThresholds) Eval(series v3.Series, unit string, evalData EvalDa
 				smpl.RecoveryTarget = threshold.RecoveryTarget
 			}
 			smpl.TargetUnit = threshold.TargetUnit
+			smpl.CompareOp = threshold.CompareOp
+			smpl.MatchType = threshold.MatchType
 			resultVector = append(resultVector, smpl)
 			continue
 		} else if evalData.SendUnmatched {
@@ -148,6 +150,8 @@ func (r BasicRuleThresholds) Eval(series v3.Series, unit string, evalData EvalDa
 				Metric:     PrepareSampleLabelsForRule(series.Labels, threshold.Name),
 				Target:     *threshold.TargetValue,
 				TargetUnit: threshold.TargetUnit,
+				CompareOp:  threshold.CompareOp,
+				MatchType:  threshold.MatchType,
 			}
 			if threshold.RecoveryTarget != nil {
 				smpl.RecoveryTarget = threshold.RecoveryTarget
@@ -169,6 +173,8 @@ func (r BasicRuleThresholds) Eval(series v3.Series, unit string, evalData EvalDa
 				smpl.Target = *threshold.TargetValue
 				smpl.RecoveryTarget = threshold.RecoveryTarget
 				smpl.TargetUnit = threshold.TargetUnit
+				smpl.CompareOp = threshold.CompareOp
+				smpl.MatchType = threshold.MatchType
 				// IsRecovering to notify that metrics is in recovery stage
 				smpl.IsRecovering = true
 				resultVector = append(resultVector, smpl)

templates/email/alert_email_notification.gotmpl

@@ -0,0 +1,120 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <title>{{.Title}}</title>
+  <style>
+    code {
+      background: #f0f0f0;
+      padding: 2px 6px;
+      border-radius: 3px;
+      font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
+      font-size: 13px;
+    }
+    pre {
+      background: #f0f0f0;
+      padding: 12px 16px;
+      border-radius: 6px;
+      font-size: 13px;
+      overflow-x: auto;
+      white-space: pre;
+    }
+    pre code {
+      background: none;
+      padding: 0;
+      border-radius: 0;
+      font-size: inherit;
+    }
+    table:not([role="presentation"]) {
+      width: 100%;
+      border-collapse: collapse;
+      font-size: 14px;
+    }
+    table:not([role="presentation"]) th {
+      font-weight: 600;
+      text-align: left;
+      padding: 8px 12px;
+      border-bottom: 2px solid #d0d0d0;
+    }
+    table:not([role="presentation"]) td {
+      padding: 8px 12px;
+      border-bottom: 1px solid #e8e8e8;
+    }
+    table:not([role="presentation"]) tr:last-child td {
+      border-bottom: none;
+    }
+  </style>
+</head>
+
+<body style="margin:0;padding:0;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,'Helvetica Neue',Arial,sans-serif;line-height:1.6;color:#333;background:#fff">
+  <table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0" style="background:#fff">
+    <tr>
+      <td align="center" style="padding:0">
+        <table role="presentation" width="600" cellspacing="0" cellpadding="0" border="0" style="max-width:600px;width:100%;border:1px solid #e2e2e2;border-radius:12px;overflow:hidden">
+
+          <tr>
+            <td align="center" style="padding:20px 20px 12px">
+              <h2 style="margin:0 0 8px;font-size:20px;color:#333">{{.Title}}</h2>
+              <p style="margin:0;font-size:14px;color:#666">
+                Status: <strong>{{.Status}}</strong>
+                {{if .TotalFiring}} | Firing: <strong style="color:#e53e3e">{{.TotalFiring}}</strong>{{end}}
+                {{if .TotalResolved}} | Resolved: <strong style="color:#38a169">{{.TotalResolved}}</strong>{{end}}
+              </p>
+            </td>
+          </tr>
+
+          <tr>
+            <td style="padding:0 20px">
+              <table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
+                <tr><td style="border-top:1px solid #e2e2e2;font-size:0;line-height:0" height="1">&nbsp;</td></tr>
+              </table>
+            </td>
+          </tr>
+
+          {{range .Bodies}}
+          <tr>
+            <td style="padding:8px 20px">
+              <table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
+                <tr>
+                  <td style="padding:16px;background:#fafafa;border:1px solid #e8e8e8;border-radius:6px">
+                    {{.}}
+                  </td>
+                </tr>
+              </table>
+            </td>
+          </tr>
+          {{end}}
+
+          {{if .ExternalURL}}
+          <tr>
+            <td style="padding:16px 20px">
+              <table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
+                <tr>
+                  <td align="center">
+                    <a href="{{.ExternalURL}}" target="_blank" style="display:inline-block;padding:12px 32px;font-size:14px;font-weight:600;color:#fff;background:#4E74F8;text-decoration:none;border-radius:4px">
+                      View in SigNoz
+                    </a>
+                  </td>
+                </tr>
+              </table>
+            </td>
+          </tr>
+          {{end}}
+
+          <tr>
+            <td align="center" style="padding:8px 16px 16px">
+              <p style="margin:0;font-size:12px;color:#999;line-height:1.5">
+                Sent by SigNoz AlertManager
+              </p>
+            </td>
+          </tr>
+        </table>
+      </td>
+    </tr>
+  </table>
+</body>
+
+</html>

tests/integration/conftest.py

@@ -22,6 +22,7 @@ pytest_plugins = [
     "fixtures.notification_channel",
     "fixtures.alerts",
     "fixtures.cloudintegrations",
+    "fixtures.maildev",
 ]
 
 

tests/integration/fixtures/alertutils.py

@@ -9,10 +9,34 @@ import requests
 
 from fixtures import types
 from fixtures.logger import setup_logger
+from fixtures.maildev import verify_email_received
 
 logger = setup_logger(__name__)
 
 
+def _is_json_subset(subset, superset) -> bool:
+    """Check if subset is contained within superset recursively.
+    - For dicts: all keys in subset must exist in superset with matching values
+    - For lists: all items in subset must be present in superset
+    - For scalars: exact equality
+    """
+    if isinstance(subset, dict):
+        if not isinstance(superset, dict):
+            return False
+        return all(
+            key in superset and _is_json_subset(value, superset[key])
+            for key, value in subset.items()
+        )
+    if isinstance(subset, list):
+        if not isinstance(superset, list):
+            return False
+        return all(
+            any(_is_json_subset(sub_item, sup_item) for sup_item in superset)
+            for sub_item in subset
+        )
+    return subset == superset
+
+
 def collect_webhook_firing_alerts(
     webhook_test_container: types.TestContainerDocker, notification_channel_name: str
 ) -> List[types.FiringAlert]:
@@ -73,6 +97,92 @@ def _verify_alerts_labels(
 
     return (fired_count, missing_alerts)
 
+def verify_webhook_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    validation_data: dict,
+) -> bool:
+    """Check if wiremock received a POST request at the given path
+    whose JSON body is a superset of the expected json_body."""
+    path = validation_data["path"]
+    json_body = validation_data["json_body"]
+
+    url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+    res = requests.post(url, json={"method": "POST", "url": path}, timeout=5)
+    assert res.status_code == HTTPStatus.OK, (
+        f"Failed to find requests for path {path}, "
+        f"status code: {res.status_code}, response: {res.text}"
+    )
+
+    for req in res.json()["requests"]:
+        body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+        if _is_json_subset(json_body, body):
+            logger.debug("Found request for path %s with body %s and expected body %s", path, json.dumps(body), json.dumps(json_body))
+            return True
+        else:
+            logger.debug("Request found for path %s with body %s does not contain expected body %s", path, json.dumps(body), json.dumps(json_body))
+    return False
+
+
+def _check_validation(
+    validation: types.NotificationValidation,
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+) -> bool:
+    """Dispatch a single validation check to the appropriate verifier."""
+    if validation.destination_type == "webhook":
+        return verify_webhook_notification_expectation(
+            notification_channel, validation.validation_data
+        )
+    if validation.destination_type == "email":
+        return verify_email_received(maildev, validation.validation_data)
+    raise ValueError(f"Invalid destination type: {validation.destination_type}")
+
+
+def verify_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    expected_notification: types.AMNotificationExpectation,
+) -> bool:
+    """Poll for expected notifications across webhook and email channels.
+    Follows the same wait-and-check pattern as verify_webhook_alert_expectation."""
+    time_to_wait = datetime.now() + timedelta(
+        seconds=expected_notification.wait_time_seconds
+    )
+
+    while datetime.now() < time_to_wait:
+        all_found = all(
+            _check_validation(v, notification_channel, maildev)
+            for v in expected_notification.notification_validations
+        )
+
+        if expected_notification.should_notify and all_found:
+            logger.info("All expected notifications found")
+            return True
+
+        time.sleep(1)
+
+    # Timeout reached
+    if not expected_notification.should_notify:
+        # Verify no notifications were received
+        for validation in expected_notification.notification_validations:
+            found = _check_validation(validation, notification_channel, maildev)
+            assert not found, (
+                f"Expected no notification but found one for "
+                f"{validation.destination_type} with data {validation.validation_data}"
+            )
+        logger.info("No notifications found, as expected")
+        return True
+
+    # Expected notifications but didn't get them all — report missing
+    missing = [
+        v for v in expected_notification.notification_validations
+        if not _check_validation(v, notification_channel, maildev)
+    ]
+    assert len(missing) == 0, (
+        f"Expected all notifications to be found but missing: {missing}"
+    )
+    return True
+
 
 def verify_webhook_alert_expectation(
     test_alert_container: types.TestContainerDocker,
@@ -135,6 +245,43 @@ def verify_webhook_alert_expectation(
     return True  # should not reach here
 
 
+def update_channel_config_urls(
+    channel_config: dict,
+    notification_channel: types.TestContainerDocker,
+) -> dict:
+    """
+    Updates the API/webhook URLs in channel config to point to the wiremock
+    notification_channel container. Only modifies URL-bearing configs:
+    slack_configs, msteams_configs, webhook_configs.
+
+    PagerDuty and OpsGenie URLs come from the alertmanager global config
+    (overridden via env vars), so they are not modified here.
+    """
+    from urllib.parse import urlparse
+
+    config = channel_config.copy()
+
+    url_field_map = {
+        "slack_configs": "api_url",
+        "msteams_configs": "webhook_url",
+        "webhook_configs": "url",
+        "pagerduty_configs": "url",
+        "opsgenie_configs": "api_url",
+    }
+
+    for config_key, url_field in url_field_map.items():
+        if config_key in config:
+            for entry in config[config_key]:
+                if url_field in entry:
+                    original_url = entry[url_field]
+                    path = urlparse(original_url).path
+                    entry[url_field] = notification_channel.container_configs[
+                        "8080"
+                    ].get(path)
+
+    return config
+
+
 def update_rule_channel_name(rule_data: dict, channel_name: str):
     """
     updates the channel name in the thresholds

tests/integration/fixtures/maildev.py

@@ -0,0 +1,136 @@
+from http import HTTPStatus
+import json
+from typing import List
+
+import docker
+import docker.errors
+import pytest
+import requests
+from testcontainers.core.container import DockerContainer, Network
+
+from fixtures import dev, types
+from fixtures.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+@pytest.fixture(name="maildev", scope="package")
+def maildev(
+    network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config
+) -> types.TestContainerDocker:
+    """
+    Package-scoped fixture for MailDev container.
+    Provides SMTP (port 1025) and HTTP API (port 1080) for email testing.
+    """
+
+    def create() -> types.TestContainerDocker:
+        container = DockerContainer(image="maildev/maildev:2.1.0")
+        container.with_exposed_ports(1025, 1080)
+        container.with_network(network=network)
+        container.start()
+
+        return types.TestContainerDocker(
+            id=container.get_wrapped_container().id,
+            host_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1025),
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1080),
+                ),
+            },
+            container_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_wrapped_container().name,
+                    port=1025,
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_wrapped_container().name,
+                    port=1080,
+                ),
+            },
+        )
+
+    def delete(container: types.TestContainerDocker):
+        client = docker.from_env()
+        try:
+            client.containers.get(container_id=container.id).stop()
+            client.containers.get(container_id=container.id).remove(v=True)
+        except docker.errors.NotFound:
+            logger.info(
+                "Skipping removal of MailDev, MailDev(%s) not found. Maybe it was manually removed?",
+                {"id": container.id},
+            )
+
+    def restore(cache: dict) -> types.TestContainerDocker:
+        return types.TestContainerDocker.from_cache(cache)
+
+    return dev.wrap(
+        request,
+        pytestconfig,
+        "maildev",
+        lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
+        create,
+        delete,
+        restore,
+    )
+
+
+def get_all_mails(maildev: types.TestContainerDocker) -> List[dict]:
+    """
+    Fetches all emails from the MailDev HTTP API.
+    Returns list of dicts with keys: subject, html, text.
+    """
+    url = maildev.host_configs["1080"].get("/email")
+    response = requests.get(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, (
+        f"Failed to fetch emails from MailDev, "
+        f"status code: {response.status_code}, response: {response.text}"
+    )
+    emails = response.json()
+    logger.debug("Emails: %s", json.dumps(emails, indent=2))
+    return [
+        {
+            "subject": email.get("subject", ""),
+            "html": email.get("html", ""),
+            "text": email.get("text", ""),
+        }
+        for email in emails
+    ]
+
+
+def verify_email_received(
+    maildev: types.TestContainerDocker, filters: dict
+) -> bool:
+    """
+    Checks if any email in MailDev matches all the given filters.
+    Filters are matched with exact equality against the email fields (subject, html, text).
+    Returns True if at least one matching email is found.
+    """
+    emails = get_all_mails(maildev)
+    for email in emails:
+        logger.debug("Email: %s", json.dumps(email, indent=2))
+        if all(
+            key in email and filter_value == email[key]
+            for key, filter_value in filters.items()
+        ):
+            return True
+    return False
+
+
+def delete_all_mails(maildev: types.TestContainerDocker) -> None:
+    """
+    Deletes all emails from the MailDev inbox.
+    """
+    url = maildev.host_configs["1080"].get("/email/all")
+    response = requests.delete(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, (
+        f"Failed to delete emails from MailDev, "
+        f"status code: {response.status_code}, response: {response.text}"
+    )

tests/integration/fixtures/notification_channel.py

@@ -70,6 +70,29 @@ def notification_channel(
         restore,
     )
 
+@pytest.fixture(name="create_notification_channel", scope="function")
+def create_notification_channel(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> Callable[[dict], str]:
+    admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    def _create_notification_channel(channel_config: dict) -> str:
+        response = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels"),
+            json=channel_config,
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        assert response.status_code == HTTPStatus.CREATED, (
+            f"Failed to create channel, "
+            f"Response: {response.text} "
+            f"Response status: {response.status_code}"
+        )
+        return response.json()["data"]["id"]
+
+    return _create_notification_channel
 
 @pytest.fixture(name="create_webhook_notification_channel", scope="function")
 def create_webhook_notification_channel(

tests/integration/fixtures/signoz.py

@@ -16,6 +16,11 @@ from fixtures.logger import setup_logger
 
 logger = setup_logger(__name__)
 
+# Toggle: True = build SigNoz image from local Dockerfile.
+# False = pull public image from Docker Hub (https://hub.docker.com/r/signoz/signoz/tags).
+BUILD_LOCAL_IMAGE = False
+REMOTE_IMAGE = "signoz/signoz:latest"
+
 
 def create_signoz(
     network: Network,
@@ -44,22 +49,28 @@ def create_signoz(
         if arch == "x86_64":
             arch = "amd64"
 
-        # Build the image
-        dockerfile_path = "cmd/enterprise/Dockerfile.integration"
-        if with_web:
-            dockerfile_path = "cmd/enterprise/Dockerfile.with-web.integration"
+        if BUILD_LOCAL_IMAGE:
+            # Build the image from local Dockerfile
+            dockerfile_path = "cmd/enterprise/Dockerfile.integration"
+            if with_web:
+                dockerfile_path = "cmd/enterprise/Dockerfile.with-web.integration"
 
-        self = DockerImage(
-            path="../../",
-            dockerfile_path=dockerfile_path,
-            tag="signoz:integration",
-            buildargs={
-                "TARGETARCH": arch,
-                "ZEUSURL": zeus.container_configs["8080"].base(),
-            },
-        )
+            self = DockerImage(
+                path="../../",
+                dockerfile_path=dockerfile_path,
+                tag="signoz:integration",
+                buildargs={
+                    "TARGETARCH": arch,
+                    "ZEUSURL": zeus.container_configs["8080"].base(),
+                },
+            )
 
-        self.build()
+            self.build()
+            image = "signoz:integration"
+        else:
+            # Pull public image from Docker Hub
+            docker.from_env().images.pull(REMOTE_IMAGE)
+            image = REMOTE_IMAGE
 
         env = (
             {
@@ -87,7 +98,7 @@ def create_signoz(
         if env_overrides:
             env = env | env_overrides
 
-        container = DockerContainer("signoz:integration")
+        container = DockerContainer(image)
         for k, v in env.items():
             container.with_env(k, v)
         container.with_exposed_ports(8080)

tests/integration/fixtures/types.py

@@ -203,3 +203,37 @@ class AlertTestCase:
     alert_data: List[AlertData]
     # list of alert expectations for the test case
     alert_expectation: AlertExpectation
+
+@dataclass(frozen=True)
+class NotificationValidation:
+    # destination type of the notification, either webhook or email
+    # slack, msteams, pagerduty, opsgenie, webhook channels send notifications through webhook
+    # email channels send notifications through email
+    destination_type: Literal["webhook", "email"]
+    # validation data for validating the received notification payload
+    validation_data: dict[str, any]
+
+@dataclass(frozen=True)
+class AMNotificationExpectation:
+    # whether we expect any notifications to be fired or not, false when testing downtime scenarios
+    # or don't expect any notifications to be fired in given time period
+    should_notify: bool
+    # seconds to wait for the notifications to be fired, if no
+    # notifications are fired in the expected time, the test will fail
+    wait_time_seconds: int
+    # list of notifications to expect, as a single rule can trigger multiple notifications
+    # spanning across different notifiers
+    notification_validations: List[NotificationValidation]
+
+@dataclass(frozen=True)
+class AlertManagerNotificationTestCase:
+    # name of the test case
+    name: str
+    # path to the rule file in testdata directory
+    rule_path: str
+    # list of alert data that will be inserted into the database for the rule to be triggered
+    alert_data: List[AlertData]
+    # configuration for the notification channel
+    channel_config: dict[str, any]
+    # notification expectations for the test case
+    notification_expectation: AMNotificationExpectation

tests/integration/src/alertmanager/01_notifers.py

@@ -0,0 +1,371 @@
+import json
+import uuid
+from datetime import datetime, timedelta, timezone
+from typing import Callable, List
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alertutils import (
+    update_channel_config_urls,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.utils import get_testdata_file_path
+
+
+"""
+Default notification configs for each of the notifiers
+"""
+
+slack_default_config = {
+    # channel name configured on runtime
+    "slack_configs": [{
+        "api_url": "services/TEAM_ID/BOT_ID/TOKEN_ID",
+        "title":"[{{ .Status | toUpper }}{{ if eq .Status \"firing\" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{\" \"}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}=\"{{ $label.Value -}}\"\n {{- end }}\n {{- end -}}\n )\n {{- end }}",
+        "text":"{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name \"ruleId\" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}"
+    }],
+}
+
+# MSTeams default config
+msteams_default_config = {
+    "msteams_configs": [{
+        "webhook_url": "msteams/webhook_url",
+        "title":"[{{ .Status | toUpper }}{{ if eq .Status \"firing\" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{\" \"}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}=\"{{ $label.Value -}}\"\n {{- end }}\n {{- end -}}\n )\n {{- end }}",
+        "text":"{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name \"ruleId\" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}"
+    }],
+}
+
+# pagerduty default config
+pagerduty_default_config = {
+    "pagerduty_configs": [{
+        "routing_key":"PagerDutyRoutingKey",
+        "url":"v2/enqueue",
+        "client":"SigNoz Alert Manager",
+        "client_url":"https://enter-signoz-host-n-port-here/alerts",
+        "description":"[{{ .Status | toUpper }}{{ if eq .Status \"firing\" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n\t{{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n\t {{\" \"}}(\n\t {{- with .CommonLabels.Remove .GroupLabels.Names }}\n\t\t{{- range $index, $label := .SortedPairs -}}\n\t\t {{ if $index }}, {{ end }}\n\t\t {{- $label.Name }}=\"{{ $label.Value -}}\"\n\t\t{{- end }}\n\t {{- end -}}\n\t )\n\t{{- end }}",
+        "details":{
+            "firing":"{{ template \"pagerduty.default.instances\" .Alerts.Firing }}",
+            "num_firing":"{{ .Alerts.Firing | len }}",
+            "num_resolved":"{{ .Alerts.Resolved | len }}",
+            "resolved":"{{ template \"pagerduty.default.instances\" .Alerts.Resolved }}"
+        },
+        "source":"SigNoz Alert Manager",
+        "severity":"{{ (index .Alerts 0).Labels.severity }}"
+    }],
+}
+# opsgenie default config
+opsgenie_default_config = {
+   "opsgenie_configs": [
+    {
+      "api_key": "OpsGenieAPIKey",
+      "api_url": "/",
+      "description": "{{ if gt (len .Alerts.Firing) 0 -}}\r\n\tAlerts Firing:\r\n\t{{ range .Alerts.Firing }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name \"ruleId\" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n\tAlerts Resolved:\r\n\t{{ range .Alerts.Resolved }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name \"ruleId\" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}",
+      "priority": "{{ if eq (index .Alerts 0).Labels.severity \"critical\" }}P1{{ else if eq (index .Alerts 0).Labels.severity \"warning\" }}P2{{ else if eq (index .Alerts 0).Labels.severity \"info\" }}P3{{ else }}P4{{ end }}",
+      "message": "{{ .CommonLabels.alertname }}",
+      "details": {}
+    }
+  ]
+}
+
+# webhook default config
+webhook_default_config = {
+    "webhook_configs": [{
+        "url": "webhook/webhook_url",
+    }],
+}
+# email default config
+email_default_config = {
+    "email_configs": [{
+        "to": "test@example.com",
+        "html": "<html><body>{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name \"ruleId\" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}</body></html>",
+        "headers": {
+            "Subject": "[{{ .Status | toUpper }}{{ if eq .Status \"firing\" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{\" \"}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}=\"{{ $label.Value -}}\"\n {{- end }}\n {{- end -}}\n )\n {{- end }}"
+        }
+    }],
+}
+
+# tests to verify the notifiers sending out the notifications with expected content
+# test out all notifiers integration weather they're sending out notifications with default 
+# templating for the following notifiers: webhook, slack, email, pagerduty, opsgenie, msteams
+NOTIFIERS_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="slack_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=30,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "attachments": [
+                                {
+                                "title": "[FIRING:1] threshold_above_at_least_once for  (alertname=\"threshold_above_at_least_once\", severity=\"critical\", threshold.name=\"critical\")",
+                                "text": "*Alert:* threshold_above_at_least_once - critical\r\n\r\n *Summary:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n *Description:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n *RelatedLogs:* \r\n *RelatedTraces:* \r\n\r\n *Details:*\r\n • *alertname:* threshold_above_at_least_once\r\n   • *severity:* critical\r\n   • *threshold.name:* critical\r\n   ",
+                            }]}
+                    },
+                ),
+            ],
+        ),
+    ),
+
+
+
+    types.AlertManagerNotificationTestCase(
+        name="msteams_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=30,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "@context": "http://schema.org/extensions",
+                            "type": "MessageCard",
+                            "title": "[FIRING:1] threshold_above_at_least_once for  (alertname=\"threshold_above_at_least_once\", severity=\"critical\", threshold.name=\"critical\")",
+                            "text": "*Alert:* threshold_above_at_least_once - critical\r\n\r\n *Summary:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n *Description:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n *RelatedLogs:* \r\n *RelatedTraces:* \r\n\r\n *Details:*\r\n • *alertname:* threshold_above_at_least_once\r\n   • *severity:* critical\r\n   • *threshold.name:* critical\r\n   ",
+                            "themeColor": "8C1A1A"
+                        }
+                    },
+                ),
+            ],
+        ),
+    ),
+
+
+
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=30,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "payload": {
+                                "summary": "[FIRING:1] threshold_above_at_least_once for  (alertname=\"threshold_above_at_least_once\", severity=\"critical\", threshold.name=\"critical\")",
+                                "custom_details": {
+                                "firing": {
+                                    "Annotations": [{"description = This alert is fired when the defined metric (current value": "15) crosses the threshold (10)"}],
+                                    "Labels": ["alertname = threshold_above_at_least_once","severity = critical","threshold.name = critical"],
+                                }}
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts"
+                            }
+                    },
+                ),
+            ],
+        ),
+    ),
+
+
+
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=30,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "threshold_above_at_least_once",
+                            "description": "Alerts Firing:\r\n\t\r\n\t - Message: This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\tLabels:\r\n\t   - alertname = threshold_above_at_least_once\r\n\t   - severity = critical\r\n\t   - threshold.name = critical\r\n\t   Annotations:\r\n\t   - description = This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\t   - summary = This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\t   Source: \r\n\t\r\n",
+                            "details": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical"
+                            },
+                            "priority": "P1"
+                        }
+                    },
+                ),
+            ],
+        ),
+    ),
+
+
+
+    types.AlertManagerNotificationTestCase(
+        name="webhook_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=30,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [{
+                                "status": "firing",
+                                "labels": {"alertname": "threshold_above_at_least_once","severity": "critical","threshold.name": "critical"},
+                                "annotations": {"summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)"
+                                }}
+                            ],
+                            "commonLabels": {"alertname": "threshold_above_at_least_once","severity": "critical","threshold.name": "critical"},
+                            "commonAnnotations": {"summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)"}
+                            }
+                    },
+                ),
+            ],
+        ),
+    ),
+
+    types.AlertManagerNotificationTestCase(
+        name="email_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=email_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=30,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="email",
+                    validation_data={
+                        "subject": "[FIRING:1] threshold_above_at_least_once for  (alertname=\"threshold_above_at_least_once\", severity=\"critical\", threshold.name=\"critical\")",
+                        "html": "<html><body>*Alert:* threshold_above_at_least_once - critical\n\n *Summary:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\n *Description:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\n *RelatedLogs:* \n *RelatedTraces:* \n\n *Details:*\n \u2022 *alertname:* threshold_above_at_least_once\n   \u2022 *severity:* critical\n   \u2022 *threshold.name:* critical\n   </body></html>"                    
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+logger = setup_logger(__name__)
+
+@pytest.mark.parametrize(
+    "notifier_test_case",
+    NOTIFIERS_TEST,
+    ids=lambda notifier_test_case: notifier_test_case.name,
+)
+def test_notifier_templating(
+    # Notification channel related fixtures
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, List[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    # Alert rule related fixtures
+    create_alert_rule: Callable[[dict], str],
+    # Alert data insertion related fixtures
+    insert_alert_data: Callable[[List[types.AlertData], datetime], None],
+    # Mail dev container for email verification
+    maildev: types.TestContainerDocker,
+    # Test case from parametrize
+    notifier_test_case: types.AlertManagerNotificationTestCase,
+):
+    # Generate unique channel name
+    channel_name = str(uuid.uuid4())
+
+    # Update channel config: set name and rewrite URLs to wiremock
+    channel_config = update_channel_config_urls(
+        notifier_test_case.channel_config, notification_channel
+    )
+    channel_config["name"] = channel_name
+
+    # Setup wiremock mocks for webhook-based validations
+    webhook_validations = [
+        v
+        for v in notifier_test_case.notification_expectation.notification_validations
+        if v.destination_type == "webhook"
+    ]
+    mock_mappings = [
+        Mapping(
+            request=MappingRequest(
+                method=HttpMethods.POST, url=v.validation_data["path"]
+            ),
+            response=MappingResponse(status=200, json_body={}),
+            persistent=False,
+        )
+        for v in webhook_validations
+    ]
+    if mock_mappings:
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created: %s", {"mock_mappings": mock_mappings})
+
+    # Create notification channel
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    # Insert alert data
+    insert_alert_data(
+        notifier_test_case.alert_data,
+        base_time=datetime.now(tz=timezone.utc) - timedelta(minutes=5),
+    )
+
+    # Create alert rule
+    rule_path = get_testdata_file_path(notifier_test_case.rule_path)
+    with open(rule_path, "r", encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info(
+        "rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]}
+    )
+
+    # Verify notification expectations
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        notifier_test_case.notification_expectation,
+    )

tests/integration/src/alertmanager/conftest.py

@@ -0,0 +1,41 @@
+from fixtures import types
+
+from fixtures.signoz import create_signoz
+import pytest
+from testcontainers.core.container import Network
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev: types.TestContainerDocker,
+    notification_channel: types.TestContainerDocker,
+) -> types.SigNoz:
+    """
+    Package-scoped fixture for setting up SigNoz.
+    Overrides SMTP, PagerDuty, and OpsGenie URLs to point to test containers.
+    """
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        env_overrides={
+            # SMTP config for email notifications via maildev
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__SMARTHOST": f"{maildev.container_configs['1025'].address}:{maildev.container_configs['1025'].port}",
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__REQUIRE__TLS": "false",
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__FROM": "alertmanager@signoz.io",
+            # PagerDuty API URL -> wiremock (default: https://events.pagerduty.com/v2/enqueue)
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_PAGERDUTY__URL": notification_channel.container_configs["8080"].get("/v2/enqueue"),
+            # OpsGenie API URL -> wiremock (default: https://api.opsgenie.com/)
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_OPSGENIE__API__URL": notification_channel.container_configs["8080"].get("/"),
+        },
+    )