chore: custom notifiers in alert manager

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/coreos/go-oidc/v3 v3.17.0
 	github.com/dgraph-io/ristretto/v2 v2.3.0
 	github.com/dustin/go-humanize v1.0.1
+	github.com/emersion/go-smtp v0.24.0
 	github.com/gin-gonic/gin v1.11.0
 	github.com/go-co-op/gocron v1.30.1
 	github.com/go-openapi/runtime v0.29.2
@@ -108,6 +109,7 @@ require (
 	github.com/bytedance/gopkg v0.1.3 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
+	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/go-openapi/swag/cmdutils v0.25.4 // indirect
 	github.com/go-openapi/swag/conv v0.25.4 // indirect
@@ -161,7 +163,7 @@ require (
 	github.com/ClickHouse/ch-go v0.67.0 // indirect
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Yiling-J/theine-go v0.6.2 // indirect
-	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
+	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beevik/etree v1.1.0 // indirect

pkg/alertmanager/alertmanagernotify/email/email.go

@@ -0,0 +1,423 @@
+package email
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math/rand"
+	"mime"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net"
+	"net/mail"
+	"net/smtp"
+	"net/textproto"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// Email implements a Notifier for email notifications.
+type Email struct {
+	conf     *config.EmailConfig
+	tmpl     *template.Template
+	logger   *slog.Logger
+	hostname string
+}
+
+// New returns a new Email notifier.
+func New(c *config.EmailConfig, t *template.Template, l *slog.Logger) *Email {
+	if _, ok := c.Headers["Subject"]; !ok {
+		c.Headers["Subject"] = config.DefaultEmailSubject
+	}
+	if _, ok := c.Headers["To"]; !ok {
+		c.Headers["To"] = c.To
+	}
+	if _, ok := c.Headers["From"]; !ok {
+		c.Headers["From"] = c.From
+	}
+
+	h, err := os.Hostname()
+	// If we can't get the hostname, we'll use localhost
+	if err != nil {
+		h = "localhost.localdomain"
+	}
+	return &Email{conf: c, tmpl: t, logger: l, hostname: h}
+}
+
+// auth resolves a string of authentication mechanisms.
+func (n *Email) auth(mechs string) (smtp.Auth, error) {
+	username := n.conf.AuthUsername
+
+	// If no username is set, keep going without authentication.
+	if n.conf.AuthUsername == "" {
+		n.logger.Debug("smtp_auth_username is not configured. Attempting to send email without authenticating")
+		return nil, nil
+	}
+
+	err := &types.MultiError{}
+	for mech := range strings.SplitSeq(mechs, " ") {
+		switch mech {
+		case "CRAM-MD5":
+			secret, secretErr := n.getAuthSecret()
+			if secretErr != nil {
+				err.Add(secretErr)
+				continue
+			}
+			if secret == "" {
+				err.Add(errors.New("missing secret for CRAM-MD5 auth mechanism"))
+				continue
+			}
+			return smtp.CRAMMD5Auth(username, secret), nil
+
+		case "PLAIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				err.Add(passwordErr)
+				continue
+			}
+			if password == "" {
+				err.Add(errors.New("missing password for PLAIN auth mechanism"))
+				continue
+			}
+			identity := n.conf.AuthIdentity
+
+			return smtp.PlainAuth(identity, username, password, n.conf.Smarthost.Host), nil
+		case "LOGIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				err.Add(passwordErr)
+				continue
+			}
+			if password == "" {
+				err.Add(errors.New("missing password for LOGIN auth mechanism"))
+				continue
+			}
+			return LoginAuth(username, password), nil
+		}
+	}
+	if err.Len() == 0 {
+		err.Add(errors.New("unknown auth mechanism: " + mechs))
+	}
+	return nil, err
+}
+
+// Notify implements the Notifier interface.
+func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var (
+		c       *smtp.Client
+		conn    net.Conn
+		err     error
+		success = false
+	)
+	// Determine whether to use Implicit TLS
+	var useImplicitTLS bool
+	if n.conf.ForceImplicitTLS != nil {
+		useImplicitTLS = *n.conf.ForceImplicitTLS
+	} else {
+		// Default logic: port 465 uses implicit TLS (backward compatibility)
+		useImplicitTLS = n.conf.Smarthost.Port == "465"
+	}
+
+	if useImplicitTLS {
+		tlsConfig, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, fmt.Errorf("parse TLS configuration: %w", err)
+		}
+		if tlsConfig.ServerName == "" {
+			tlsConfig.ServerName = n.conf.Smarthost.Host
+		}
+
+		conn, err = tls.Dial("tcp", n.conf.Smarthost.String(), tlsConfig)
+		if err != nil {
+			return true, fmt.Errorf("establish TLS connection to server: %w", err)
+		}
+	} else {
+		var (
+			d   = net.Dialer{}
+			err error
+		)
+		conn, err = d.DialContext(ctx, "tcp", n.conf.Smarthost.String())
+		if err != nil {
+			return true, fmt.Errorf("establish connection to server: %w", err)
+		}
+	}
+	c, err = smtp.NewClient(conn, n.conf.Smarthost.Host)
+	if err != nil {
+		conn.Close()
+		return true, fmt.Errorf("create SMTP client: %w", err)
+	}
+	defer func() {
+		// Try to clean up after ourselves but don't log anything if something has failed.
+		if err := c.Quit(); success && err != nil {
+			n.logger.Warn("failed to close SMTP connection", "err", err)
+		}
+	}()
+
+	if n.conf.Hello != "" {
+		err = c.Hello(n.conf.Hello)
+		if err != nil {
+			return true, fmt.Errorf("send EHLO command: %w", err)
+		}
+	}
+
+	// Global Config guarantees RequireTLS is not nil.
+	if *n.conf.RequireTLS && !useImplicitTLS {
+		if ok, _ := c.Extension("STARTTLS"); !ok {
+			return true, fmt.Errorf("'require_tls' is true (default) but %q does not advertise the STARTTLS extension", n.conf.Smarthost)
+		}
+
+		tlsConf, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, fmt.Errorf("parse TLS configuration: %w", err)
+		}
+		if tlsConf.ServerName == "" {
+			tlsConf.ServerName = n.conf.Smarthost.Host
+		}
+
+		if err := c.StartTLS(tlsConf); err != nil {
+			return true, fmt.Errorf("send STARTTLS command: %w", err)
+		}
+	}
+
+	if ok, mech := c.Extension("AUTH"); ok {
+		auth, err := n.auth(mech)
+		if err != nil {
+			return true, fmt.Errorf("find auth mechanism: %w", err)
+		}
+		if auth != nil {
+			if err := c.Auth(auth); err != nil {
+				return true, fmt.Errorf("%T auth: %w", auth, err)
+			}
+		}
+	}
+
+	var (
+		tmplErr error
+		data    = notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
+		tmpl    = notify.TmplText(n.tmpl, data, &tmplErr)
+	)
+	from := tmpl(n.conf.From)
+	if tmplErr != nil {
+		return false, fmt.Errorf("execute 'from' template: %w", tmplErr)
+	}
+	to := tmpl(n.conf.To)
+	if tmplErr != nil {
+		return false, fmt.Errorf("execute 'to' template: %w", tmplErr)
+	}
+
+	addrs, err := mail.ParseAddressList(from)
+	if err != nil {
+		return false, fmt.Errorf("parse 'from' addresses: %w", err)
+	}
+	if len(addrs) != 1 {
+		return false, fmt.Errorf("must be exactly one 'from' address (got: %d)", len(addrs))
+	}
+	if err = c.Mail(addrs[0].Address); err != nil {
+		return true, fmt.Errorf("send MAIL command: %w", err)
+	}
+	addrs, err = mail.ParseAddressList(to)
+	if err != nil {
+		return false, fmt.Errorf("parse 'to' addresses: %w", err)
+	}
+	for _, addr := range addrs {
+		if err = c.Rcpt(addr.Address); err != nil {
+			return true, fmt.Errorf("send RCPT command: %w", err)
+		}
+	}
+
+	// Send the email headers and body.
+	message, err := c.Data()
+	if err != nil {
+		return true, fmt.Errorf("send DATA command: %w", err)
+	}
+	closeOnce := sync.OnceValue(func() error {
+		return message.Close()
+	})
+	// Close the message when this method exits in order to not leak resources. Even though we're calling this explicitly
+	// further down, the method may exit before then.
+	defer func() {
+		// If we try close an already-closed writer, it'll send a subsequent request to the server which is invalid.
+		_ = closeOnce()
+	}()
+
+	buffer := &bytes.Buffer{}
+	for header, t := range n.conf.Headers {
+		value, err := n.tmpl.ExecuteTextString(t, data)
+		if err != nil {
+			return false, fmt.Errorf("execute %q header template: %w", header, err)
+		}
+		fmt.Fprintf(buffer, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", value))
+	}
+
+	if _, ok := n.conf.Headers["Message-Id"]; !ok {
+		fmt.Fprintf(buffer, "Message-Id: %s\r\n", fmt.Sprintf("<%d.%d@%s>", time.Now().UnixNano(), rand.Uint64(), n.hostname))
+	}
+
+	if n.conf.Threading.Enabled {
+		key, err := notify.ExtractGroupKey(ctx)
+		if err != nil {
+			return false, err
+		}
+		// Add threading headers. All notifications for the same alert group
+		// (identified by key hash) are threaded together.
+		threadBy := ""
+		if n.conf.Threading.ThreadByDate != "none" {
+			// ThreadByDate is 'daily':
+			// Use current date so all mails for this alert today thread together.
+			threadBy = time.Now().Format("2006-01-02")
+		}
+		keyHash := key.Hash()
+		if len(keyHash) > 16 {
+			keyHash = keyHash[:16]
+		}
+		// The thread root ID is a Message-ID that doesn't correspond to
+		// any actual email. Email clients following the (commonly used) JWZ
+		// algorithm will create a dummy container to group these messages.
+		threadRootID := fmt.Sprintf("<alert-%s-%s@alertmanager>", keyHash, threadBy)
+		fmt.Fprintf(buffer, "References: %s\r\n", threadRootID)
+		fmt.Fprintf(buffer, "In-Reply-To: %s\r\n", threadRootID)
+	}
+
+	multipartBuffer := &bytes.Buffer{}
+	multipartWriter := multipart.NewWriter(multipartBuffer)
+
+	fmt.Fprintf(buffer, "Date: %s\r\n", time.Now().Format(time.RFC1123Z))
+	fmt.Fprintf(buffer, "Content-Type: multipart/alternative;  boundary=%s\r\n", multipartWriter.Boundary())
+	fmt.Fprintf(buffer, "MIME-Version: 1.0\r\n\r\n")
+
+	// TODO: Add some useful headers here, such as URL of the alertmanager
+	// and active/resolved.
+	_, err = message.Write(buffer.Bytes())
+	if err != nil {
+		return false, fmt.Errorf("write headers: %w", err)
+	}
+
+	if len(n.conf.Text) > 0 {
+		// Text template
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/plain; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, fmt.Errorf("create part for text template: %w", err)
+		}
+		body, err := n.tmpl.ExecuteTextString(n.conf.Text, data)
+		if err != nil {
+			return false, fmt.Errorf("execute text template: %w", err)
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, fmt.Errorf("write text part: %w", err)
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, fmt.Errorf("close text part: %w", err)
+		}
+	}
+
+	if len(n.conf.HTML) > 0 {
+		// Html template
+		// Preferred alternative placed last per section 5.1.4 of RFC 2046
+		// https://www.ietf.org/rfc/rfc2046.txt
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/html; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, fmt.Errorf("create part for html template: %w", err)
+		}
+		body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
+		if err != nil {
+			return false, fmt.Errorf("execute html template: %w", err)
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, fmt.Errorf("write HTML part: %w", err)
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, fmt.Errorf("close HTML part: %w", err)
+		}
+	}
+
+	err = multipartWriter.Close()
+	if err != nil {
+		return false, fmt.Errorf("close multipartWriter: %w", err)
+	}
+
+	_, err = message.Write(multipartBuffer.Bytes())
+	if err != nil {
+		return false, fmt.Errorf("write body buffer: %w", err)
+	}
+
+	// Complete the message and await response.
+	if err = closeOnce(); err != nil {
+		return true, fmt.Errorf("delivery failure: %w", err)
+	}
+
+	success = true
+	return false, nil
+}
+
+type loginAuth struct {
+	username, password string
+}
+
+func LoginAuth(username, password string) smtp.Auth {
+	return &loginAuth{username, password}
+}
+
+func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
+	return "LOGIN", []byte{}, nil
+}
+
+// Used for AUTH LOGIN. (Maybe password should be encrypted).
+func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
+	if more {
+		switch strings.ToLower(string(fromServer)) {
+		case "username:":
+			return []byte(a.username), nil
+		case "password:":
+			return []byte(a.password), nil
+		default:
+			return nil, errors.New("unexpected server challenge")
+		}
+	}
+	return nil, nil
+}
+
+func (n *Email) getPassword() (string, error) {
+	if len(n.conf.AuthPasswordFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthPasswordFile)
+		if err != nil {
+			return "", fmt.Errorf("could not read %s: %w", n.conf.AuthPasswordFile, err)
+		}
+		return strings.TrimSpace(string(content)), nil
+	}
+	return string(n.conf.AuthPassword), nil
+}
+
+func (n *Email) getAuthSecret() (string, error) {
+	if len(n.conf.AuthSecretFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthSecretFile)
+		if err != nil {
+			return "", fmt.Errorf("could not read %s: %w", n.conf.AuthSecretFile, err)
+		}
+		return string(content), nil
+	}
+	return string(n.conf.AuthSecret), nil
+}

pkg/alertmanager/alertmanagernotify/email/testdata/auth-local.yml

@@ -0,0 +1,4 @@
+smarthost: 127.0.0.1:1026
+server: http://127.0.0.1:1081/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/auth.yml

@@ -0,0 +1,4 @@
+smarthost: maildev-auth:1025
+server: http://maildev-auth:1080/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/noauth-local.yml

@@ -0,0 +1,2 @@
+smarthost: 127.0.0.1:1025
+server: http://127.0.0.1:1080/

pkg/alertmanager/alertmanagernotify/email/testdata/noauth.yml

@@ -0,0 +1,2 @@
+smarthost: maildev-noauth:1025
+server: http://maildev-noauth:1080/

pkg/alertmanager/alertmanagernotify/opsgenie/api_key_file

@@ -0,0 +1,2 @@
+my_secret_api_key
+

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie.go

@@ -0,0 +1,285 @@
+package opsgenie
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"maps"
+	"net/http"
+	"os"
+	"strings"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
+const maxMessageLenRunes = 130
+
+// Notifier implements a Notifier for OpsGenie notifications.
+type Notifier struct {
+	conf    *config.OpsGenieConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new OpsGenie notifier.
+func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, "opsgenie", httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:    c,
+		tmpl:    t,
+		logger:  l,
+		client:  client,
+		retrier: &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
+	}, nil
+}
+
+type opsGenieCreateMessage struct {
+	Alias       string                           `json:"alias"`
+	Message     string                           `json:"message"`
+	Description string                           `json:"description,omitempty"`
+	Details     map[string]string                `json:"details"`
+	Source      string                           `json:"source"`
+	Responders  []opsGenieCreateMessageResponder `json:"responders,omitempty"`
+	Tags        []string                         `json:"tags,omitempty"`
+	Note        string                           `json:"note,omitempty"`
+	Priority    string                           `json:"priority,omitempty"`
+	Entity      string                           `json:"entity,omitempty"`
+	Actions     []string                         `json:"actions,omitempty"`
+}
+
+type opsGenieCreateMessageResponder struct {
+	ID       string `json:"id,omitempty"`
+	Name     string `json:"name,omitempty"`
+	Username string `json:"username,omitempty"`
+	Type     string `json:"type"` // team, user, escalation, schedule etc.
+}
+
+type opsGenieCloseMessage struct {
+	Source string `json:"source"`
+}
+
+type opsGenieUpdateMessageMessage struct {
+	Message string `json:"message,omitempty"`
+}
+
+type opsGenieUpdateDescriptionMessage struct {
+	Description string `json:"description,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	requests, retry, err := n.createRequests(ctx, as...)
+	if err != nil {
+		return retry, err
+	}
+
+	for _, req := range requests {
+		req.Header.Set("User-Agent", notify.UserAgentHeader)
+		resp, err := n.client.Do(req)
+		if err != nil {
+			return true, err
+		}
+		shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+		notify.Drain(resp)
+		if err != nil {
+			return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+		}
+	}
+	return true, nil
+}
+
+// Like Split but filter out empty strings.
+func safeSplit(s, sep string) []string {
+	a := strings.Split(strings.TrimSpace(s), sep)
+	b := a[:0]
+	for _, x := range a {
+		if x != "" {
+			b = append(b, x)
+		}
+	}
+	return b
+}
+
+// Create requests for a list of alerts.
+func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+	logger := n.logger.With("group_key", key)
+	logger.Debug("extracted group key")
+
+	data := notify.GetTemplateData(ctx, n.tmpl, as, logger)
+
+	tmpl := notify.TmplText(n.tmpl, data, &err)
+
+	details := make(map[string]string)
+
+	maps.Copy(details, data.CommonLabels)
+
+	for k, v := range n.conf.Details {
+		details[k] = tmpl(v)
+	}
+
+	requests := []*http.Request{}
+
+	var (
+		alias  = key.Hash()
+		alerts = types.Alerts(as...)
+	)
+	switch alerts.Status() {
+	case model.AlertResolved:
+		resolvedEndpointURL := n.conf.APIURL.Copy()
+		resolvedEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
+		q := resolvedEndpointURL.Query()
+		q.Set("identifierType", "alias")
+		resolvedEndpointURL.RawQuery = q.Encode()
+		msg := &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", resolvedEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+	default:
+		message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
+		if truncated {
+			logger.Warn("Truncated message", "alert", key, "max_runes", maxMessageLenRunes)
+		}
+
+		createEndpointURL := n.conf.APIURL.Copy()
+		createEndpointURL.Path += "v2/alerts"
+
+		var responders []opsGenieCreateMessageResponder
+		for _, r := range n.conf.Responders {
+			responder := opsGenieCreateMessageResponder{
+				ID:       tmpl(r.ID),
+				Name:     tmpl(r.Name),
+				Username: tmpl(r.Username),
+				Type:     tmpl(r.Type),
+			}
+
+			if responder == (opsGenieCreateMessageResponder{}) {
+				// Filter out empty responders. This is useful if you want to fill
+				// responders dynamically from alert's common labels.
+				continue
+			}
+
+			if responder.Type == "teams" {
+				teams := safeSplit(responder.Name, ",")
+				for _, team := range teams {
+					newResponder := opsGenieCreateMessageResponder{
+						Name: tmpl(team),
+						Type: tmpl("team"),
+					}
+					responders = append(responders, newResponder)
+				}
+				continue
+			}
+
+			responders = append(responders, responder)
+		}
+
+		msg := &opsGenieCreateMessage{
+			Alias:       alias,
+			Message:     message,
+			Description: tmpl(n.conf.Description),
+			Details:     details,
+			Source:      tmpl(n.conf.Source),
+			Responders:  responders,
+			Tags:        safeSplit(tmpl(n.conf.Tags), ","),
+			Note:        tmpl(n.conf.Note),
+			Priority:    tmpl(n.conf.Priority),
+			Entity:      tmpl(n.conf.Entity),
+			Actions:     safeSplit(tmpl(n.conf.Actions), ","),
+		}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", createEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+
+		if n.conf.UpdateAlerts {
+			updateMessageEndpointURL := n.conf.APIURL.Copy()
+			updateMessageEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/message", alias)
+			q := updateMessageEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateMessageEndpointURL.RawQuery = q.Encode()
+			updateMsgMsg := &opsGenieUpdateMessageMessage{
+				Message: msg.Message,
+			}
+			var updateMessageBuf bytes.Buffer
+			if err := json.NewEncoder(&updateMessageBuf).Encode(updateMsgMsg); err != nil {
+				return nil, false, err
+			}
+			req, err := http.NewRequest("PUT", updateMessageEndpointURL.String(), &updateMessageBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req)
+
+			updateDescriptionEndpointURL := n.conf.APIURL.Copy()
+			updateDescriptionEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/description", alias)
+			q = updateDescriptionEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateDescriptionEndpointURL.RawQuery = q.Encode()
+			updateDescMsg := &opsGenieUpdateDescriptionMessage{
+				Description: msg.Description,
+			}
+
+			var updateDescriptionBuf bytes.Buffer
+			if err := json.NewEncoder(&updateDescriptionBuf).Encode(updateDescMsg); err != nil {
+				return nil, false, err
+			}
+			req, err = http.NewRequest("PUT", updateDescriptionEndpointURL.String(), &updateDescriptionBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req.WithContext(ctx))
+		}
+	}
+
+	var apiKey string
+	if n.conf.APIKey != "" {
+		apiKey = tmpl(string(n.conf.APIKey))
+	} else {
+		content, err := os.ReadFile(n.conf.APIKeyFile)
+		if err != nil {
+			return nil, false, fmt.Errorf("read key_file error: %w", err)
+		}
+		apiKey = tmpl(string(content))
+		apiKey = strings.TrimSpace(string(apiKey))
+	}
+
+	if err != nil {
+		return nil, false, fmt.Errorf("templating error: %w", err)
+	}
+
+	for _, req := range requests {
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
+	}
+
+	return requests, true, nil
+}

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie_test.go

@@ -0,0 +1,333 @@
+package opsgenie
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestOpsGenieRetry(t *testing.T) {
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestOpsGenieRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKey:     config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestGettingOpsGegineApikeyFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+
+	f, err := os.CreateTemp(t.TempDir(), "opsgenie_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKeyFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestOpsGenie(t *testing.T) {
+	u, err := url.Parse("https://opsgenie/api")
+	if err != nil {
+		t.Fatalf("failed to parse URL: %v", err)
+	}
+	logger := promslog.NewNopLogger()
+	tmpl := test.CreateTmpl(t)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.OpsGenieConfig
+
+		expectedEmptyAlertBody string
+		expectedBody           string
+	}{
+		{
+			title: "config without details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with multiple teams",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName3 }}`,
+						Type: `{{ .CommonLabels.ResponderType3 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"TeamB","type":"team"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1"}
+`,
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			notifier, err := New(tc.cfg, tmpl, logger)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			expectedURL, _ := url.Parse("https://opsgenie/apiv2/alerts")
+
+			// Empty alert.
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+
+			req, retry, err := notifier.createRequests(ctx, alert1)
+			require.NoError(t, err)
+			require.Len(t, req, 1)
+			require.True(t, retry)
+			require.Equal(t, expectedURL, req[0].URL)
+			require.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization"))
+			require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0]))
+
+			// Fully defined alert.
+			alert2 := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"Message":        "message",
+						"Description":    "description",
+						"Source":         "http://prometheus",
+						"ResponderName1": "TeamA",
+						"ResponderType1": "team",
+						"ResponderName2": "EscalationA",
+						"ResponderType2": "escalation",
+						"ResponderName3": "TeamA,TeamB",
+						"ResponderType3": "teams",
+						"Tags":           "tag1,tag2",
+						"Note":           "this is a note",
+						"Priority":       "P1",
+						"Entity":         "test-domain",
+						"Actions":        "doThis,doThat",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			req, retry, err = notifier.createRequests(ctx, alert2)
+			require.NoError(t, err)
+			require.True(t, retry)
+			require.Len(t, req, 1)
+			require.Equal(t, tc.expectedBody, readBody(t, req[0]))
+
+			// Broken API Key Template.
+			tc.cfg.APIKey = "{{ kaput "
+			_, _, err = notifier.createRequests(ctx, alert2)
+			require.Error(t, err)
+			require.Equal(t, "templating error: template: :1: function \"kaput\" not defined", err.Error())
+		})
+	}
+}
+
+func TestOpsGenieWithUpdate(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		Message:      `{{ .CommonLabels.Message }}`,
+		Description:  `{{ .CommonLabels.Description }}`,
+		UpdateAlerts: true,
+		APIKey:       "test-api-key",
+		APIURL:       &config.URL{URL: u},
+		HTTPConfig:   &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+	alert := &types.Alert{
+		Alert: model.Alert{
+			StartsAt: time.Now(),
+			EndsAt:   time.Now().Add(time.Hour),
+			Labels: model.LabelSet{
+				"Message":     "new message",
+				"Description": "new description",
+			},
+		},
+	}
+	require.NoError(t, err)
+	requests, retry, err := notifierWithUpdate.createRequests(ctx, alert)
+	require.NoError(t, err)
+	require.True(t, retry)
+	require.Len(t, requests, 3)
+
+	body0 := readBody(t, requests[0])
+	body1 := readBody(t, requests[1])
+	body2 := readBody(t, requests[2])
+	key, _ := notify.ExtractGroupKey(ctx)
+	alias := key.Hash()
+
+	require.Equal(t, "https://test-opsgenie-url/v2/alerts", requests[0].URL.String())
+	require.NotEmpty(t, body0)
+
+	require.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias))
+	require.JSONEq(t, `{"message":"new message"}`, body1)
+	require.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias))
+	require.JSONEq(t, `{"description":"new description"}`, body2)
+}
+
+func TestOpsGenieApiKeyFile(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		APIKeyFile: `./api_key_file`,
+		APIURL:     &config.URL{URL: u},
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+
+	require.NoError(t, err)
+	requests, _, err := notifierWithUpdate.createRequests(ctx)
+	require.NoError(t, err)
+	require.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
+}
+
+func readBody(t *testing.T, r *http.Request) string {
+	t.Helper()
+	body, err := io.ReadAll(r.Body)
+	require.NoError(t, err)
+	return string(body)
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty.go

@@ -0,0 +1,371 @@
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/alecthomas/units"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	maxEventSize int = 512000
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTc4-send-a-v1-event - 1024 characters or runes.
+	maxV1DescriptionLenRunes = 1024
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1024 characters or runes.
+	maxV2SummaryLenRunes = 1024
+)
+
+// Notifier implements a Notifier for PagerDuty notifications.
+type Notifier struct {
+	conf    *config.PagerdutyConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	apiV1   string // for tests.
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new PagerDuty notifier.
+func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, "pagerduty", httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	n := &Notifier{conf: c, tmpl: t, logger: l, client: client}
+	if c.ServiceKey != "" || c.ServiceKeyFile != "" {
+		n.apiV1 = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+		// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v1-overview#api-response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusForbidden}, CustomDetailsFunc: errDetails}
+	} else {
+		// Retrying can solve the issue on 429 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v2-overview#response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}, CustomDetailsFunc: errDetails}
+	}
+	return n, nil
+}
+
+const (
+	pagerDutyEventTrigger = "trigger"
+	pagerDutyEventResolve = "resolve"
+)
+
+type pagerDutyMessage struct {
+	RoutingKey  string            `json:"routing_key,omitempty"`
+	ServiceKey  string            `json:"service_key,omitempty"`
+	DedupKey    string            `json:"dedup_key,omitempty"`
+	IncidentKey string            `json:"incident_key,omitempty"`
+	EventType   string            `json:"event_type,omitempty"`
+	Description string            `json:"description,omitempty"`
+	EventAction string            `json:"event_action"`
+	Payload     *pagerDutyPayload `json:"payload"`
+	Client      string            `json:"client,omitempty"`
+	ClientURL   string            `json:"client_url,omitempty"`
+	Details     map[string]any    `json:"details,omitempty"`
+	Images      []pagerDutyImage  `json:"images,omitempty"`
+	Links       []pagerDutyLink   `json:"links,omitempty"`
+}
+
+type pagerDutyLink struct {
+	HRef string `json:"href"`
+	Text string `json:"text"`
+}
+
+type pagerDutyImage struct {
+	Src  string `json:"src"`
+	Alt  string `json:"alt"`
+	Href string `json:"href"`
+}
+
+type pagerDutyPayload struct {
+	Summary       string         `json:"summary"`
+	Source        string         `json:"source"`
+	Severity      string         `json:"severity"`
+	Timestamp     string         `json:"timestamp,omitempty"`
+	Class         string         `json:"class,omitempty"`
+	Component     string         `json:"component,omitempty"`
+	Group         string         `json:"group,omitempty"`
+	CustomDetails map[string]any `json:"custom_details,omitempty"`
+}
+
+func (n *Notifier) encodeMessage(msg *pagerDutyMessage) (bytes.Buffer, error) {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return buf, fmt.Errorf("failed to encode PagerDuty message: %w", err)
+	}
+
+	if buf.Len() > maxEventSize {
+		truncatedMsg := fmt.Sprintf("Custom details have been removed because the original event exceeds the maximum size of %s", units.MetricBytes(maxEventSize).String())
+
+		if n.apiV1 != "" {
+			msg.Details = map[string]any{"error": truncatedMsg}
+		} else {
+			msg.Payload.CustomDetails = map[string]any{"error": truncatedMsg}
+		}
+
+		warningMsg := fmt.Sprintf("Truncated Details because message of size %s exceeds limit %s", units.MetricBytes(buf.Len()).String(), units.MetricBytes(maxEventSize).String())
+		n.logger.Warn(warningMsg)
+
+		buf.Reset()
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return buf, fmt.Errorf("failed to encode PagerDuty message: %w", err)
+		}
+	}
+
+	return buf, nil
+}
+
+func (n *Notifier) notifyV1(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes)
+	if truncated {
+		n.logger.Warn("Truncated description", "key", key, "max_runes", maxV1DescriptionLenRunes)
+	}
+
+	serviceKey := string(n.conf.ServiceKey)
+	if serviceKey == "" {
+		content, fileErr := os.ReadFile(n.conf.ServiceKeyFile)
+		if fileErr != nil {
+			return false, fmt.Errorf("failed to read service key from file: %w", fileErr)
+		}
+		serviceKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		ServiceKey:  tmpl(serviceKey),
+		EventType:   eventType,
+		IncidentKey: key.Hash(),
+		Description: description,
+		Details:     details,
+	}
+
+	if eventType == pagerDutyEventTrigger {
+		msg.Client = tmpl(n.conf.Client)
+		msg.ClientURL = tmpl(n.conf.ClientURL)
+	}
+
+	if tmplErr != nil {
+		return false, fmt.Errorf("failed to template PagerDuty v1 message: %w", tmplErr)
+	}
+
+	// Ensure that the service key isn't empty after templating.
+	if msg.ServiceKey == "" {
+		return false, errors.New("service key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &encodedMsg)
+	if err != nil {
+		return true, fmt.Errorf("failed to post message to PagerDuty v1: %w", err)
+	}
+	defer notify.Drain(resp)
+
+	return n.retrier.Check(resp.StatusCode, resp.Body)
+}
+
+func (n *Notifier) notifyV2(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.Severity == "" {
+		n.conf.Severity = "error"
+	}
+
+	summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV2SummaryLenRunes)
+	if truncated {
+		n.logger.Warn("Truncated summary", "key", key, "max_runes", maxV2SummaryLenRunes)
+	}
+
+	routingKey := string(n.conf.RoutingKey)
+	if routingKey == "" {
+		content, fileErr := os.ReadFile(n.conf.RoutingKeyFile)
+		if fileErr != nil {
+			return false, fmt.Errorf("failed to read routing key from file: %w", fileErr)
+		}
+		routingKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		Client:      tmpl(n.conf.Client),
+		ClientURL:   tmpl(n.conf.ClientURL),
+		RoutingKey:  tmpl(routingKey),
+		EventAction: eventType,
+		DedupKey:    key.Hash(),
+		Images:      make([]pagerDutyImage, 0, len(n.conf.Images)),
+		Links:       make([]pagerDutyLink, 0, len(n.conf.Links)),
+		Payload: &pagerDutyPayload{
+			Summary:       summary,
+			Source:        tmpl(n.conf.Source),
+			Severity:      tmpl(n.conf.Severity),
+			CustomDetails: details,
+			Class:         tmpl(n.conf.Class),
+			Component:     tmpl(n.conf.Component),
+			Group:         tmpl(n.conf.Group),
+		},
+	}
+
+	for _, item := range n.conf.Images {
+		image := pagerDutyImage{
+			Src:  tmpl(item.Src),
+			Alt:  tmpl(item.Alt),
+			Href: tmpl(item.Href),
+		}
+
+		if image.Src != "" {
+			msg.Images = append(msg.Images, image)
+		}
+	}
+
+	for _, item := range n.conf.Links {
+		link := pagerDutyLink{
+			HRef: tmpl(item.Href),
+			Text: tmpl(item.Text),
+		}
+
+		if link.HRef != "" {
+			msg.Links = append(msg.Links, link)
+		}
+	}
+
+	if tmplErr != nil {
+		return false, fmt.Errorf("failed to template PagerDuty v2 message: %w", tmplErr)
+	}
+
+	// Ensure that the routing key isn't empty after templating.
+	if msg.RoutingKey == "" {
+		return false, errors.New("routing key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &encodedMsg)
+	if err != nil {
+		return true, fmt.Errorf("failed to post message to PagerDuty: %w", err)
+	}
+	defer notify.Drain(resp)
+
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return retry, err
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With("group_key", key)
+
+	var (
+		alerts    = types.Alerts(as...)
+		data      = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		eventType = pagerDutyEventTrigger
+	)
+
+	if alerts.Status() == model.AlertResolved {
+		eventType = pagerDutyEventResolve
+	}
+
+	logger.Debug("extracted group key", "eventType", eventType)
+
+	details, err := n.renderDetails(data)
+	if err != nil {
+		return false, fmt.Errorf("failed to render details: %w", err)
+	}
+
+	if n.conf.Timeout > 0 {
+		nfCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, fmt.Errorf("configured pagerduty timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = nfCtx
+	}
+
+	nf := n.notifyV2
+	if n.apiV1 != "" {
+		nf = n.notifyV1
+	}
+	retry, err := nf(ctx, eventType, key, data, details)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
+		}
+		return retry, err
+	}
+	return retry, nil
+}
+
+func errDetails(status int, body io.Reader) string {
+	// See https://v2.developer.pagerduty.com/docs/trigger-events for the v1 events API.
+	// See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 for the v2 events API.
+	if status != http.StatusBadRequest || body == nil {
+		return ""
+	}
+	var pgr struct {
+		Status  string   `json:"status"`
+		Message string   `json:"message"`
+		Errors  []string `json:"errors"`
+	}
+	if err := json.NewDecoder(body).Decode(&pgr); err != nil {
+		return ""
+	}
+	return fmt.Sprintf("%s: %s", pgr.Message, strings.Join(pgr.Errors, ","))
+}
+
+func (n *Notifier) renderDetails(
+	data *template.Data,
+) (map[string]any, error) {
+	var (
+		tmplTextErr  error
+		tmplText     = notify.TmplText(n.tmpl, data, &tmplTextErr)
+		tmplTextFunc = func(tmpl string) (string, error) {
+			return tmplText(tmpl), tmplTextErr
+		}
+	)
+	var err error
+	rendered := make(map[string]any, len(n.conf.Details))
+	for k, v := range n.conf.Details {
+		rendered[k], err = template.DeepCopyWithTemplate(v, tmplTextFunc)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return rendered, nil
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty_test.go

@@ -0,0 +1,873 @@
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestPagerDutyRetryV1(t *testing.T) {
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusForbidden)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "retryv1 - error on status %d", statusCode)
+	}
+}
+
+func TestPagerDutyRetryV2(t *testing.T) {
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "retryv2 - error on status %d", statusCode)
+	}
+}
+
+func TestPagerDutyRedactedURLV1(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "01234567890123456789012345678901"
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+	notifier.apiV1 = u.String()
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyRedactedURLV2(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "01234567890123456789012345678901"
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			URL:        &config.URL{URL: u},
+			RoutingKey: config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyV1ServiceKeyFromFile(t *testing.T) {
+	key := "01234567890123456789012345678901"
+	f, err := os.CreateTemp(t.TempDir(), "pagerduty_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKeyFile: f.Name(),
+			HTTPConfig:     &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+	notifier.apiV1 = u.String()
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyV2RoutingKeyFromFile(t *testing.T) {
+	key := "01234567890123456789012345678901"
+	f, err := os.CreateTemp(t.TempDir(), "pagerduty_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			URL:            &config.URL{URL: u},
+			RoutingKeyFile: f.Name(),
+			HTTPConfig:     &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyTemplating(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		dec := json.NewDecoder(r.Body)
+		out := make(map[string]any)
+		err := dec.Decode(&out)
+		if err != nil {
+			panic(err)
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.PagerdutyConfig
+
+		retry  bool
+		errMsg string
+	}{
+		{
+			title: "full-blown legacy message",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Images: []config.PagerdutyImage{
+					{
+						Src:  "{{ .Status }}",
+						Alt:  "{{ .Status }}",
+						Href: "{{ .Status }}",
+					},
+				},
+				Links: []config.PagerdutyLink{
+					{
+						Href: "{{ .Status }}",
+						Text: "{{ .Status }}",
+					},
+				},
+				Details: map[string]any{
+					"firing":       `{{ .Alerts.Firing | toJson }}`,
+					"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+		},
+		{
+			title: "full-blown legacy message",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Images: []config.PagerdutyImage{
+					{
+						Src:  "{{ .Status }}",
+						Alt:  "{{ .Status }}",
+						Href: "{{ .Status }}",
+					},
+				},
+				Links: []config.PagerdutyLink{
+					{
+						Href: "{{ .Status }}",
+						Text: "{{ .Status }}",
+					},
+				},
+				Details: map[string]any{
+					"firing":       `{{ template "pagerduty.default.instances" .Alerts.Firing }}`,
+					"resolved":     `{{ template "pagerduty.default.instances" .Alerts.Resolved }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+		},
+		{
+			title: "nested details",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": map[string]any{
+								"firing":       `{{ .Alerts.Firing | toJson }}`,
+								"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+								"num_firing":   `{{ .Alerts.Firing | len }}`,
+								"num_resolved": `{{ .Alerts.Resolved | len }}`,
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			title: "nested details with template error",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": map[string]any{
+								"firing": `{{ template "pagerduty.default.instances" .Alerts.Firing`,
+							},
+						},
+					},
+				},
+			},
+			errMsg: "failed to render details: template: :1: unclosed action",
+		},
+		{
+			title: "details with templating errors",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"firing":       `{{ .Alerts.Firing | toJson`,
+					"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+			errMsg: "failed to render details: template: :1: unclosed action",
+		},
+		{
+			title: "v2 message with templating errors",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Severity:   "{{ ",
+			},
+			errMsg: "failed to template",
+		},
+		{
+			title: "v1 message with templating errors",
+			cfg: &config.PagerdutyConfig{
+				ServiceKey: config.Secret("01234567890123456789012345678901"),
+				Client:     "{{ ",
+			},
+			errMsg: "failed to template",
+		},
+		{
+			title: "routing key cannot be empty",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret(`{{ "" }}`),
+			},
+			errMsg: "routing key cannot be empty",
+		},
+		{
+			title: "service_key cannot be empty",
+			cfg: &config.PagerdutyConfig{
+				ServiceKey: config.Secret(`{{ "" }}`),
+			},
+			errMsg: "service key cannot be empty",
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			tc.cfg.URL = &config.URL{URL: u}
+			tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
+			pd, err := New(tc.cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+			if pd.apiV1 != "" {
+				pd.apiV1 = u.String()
+			}
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			ok, err := pd.Notify(ctx, []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels: model.LabelSet{
+							"lbl1": "val1",
+						},
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}...)
+			if tc.errMsg == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errMsg)
+			}
+			require.Equal(t, tc.retry, ok)
+		})
+	}
+}
+
+func TestErrDetails(t *testing.T) {
+	for _, tc := range []struct {
+		status int
+		body   io.Reader
+
+		exp string
+	}{
+		{
+			status: http.StatusBadRequest,
+			body: bytes.NewBuffer([]byte(
+				`{"status":"invalid event","message":"Event object is invalid","errors":["Length of 'routing_key' is incorrect (should be 32 characters)"]}`,
+			)),
+
+			exp: "Length of 'routing_key' is incorrect",
+		},
+		{
+			status: http.StatusBadRequest,
+			body:   bytes.NewBuffer([]byte(`{"status"}`)),
+
+			exp: "",
+		},
+		{
+			status: http.StatusBadRequest,
+
+			exp: "",
+		},
+		{
+			status: http.StatusTooManyRequests,
+
+			exp: "",
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			err := errDetails(tc.status, tc.body)
+			require.Contains(t, err, tc.exp)
+		})
+	}
+}
+
+func TestEventSizeEnforcement(t *testing.T) {
+	bigDetailsV1 := map[string]any{
+		"firing": strings.Repeat("a", 513000),
+	}
+	bigDetailsV2 := map[string]any{
+		"firing": strings.Repeat("a", 513000),
+	}
+
+	// V1 Messages
+	msgV1 := &pagerDutyMessage{
+		ServiceKey: "01234567890123456789012345678901",
+		EventType:  "trigger",
+		Details:    bigDetailsV1,
+	}
+
+	notifierV1, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV1, err := notifierV1.encodeMessage(msgV1)
+	require.NoError(t, err)
+	require.Contains(t, encodedV1.String(), `"details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+
+	// V2 Messages
+	msgV2 := &pagerDutyMessage{
+		RoutingKey:  "01234567890123456789012345678901",
+		EventAction: "trigger",
+		Payload: &pagerDutyPayload{
+			CustomDetails: bigDetailsV2,
+		},
+	}
+
+	notifierV2, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV2, err := notifierV2.encodeMessage(msgV2)
+	require.NoError(t, err)
+	require.Contains(t, encodedV2.String(), `"custom_details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+}
+
+func TestPagerDutyEmptySrcHref(t *testing.T) {
+	type pagerDutyEvent struct {
+		RoutingKey  string           `json:"routing_key"`
+		EventAction string           `json:"event_action"`
+		DedupKey    string           `json:"dedup_key"`
+		Payload     pagerDutyPayload `json:"payload"`
+		Images      []pagerDutyImage
+		Links       []pagerDutyLink
+	}
+
+	images := []config.PagerdutyImage{
+		{
+			Src:  "",
+			Alt:  "Empty src",
+			Href: "https://example.com/",
+		},
+		{
+			Src:  "https://example.com/cat.jpg",
+			Alt:  "Empty href",
+			Href: "",
+		},
+		{
+			Src:  "https://example.com/cat.jpg",
+			Alt:  "",
+			Href: "https://example.com/",
+		},
+	}
+
+	links := []config.PagerdutyLink{
+		{
+			Href: "",
+			Text: "Empty href",
+		},
+		{
+			Href: "https://example.com/",
+			Text: "",
+		},
+	}
+
+	expectedImages := make([]pagerDutyImage, 0, len(images))
+	for _, image := range images {
+		if image.Src == "" {
+			continue
+		}
+		expectedImages = append(expectedImages, pagerDutyImage{
+			Src:  image.Src,
+			Alt:  image.Alt,
+			Href: image.Href,
+		})
+	}
+
+	expectedLinks := make([]pagerDutyLink, 0, len(links))
+	for _, link := range links {
+		if link.Href == "" {
+			continue
+		}
+		expectedLinks = append(expectedLinks, pagerDutyLink{
+			HRef: link.Href,
+			Text: link.Text,
+		})
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			decoder := json.NewDecoder(r.Body)
+			var event pagerDutyEvent
+			if err := decoder.Decode(&event); err != nil {
+				panic(err)
+			}
+
+			if event.RoutingKey == "" || event.EventAction == "" {
+				http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+				return
+			}
+
+			for _, image := range event.Images {
+				if image.Src == "" {
+					http.Error(w, "Event object is invalid: 'image src' is missing or blank", http.StatusBadRequest)
+					return
+				}
+			}
+
+			for _, link := range event.Links {
+				if link.HRef == "" {
+					http.Error(w, "Event object is invalid: 'link href' is missing or blank", http.StatusBadRequest)
+					return
+				}
+			}
+
+			require.Equal(t, expectedImages, event.Images)
+			require.Equal(t, expectedLinks, event.Links)
+		},
+	))
+	defer server.Close()
+
+	url, err := url.Parse(server.URL)
+	require.NoError(t, err)
+
+	pagerDutyConfig := config.PagerdutyConfig{
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+		RoutingKey: config.Secret("01234567890123456789012345678901"),
+		URL:        &config.URL{URL: url},
+		Images:     images,
+		Links:      links,
+	}
+
+	pagerDuty, err := New(&pagerDutyConfig, test.CreateTmpl(t), promslog.NewNopLogger())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+
+	_, err = pagerDuty.Notify(ctx, []*types.Alert{
+		{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"lbl1": "val1",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		},
+	}...)
+	require.NoError(t, err)
+}
+
+func TestPagerDutyTimeout(t *testing.T) {
+	type pagerDutyEvent struct {
+		RoutingKey  string           `json:"routing_key"`
+		EventAction string           `json:"event_action"`
+		DedupKey    string           `json:"dedup_key"`
+		Payload     pagerDutyPayload `json:"payload"`
+		Images      []pagerDutyImage
+		Links       []pagerDutyLink
+	}
+
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(
+				func(w http.ResponseWriter, r *http.Request) {
+					decoder := json.NewDecoder(r.Body)
+					var event pagerDutyEvent
+					if err := decoder.Decode(&event); err != nil {
+						panic(err)
+					}
+
+					if event.RoutingKey == "" || event.EventAction == "" {
+						http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+						return
+					}
+					time.Sleep(tt.latency)
+				},
+			))
+			defer srv.Close()
+			u, err := url.Parse(srv.URL)
+			require.NoError(t, err)
+
+			cfg := config.PagerdutyConfig{
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				URL:        &config.URL{URL: u},
+				Timeout:    tt.timeout,
+			}
+
+			pd, err := New(&cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+			alert := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"lbl1": "val1",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = pd.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+func TestRenderDetails(t *testing.T) {
+	type args struct {
+		details map[string]any
+		data    *template.Data
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    map[string]any
+		wantErr bool
+	}{
+		{
+			name: "flat",
+			args: args{
+				details: map[string]any{
+					"a": "{{ .Status }}",
+					"b": "String",
+				},
+				data: &template.Data{
+					Status: "Flat",
+				},
+			},
+			want: map[string]any{
+				"a": "Flat",
+				"b": "String",
+			},
+			wantErr: false,
+		},
+		{
+			name: "flat error",
+			args: args{
+				details: map[string]any{
+					"a": "{{ .Status",
+				},
+				data: &template.Data{
+					Status: "Error",
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "nested",
+			args: args{
+				details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": "{{ .Status }}",
+							"d": "String",
+						},
+					},
+				},
+				data: &template.Data{
+					Status: "Nested",
+				},
+			},
+			want: map[string]any{
+				"a": map[string]any{
+					"b": map[string]any{
+						"c": "Nested",
+						"d": "String",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "nested error",
+			args: args{
+				details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": "{{ .Status",
+						},
+					},
+				},
+				data: &template.Data{
+					Status: "Error",
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "alerts",
+			args: args{
+				details: map[string]any{
+					"alerts": map[string]any{
+						"firing":       "{{ .Alerts.Firing | toJson }}",
+						"resolved":     "{{ .Alerts.Resolved | toJson }}",
+						"num_firing":   "{{ len .Alerts.Firing }}",
+						"num_resolved": "{{ len .Alerts.Resolved }}",
+					},
+				},
+				data: &template.Data{
+					Alerts: template.Alerts{
+						{
+							Status: "firing",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Firing1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint1",
+							GeneratorURL: "http://generator1",
+							StartsAt:     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "firing",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Firing2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint2",
+							GeneratorURL: "http://generator2",
+							StartsAt:     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "resolved",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Resolved1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint3",
+							GeneratorURL: "http://generator3",
+							StartsAt:     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "resolved",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Resolved2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint4",
+							GeneratorURL: "http://generator4",
+							StartsAt:     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+					},
+				},
+			},
+			want: map[string]any{
+				"alerts": map[string]any{
+					"firing": []any{
+						map[string]any{
+							"status": "firing",
+							"labels": map[string]any{
+								"alertname": "Firing1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint1",
+							"generatorURL": "http://generator1",
+						},
+						map[string]any{
+							"status": "firing",
+							"labels": map[string]any{
+								"alertname": "Firing2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint2",
+							"generatorURL": "http://generator2",
+						},
+					},
+					"resolved": []any{
+						map[string]any{
+							"status": "resolved",
+							"labels": map[string]any{
+								"alertname": "Resolved1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint3",
+							"generatorURL": "http://generator3",
+						},
+						map[string]any{
+							"status": "resolved",
+							"labels": map[string]any{
+								"alertname": "Resolved2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint4",
+							"generatorURL": "http://generator4",
+						},
+					},
+					"num_firing":   2,
+					"num_resolved": 2,
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			n := &Notifier{
+				conf: &config.PagerdutyConfig{
+					Details: tt.args.details,
+				},
+				tmpl: test.CreateTmpl(t),
+			}
+			got, err := n.renderDetails(tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("renderDetails() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			require.Equal(t, tt.want, got)
+		})
+	}
+}

pkg/alertmanager/alertmanagernotify/receiver.go

@@ -2,8 +2,14 @@ package alertmanagernotify
 
 import (
 	"log/slog"
+	"slices"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/email"
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/msteamsv2"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/opsgenie"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/pagerduty"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/slack"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/webhook"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/prometheus/alertmanager/config/receiver"
 	"github.com/prometheus/alertmanager/notify"
@@ -11,6 +17,24 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+const (
+	WebhookIntegration   = "webhook"
+	EmailIntegration     = "email"
+	PagerdutyIntegration = "pagerduty"
+	OpsGenieIntegration  = "opsgenie"
+	SlackIntegration     = "slack"
+	MsTeamsV2Integration = "msteamsv2"
+)
+
+var customNotifierIntegrations = []string{
+	WebhookIntegration,
+	EmailIntegration,
+	PagerdutyIntegration,
+	OpsGenieIntegration,
+	SlackIntegration,
+	MsTeamsV2Integration,
+}
+
 func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error) {
 	upstreamIntegrations, err := receiver.BuildReceiverIntegrations(nc, tmpl, logger)
 	if err != nil {
@@ -31,14 +55,29 @@ func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Templ
 	)
 
 	for _, integration := range upstreamIntegrations {
-		// skip upstream msteamsv2 integration
-		if integration.Name() != "msteamsv2" {
+		// skip upstream integration if we support custom integration for it
+		if !slices.Contains(customNotifierIntegrations, integration.Name()) {
 			integrations = append(integrations, integration)
 		}
 	}
 
+	for i, c := range nc.WebhookConfigs {
+		add(WebhookIntegration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) })
+	}
+	for i, c := range nc.EmailConfigs {
+		add(EmailIntegration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil })
+	}
+	for i, c := range nc.PagerdutyConfigs {
+		add(PagerdutyIntegration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) })
+	}
+	for i, c := range nc.OpsGenieConfigs {
+		add(OpsGenieIntegration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) })
+	}
+	for i, c := range nc.SlackConfigs {
+		add(SlackIntegration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) })
+	}
 	for i, c := range nc.MSTeamsV2Configs {
-		add("msteamsv2", i, c, func(l *slog.Logger) (notify.Notifier, error) {
+		add(MsTeamsV2Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
 			return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l)
 		})
 	}

pkg/alertmanager/alertmanagernotify/slack/slack.go

@@ -0,0 +1,274 @@
+package slack
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
+const maxTitleLenRunes = 1024
+
+// Notifier implements a Notifier for Slack notifications.
+type Notifier struct {
+	conf    *config.SlackConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+
+	postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
+}
+
+// New returns a new Slack notification handler.
+func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, "slack", httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Notifier{
+		conf:         c,
+		tmpl:         t,
+		logger:       l,
+		client:       client,
+		retrier:      &notify.Retrier{},
+		postJSONFunc: notify.PostJSON,
+	}, nil
+}
+
+// request is the request for sending a slack notification.
+type request struct {
+	Channel     string       `json:"channel,omitempty"`
+	Username    string       `json:"username,omitempty"`
+	IconEmoji   string       `json:"icon_emoji,omitempty"`
+	IconURL     string       `json:"icon_url,omitempty"`
+	LinkNames   bool         `json:"link_names,omitempty"`
+	Text        string       `json:"text,omitempty"`
+	Attachments []attachment `json:"attachments"`
+}
+
+// attachment is used to display a richly-formatted message block.
+type attachment struct {
+	Title      string               `json:"title,omitempty"`
+	TitleLink  string               `json:"title_link,omitempty"`
+	Pretext    string               `json:"pretext,omitempty"`
+	Text       string               `json:"text"`
+	Fallback   string               `json:"fallback"`
+	CallbackID string               `json:"callback_id"`
+	Fields     []config.SlackField  `json:"fields,omitempty"`
+	Actions    []config.SlackAction `json:"actions,omitempty"`
+	ImageURL   string               `json:"image_url,omitempty"`
+	ThumbURL   string               `json:"thumb_url,omitempty"`
+	Footer     string               `json:"footer"`
+	Color      string               `json:"color,omitempty"`
+	MrkdwnIn   []string             `json:"mrkdwn_in,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var err error
+
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With("group_key", key)
+	logger.Debug("extracted group key")
+
+	var (
+		data     = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		tmplText = notify.TmplText(n.tmpl, data, &err)
+	)
+	var markdownIn []string
+
+	if len(n.conf.MrkdwnIn) == 0 {
+		markdownIn = []string{"fallback", "pretext", "text"}
+	} else {
+		markdownIn = n.conf.MrkdwnIn
+	}
+
+	title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), maxTitleLenRunes)
+	if truncated {
+		logger.Warn("Truncated title", "max_runes", maxTitleLenRunes)
+	}
+	att := &attachment{
+		Title:      title,
+		TitleLink:  tmplText(n.conf.TitleLink),
+		Pretext:    tmplText(n.conf.Pretext),
+		Text:       tmplText(n.conf.Text),
+		Fallback:   tmplText(n.conf.Fallback),
+		CallbackID: tmplText(n.conf.CallbackID),
+		ImageURL:   tmplText(n.conf.ImageURL),
+		ThumbURL:   tmplText(n.conf.ThumbURL),
+		Footer:     tmplText(n.conf.Footer),
+		Color:      tmplText(n.conf.Color),
+		MrkdwnIn:   markdownIn,
+	}
+
+	numFields := len(n.conf.Fields)
+	if numFields > 0 {
+		fields := make([]config.SlackField, numFields)
+		for index, field := range n.conf.Fields {
+			// Check if short was defined for the field otherwise fallback to the global setting
+			var short bool
+			if field.Short != nil {
+				short = *field.Short
+			} else {
+				short = n.conf.ShortFields
+			}
+
+			// Rebuild the field by executing any templates and setting the new value for short
+			fields[index] = config.SlackField{
+				Title: tmplText(field.Title),
+				Value: tmplText(field.Value),
+				Short: &short,
+			}
+		}
+		att.Fields = fields
+	}
+
+	numActions := len(n.conf.Actions)
+	if numActions > 0 {
+		actions := make([]config.SlackAction, numActions)
+		for index, action := range n.conf.Actions {
+			slackAction := config.SlackAction{
+				Type:  tmplText(action.Type),
+				Text:  tmplText(action.Text),
+				URL:   tmplText(action.URL),
+				Style: tmplText(action.Style),
+				Name:  tmplText(action.Name),
+				Value: tmplText(action.Value),
+			}
+
+			if action.ConfirmField != nil {
+				slackAction.ConfirmField = &config.SlackConfirmationField{
+					Title:       tmplText(action.ConfirmField.Title),
+					Text:        tmplText(action.ConfirmField.Text),
+					OkText:      tmplText(action.ConfirmField.OkText),
+					DismissText: tmplText(action.ConfirmField.DismissText),
+				}
+			}
+
+			actions[index] = slackAction
+		}
+		att.Actions = actions
+	}
+
+	req := &request{
+		Channel:     tmplText(n.conf.Channel),
+		Username:    tmplText(n.conf.Username),
+		IconEmoji:   tmplText(n.conf.IconEmoji),
+		IconURL:     tmplText(n.conf.IconURL),
+		LinkNames:   n.conf.LinkNames,
+		Text:        tmplText(n.conf.MessageText),
+		Attachments: []attachment{*att},
+	}
+	if err != nil {
+		return false, err
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(req); err != nil {
+		return false, err
+	}
+
+	var u string
+	if n.conf.APIURL != nil {
+		u = n.conf.APIURL.String()
+	} else {
+		content, err := os.ReadFile(n.conf.APIURLFile)
+		if err != nil {
+			return false, err
+		}
+		u = strings.TrimSpace(string(content))
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, fmt.Errorf("configured slack timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := n.postJSONFunc(ctx, n.client, u, &buf)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	// Use a retrier to generate an error message for non-200 responses and
+	// classify them as retriable or not.
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		err = fmt.Errorf("channel %q: %w", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+
+	// Slack web API might return errors with a 200 response code.
+	// https://slack.dev/node-slack-sdk/web-api#handle-errors
+	retry, err = checkResponseError(resp)
+	if err != nil {
+		err = fmt.Errorf("channel %q: %w", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.ClientErrorReason, err)
+	}
+
+	return retry, nil
+}
+
+// checkResponseError parses out the error message from Slack API response.
+func checkResponseError(resp *http.Response) (bool, error) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return true, fmt.Errorf("could not read response body: %w", err)
+	}
+
+	if strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") {
+		return checkJSONResponseError(body)
+	}
+	return checkTextResponseError(body)
+}
+
+// checkTextResponseError classifies plaintext responses from Slack.
+// A plaintext (non-JSON) response is successful if it's a string "ok".
+// This is typically a response for an Incoming Webhook
+// (https://api.slack.com/messaging/webhooks#handling_errors)
+func checkTextResponseError(body []byte) (bool, error) {
+	if !bytes.Equal(body, []byte("ok")) {
+		return false, fmt.Errorf("received an error response from Slack: %s", string(body))
+	}
+	return false, nil
+}
+
+// checkJSONResponseError classifies JSON responses from Slack.
+func checkJSONResponseError(body []byte) (bool, error) {
+	// response is for parsing out errors from the JSON response.
+	type response struct {
+		OK    bool   `json:"ok"`
+		Error string `json:"error"`
+	}
+
+	var data response
+	if err := json.Unmarshal(body, &data); err != nil {
+		return true, fmt.Errorf("could not unmarshal JSON response %q: %w", string(body), err)
+	}
+	if !data.OK {
+		return false, fmt.Errorf("error response from Slack: %s", data.Error)
+	}
+	return false, nil
+}

pkg/alertmanager/alertmanagernotify/slack/slack_test.go

@@ -0,0 +1,339 @@
+package slack
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestSlackRetry(t *testing.T) {
+	notifier, err := New(
+		&config.SlackConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestSlackRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURL:     &config.SecretURL{URL: u},
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestGettingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String())
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestTrimmingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test_newline")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestNotifier_Notify_WithReason(t *testing.T) {
+	tests := []struct {
+		name           string
+		statusCode     int
+		responseBody   string
+		expectedReason notify.Reason
+		expectedErr    string
+		expectedRetry  bool
+		noError        bool
+	}{
+		{
+			name:           "with a 4xx status code",
+			statusCode:     http.StatusUnauthorized,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 401",
+		},
+		{
+			name:           "with a 5xx status code",
+			statusCode:     http.StatusInternalServerError,
+			expectedReason: notify.ServerErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "unexpected status code 500",
+		},
+		{
+			name:           "with a 3xx status code",
+			statusCode:     http.StatusTemporaryRedirect,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 307",
+		},
+		{
+			name:           "with a 1xx status code",
+			statusCode:     http.StatusSwitchingProtocols,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 101",
+		},
+		{
+			name:           "2xx response with invalid JSON",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"not valid json"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "could not unmarshal",
+		},
+		{
+			name:           "2xx response with a JSON error",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"ok":false,"error":"error_message"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: error_message",
+		},
+		{
+			name:           "2xx response with a plaintext error",
+			statusCode:     http.StatusOK,
+			responseBody:   "no_channel",
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: no_channel",
+		},
+		{
+			name:         "successful JSON response",
+			statusCode:   http.StatusOK,
+			responseBody: `{"ok":true}`,
+			noError:      true,
+		},
+		{
+			name:         "successful plaintext response",
+			statusCode:   http.StatusOK,
+			responseBody: "ok",
+			noError:      true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			apiurl, _ := url.Parse("https://slack.com/post.Message")
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: apiurl},
+					Channel:        "channelname",
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				resp := httptest.NewRecorder()
+				if strings.HasPrefix(tt.responseBody, "{") {
+					resp.Header().Add("Content-Type", "application/json; charset=utf-8")
+				}
+				resp.WriteHeader(tt.statusCode)
+				resp.WriteString(tt.responseBody)
+				return resp.Result(), nil
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			retry, err := notifier.Notify(ctx, alert1)
+			require.Equal(t, tt.expectedRetry, retry)
+			if tt.noError {
+				require.NoError(t, err)
+			} else {
+				var reasonError *notify.ErrorWithReason
+				require.ErrorAs(t, err, &reasonError)
+				require.Equal(t, tt.expectedReason, reasonError.Reason)
+				require.Contains(t, err.Error(), tt.expectedErr)
+				require.Contains(t, err.Error(), "channelname")
+			}
+		})
+	}
+}
+
+func TestSlackTimeout(t *testing.T) {
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			u, _ := url.Parse("https://slack.com/post.Message")
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: u},
+					Channel:        "channelname",
+					Timeout:        tt.timeout,
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				select {
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				case <-time.After(tt.latency):
+					resp := httptest.NewRecorder()
+					resp.Header().Set("Content-Type", "application/json; charset=utf-8")
+					resp.WriteHeader(http.StatusOK)
+					resp.WriteString(`{"ok":true}`)
+
+					return resp.Result(), nil
+				}
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = notifier.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+func TestSlackMessageField(t *testing.T) {
+	// 1. Setup a fake Slack server
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatal(err)
+		}
+
+		// 2. VERIFY: Top-level text exists
+		if body["text"] != "My Top Level Message" {
+			t.Errorf("Expected top-level 'text' to be 'My Top Level Message', got %v", body["text"])
+		}
+
+		// 3. VERIFY: Old attachments still exist
+		attachments, ok := body["attachments"].([]any)
+		if !ok || len(attachments) == 0 {
+			t.Errorf("Expected attachments to exist")
+		} else {
+			first := attachments[0].(map[string]any)
+			if first["title"] != "Old Attachment Title" {
+				t.Errorf("Expected attachment title 'Old Attachment Title', got %v", first["title"])
+			}
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(`{"ok": true}`))
+	}))
+	defer server.Close()
+
+	// 4. Configure Notifier with BOTH new and old fields
+	u, _ := url.Parse(server.URL)
+	conf := &config.SlackConfig{
+		APIURL:      &config.SecretURL{URL: u},
+		MessageText: "My Top Level Message", // Your NEW field
+		Title:       "Old Attachment Title", // An OLD field
+		Channel:     "#test-channel",
+		HTTPConfig:  &commoncfg.HTTPClientConfig{},
+	}
+
+	tmpl, err := template.FromGlobs([]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpl.ExternalURL = u
+
+	logger := slog.New(slog.DiscardHandler)
+	notifier, err := New(conf, tmpl, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group-key")
+
+	if _, err := notifier.Notify(ctx); err != nil {
+		t.Fatal("Notify failed:", err)
+	}
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook.go

@@ -0,0 +1,133 @@
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// Notifier implements a Notifier for generic webhooks.
+type Notifier struct {
+	conf    *config.WebhookConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new Webhook.
+func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*conf.HTTPConfig, "webhook", httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:   conf,
+		tmpl:   t,
+		logger: l,
+		client: client,
+		// Webhooks are assumed to respond with 2xx response codes on a successful
+		// request and 5xx response codes are assumed to be recoverable.
+		retrier: &notify.Retrier{},
+	}, nil
+}
+
+// Message defines the JSON object send to webhook endpoints.
+type Message struct {
+	*template.Data
+
+	// The protocol version.
+	Version         string `json:"version"`
+	GroupKey        string `json:"groupKey"`
+	TruncatedAlerts uint64 `json:"truncatedAlerts"`
+}
+
+func truncateAlerts(maxAlerts uint64, alerts []*types.Alert) ([]*types.Alert, uint64) {
+	if maxAlerts != 0 && uint64(len(alerts)) > maxAlerts {
+		return alerts[:maxAlerts], uint64(len(alerts)) - maxAlerts
+	}
+
+	return alerts, 0
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
+	alerts, numTruncated := truncateAlerts(n.conf.MaxAlerts, alerts)
+	data := notify.GetTemplateData(ctx, n.tmpl, alerts, n.logger)
+
+	groupKey, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	logger := n.logger.With("group_key", groupKey)
+	logger.Debug("extracted group key")
+
+	msg := &Message{
+		Version:         "4",
+		Data:            data,
+		GroupKey:        groupKey.String(),
+		TruncatedAlerts: numTruncated,
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return false, err
+	}
+
+	var url string
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.URL != "" {
+		url = tmpl(string(n.conf.URL))
+	} else {
+		content, err := os.ReadFile(n.conf.URLFile)
+		if err != nil {
+			return false, fmt.Errorf("read url_file: %w", err)
+		}
+		url = tmpl(strings.TrimSpace(string(content)))
+	}
+
+	if tmplErr != nil {
+		return false, fmt.Errorf("failed to template webhook URL: %w", tmplErr)
+	}
+
+	if url == "" {
+		return false, errors.New("webhook URL is empty after templating")
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, fmt.Errorf("configured webhook timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, url, &buf)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return shouldRetry, err
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook_test.go

@@ -0,0 +1,214 @@
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestWebhookRetry(t *testing.T) {
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL("http://example.com"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	if err != nil {
+		require.NoError(t, err)
+	}
+
+	t.Run("test retry status code", func(t *testing.T) {
+		for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+			actual, _ := notifier.retrier.Check(statusCode, nil)
+			require.Equal(t, expected, actual, "error on status %d", statusCode)
+		}
+	})
+
+	t.Run("test retry error details", func(t *testing.T) {
+		for _, tc := range []struct {
+			status int
+			body   io.Reader
+
+			exp string
+		}{
+			{
+				status: http.StatusBadRequest,
+				body: bytes.NewBuffer([]byte(
+					`{"status":"invalid event"}`,
+				)),
+
+				exp: fmt.Sprintf(`unexpected status code %d: {"status":"invalid event"}`, http.StatusBadRequest),
+			},
+			{
+				status: http.StatusBadRequest,
+
+				exp: fmt.Sprintf(`unexpected status code %d`, http.StatusBadRequest),
+			},
+		} {
+			t.Run("", func(t *testing.T) {
+				_, err = notifier.retrier.Check(tc.status, tc.body)
+				require.Equal(t, tc.exp, err.Error())
+			})
+		}
+	})
+}
+
+func TestWebhookTruncateAlerts(t *testing.T) {
+	alerts := make([]*types.Alert, 10)
+
+	truncatedAlerts, numTruncated := truncateAlerts(0, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(4, alerts)
+	require.Len(t, truncatedAlerts, 4)
+	require.EqualValues(t, 6, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(100, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+}
+
+func TestWebhookRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	secret := "secret"
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL(u.String()),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, secret)
+}
+
+func TestWebhookReadingURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "webhook_url")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.WebhookConfig{
+			URLFile:    f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestWebhookURLTemplating(t *testing.T) {
+	var calledURL string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calledURL = r.URL.Path
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	tests := []struct {
+		name           string
+		url            string
+		groupLabels    model.LabelSet
+		alertLabels    model.LabelSet
+		expectError    bool
+		expectedErrMsg string
+		expectedPath   string
+	}{
+		{
+			name:         "templating with alert labels",
+			url:          srv.URL + "/{{ .GroupLabels.alertname }}/{{ .CommonLabels.severity }}",
+			groupLabels:  model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:  model.LabelSet{"alertname": "TestAlert", "severity": "critical"},
+			expectError:  false,
+			expectedPath: "/TestAlert/critical",
+		},
+		{
+			name:           "invalid template field",
+			url:            srv.URL + "/{{ .InvalidField }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "failed to template webhook URL",
+		},
+		{
+			name:           "template renders to empty string",
+			url:            "{{ if .CommonLabels.nonexistent }}http://example.com{{ end }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "webhook URL is empty after templating",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			calledURL = "" // Reset for each test
+
+			notifier, err := New(
+				&config.WebhookConfig{
+					URL:        config.SecretTemplateURL(tc.url),
+					HTTPConfig: &commoncfg.HTTPClientConfig{},
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "test-group")
+			if tc.groupLabels != nil {
+				ctx = notify.WithGroupLabels(ctx, tc.groupLabels)
+			}
+
+			alerts := []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels:   tc.alertLabels,
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}
+
+			_, err = notifier.Notify(ctx, alerts...)
+
+			if tc.expectError {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expectedErrMsg)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedPath, calledURL)
+			}
+		})
+	}
+}