feat: alert manager template to template title and notification body

* chore: upgrade prometheus/common to latest available version

* chore: upgrade prometheus/prometheus to latest available version

* chore: easy changes first

* chore: slightly unsure changes

* fix: correct imported version of semconv in sdk.go

* test: ut fix, just matched expected and actual nothing else

* test: ut fix, just matched expected and actual nothing else

* test: ut fix, just matched expected and actual nothing else

* test: ut fix, just matched expected and actual nothing else

* test: ut fix, pass no nil prometheus registry

* chore: upgrade go version in dockerfile to 1.25

* chore: no need for our own alert store callback

* chore: 1.25 bullseye is still an rc so shifting to bookworm

* fix: parallel calls for each query in readmultiple method

* chore: remove unused var

* Sync PagerDuty frontend defaults with Alertmanager v0.31

Applied via @cursor push command

* chore: make ctx the first param

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>

cmd/enterprise/Dockerfile.integration

@@ -1,4 +1,4 @@
-FROM golang:1.24-bullseye
+FROM golang:1.25-bookworm
 
 ARG OS="linux"
 ARG TARGETARCH

cmd/enterprise/Dockerfile.with-web.integration

@@ -6,7 +6,7 @@ ENV NODE_OPTIONS=--max-old-space-size=8192
 RUN CI=1 yarn install
 RUN CI=1 yarn build
 
-FROM golang:1.24-bullseye
+FROM golang:1.25-bookworm
 
 ARG OS="linux"
 ARG TARGETARCH

frontend/src/container/CreateAlertChannels/defaults.ts

@@ -16,8 +16,8 @@ export const PagerInitialConfig: Partial<PagerChannel> = {
 	client: 'SigNoz Alert Manager',
 	client_url: 'https://enter-signoz-host-n-port-here/alerts',
 	details: JSON.stringify({
-		firing: `{{ template "pagerduty.default.instances" .Alerts.Firing }}`,
-		resolved: `{{ template "pagerduty.default.instances" .Alerts.Resolved }}`,
+		firing: `{{ .Alerts.Firing | toJson }}`,
+		resolved: `{{ .Alerts.Resolved | toJson }}`,
 		num_firing: '{{ .Alerts.Firing | len }}',
 		num_resolved: '{{ .Alerts.Resolved | len }}',
 	}),

frontend/src/container/DashboardContainer/DashboardDescription/__tests__/DashboardDescription.test.tsx

@@ -193,7 +193,6 @@ describe('Dashboard landing page actions header tests', () => {
 			handleDashboardLockToggle: jest.fn(),
 			dashboardResponse: {} as IDashboardContext['dashboardResponse'],
 			selectedDashboard: (getDashboardById.data as unknown) as Dashboard,
-			dashboardId: '4',
 			layouts: [],
 			panelMap: {},
 			setPanelMap: jest.fn(),

frontend/src/mocks-server/__mockdata__/alerts.ts

@@ -32,8 +32,8 @@ export const editSlackDescriptionDefaultValue = `{{ range .Alerts -}} *Alert:* {
 export const pagerDutyDescriptionDefaultVaule = `{{ if gt (len .Alerts.Firing) 0 -}} Alerts Firing: {{ range .Alerts.Firing }} - Message: {{ .Annotations.description }} Labels: {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} {{ end }} Annotations: {{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} {{ end }} Source: {{ .GeneratorURL }} {{ end }} {{- end }} {{ if gt (len .Alerts.Resolved) 0 -}} Alerts Resolved: {{ range .Alerts.Resolved }} - Message: {{ .Annotations.description }} Labels: {{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }} {{ end }} Annotations: {{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }} {{ end }} Source: {{ .GeneratorURL }} {{ end }} {{- end }}`;
 
 export const pagerDutyAdditionalDetailsDefaultValue = JSON.stringify({
-	firing: `{{ template "pagerduty.default.instances" .Alerts.Firing }}`,
-	resolved: `{{ template "pagerduty.default.instances" .Alerts.Resolved }}`,
+	firing: `{{ .Alerts.Firing | toJson }}`,
+	resolved: `{{ .Alerts.Resolved | toJson }}`,
 	num_firing: '{{ .Alerts.Firing | len }}',
 	num_resolved: '{{ .Alerts.Resolved | len }}',
 });

frontend/src/providers/Dashboard/Dashboard.tsx

@@ -68,7 +68,6 @@ export const DashboardContext = createContext<IDashboardContext>({
 		APIError
 	>,
 	selectedDashboard: {} as Dashboard,
-	dashboardId: '',
 	layouts: [],
 	panelMap: {},
 	setPanelMap: () => {},

frontend/src/providers/Dashboard/__tests__/Dashboard.test.tsx

@@ -58,8 +58,9 @@ jest.mock('react-redux', () => ({
 jest.mock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
 function TestComponent(): JSX.Element {
-	const { dashboardResponse, dashboardId, selectedDashboard } = useDashboard();
+	const { dashboardResponse, selectedDashboard } = useDashboard();
 	const { dashboardVariables } = useDashboardVariables();
+	const dashboardId = selectedDashboard?.id;
 
 	return (
 		<div>

frontend/src/providers/Dashboard/types.ts

@@ -15,7 +15,6 @@ export interface IDashboardContext {
 	handleDashboardLockToggle: (value: boolean) => void;
 	dashboardResponse: UseQueryResult<SuccessResponseV2<Dashboard>, unknown>;
 	selectedDashboard: Dashboard | undefined;
-	dashboardId: string;
 	layouts: Layout[];
 	panelMap: Record<string, { widgets: Layout[]; collapsed: boolean }>;
 	setPanelMap: React.Dispatch<React.SetStateAction<Record<string, any>>>;

go.mod

@@ -1,51 +1,52 @@
 module github.com/SigNoz/signoz
 
-go 1.24.0
+go 1.25.0
 
 require (
-	dario.cat/mergo v1.0.1
+	dario.cat/mergo v1.0.2
 	github.com/AfterShip/clickhouse-sql-parser v0.4.16
 	github.com/ClickHouse/clickhouse-go/v2 v2.40.1
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.129.10-rc.9
+	github.com/SigNoz/signoz-otel-collector v0.144.2
 	github.com/antlr4-go/antlr/v4 v4.13.1
 	github.com/antonmedv/expr v1.15.3
 	github.com/bytedance/sonic v1.14.1
 	github.com/cespare/xxhash/v2 v2.3.0
-	github.com/coreos/go-oidc/v3 v3.14.1
+	github.com/coreos/go-oidc/v3 v3.17.0
 	github.com/dgraph-io/ristretto/v2 v2.3.0
 	github.com/dustin/go-humanize v1.0.1
+	github.com/emersion/go-smtp v0.24.0
 	github.com/gin-gonic/gin v1.11.0
 	github.com/go-co-op/gocron v1.30.1
-	github.com/go-openapi/runtime v0.28.0
-	github.com/go-openapi/strfmt v0.23.0
+	github.com/go-openapi/runtime v0.29.2
+	github.com/go-openapi/strfmt v0.25.0
 	github.com/go-redis/redismock/v9 v9.2.0
-	github.com/go-viper/mapstructure/v2 v2.4.0
+	github.com/go-viper/mapstructure/v2 v2.5.0
 	github.com/gojek/heimdall/v7 v7.0.3
-	github.com/golang-jwt/jwt/v5 v5.3.0
+	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
 	github.com/huandu/go-sqlbuilder v1.35.0
 	github.com/jackc/pgx/v5 v5.7.6
-	github.com/json-iterator/go v1.1.12
+	github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12
 	github.com/knadh/koanf v1.5.0
-	github.com/knadh/koanf/v2 v2.2.0
-	github.com/mailru/easyjson v0.7.7
-	github.com/open-telemetry/opamp-go v0.19.0
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.128.0
+	github.com/knadh/koanf/v2 v2.3.2
+	github.com/mailru/easyjson v0.9.0
+	github.com/open-telemetry/opamp-go v0.22.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.144.0
 	github.com/openfga/api/proto v0.0.0-20250909172242-b4b2a12f5c67
 	github.com/openfga/language/pkg/go v0.2.0-beta.2.0.20250428093642-7aeebe78bbfe
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/alertmanager v0.28.1
+	github.com/prometheus/alertmanager v0.31.0
 	github.com/prometheus/client_golang v1.23.2
-	github.com/prometheus/common v0.66.1
-	github.com/prometheus/prometheus v0.304.1
+	github.com/prometheus/common v0.67.5
+	github.com/prometheus/prometheus v0.310.0
 	github.com/redis/go-redis/extra/redisotel/v9 v9.15.1
-	github.com/redis/go-redis/v9 v9.15.1
+	github.com/redis/go-redis/v9 v9.17.2
 	github.com/rs/cors v1.11.1
 	github.com/russellhaering/gosaml2 v0.9.0
 	github.com/russellhaering/goxmldsig v1.2.0
@@ -54,7 +55,7 @@ require (
 	github.com/sethvargo/go-password v0.2.0
 	github.com/smartystreets/goconvey v1.8.1
 	github.com/soheilhy/cmux v0.1.5
-	github.com/spf13/cobra v1.10.1
+	github.com/spf13/cobra v1.10.2
 	github.com/srikanthccv/ClickHouse-go-mock v0.13.0
 	github.com/stretchr/testify v1.11.1
 	github.com/swaggest/jsonschema-go v0.3.78
@@ -64,43 +65,72 @@ require (
 	github.com/uptrace/bun/dialect/pgdialect v1.2.9
 	github.com/uptrace/bun/dialect/sqlitedialect v1.2.9
 	github.com/uptrace/bun/extra/bunotel v1.2.9
-	go.opentelemetry.io/collector/confmap v1.34.0
-	go.opentelemetry.io/collector/otelcol v0.128.0
-	go.opentelemetry.io/collector/pdata v1.34.0
+	go.opentelemetry.io/collector/confmap v1.51.0
+	go.opentelemetry.io/collector/otelcol v0.144.0
+	go.opentelemetry.io/collector/pdata v1.51.0
 	go.opentelemetry.io/contrib/config v0.10.0
 	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0
-	go.opentelemetry.io/otel v1.38.0
-	go.opentelemetry.io/otel/metric v1.38.0
-	go.opentelemetry.io/otel/sdk v1.38.0
-	go.opentelemetry.io/otel/trace v1.38.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0
+	go.opentelemetry.io/otel v1.40.0
+	go.opentelemetry.io/otel/metric v1.40.0
+	go.opentelemetry.io/otel/sdk v1.40.0
+	go.opentelemetry.io/otel/trace v1.40.0
 	go.uber.org/multierr v1.11.0
-	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.46.0
-	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b
-	golang.org/x/net v0.47.0
-	golang.org/x/oauth2 v0.30.0
+	go.uber.org/zap v1.27.1
+	golang.org/x/crypto v0.47.0
+	golang.org/x/exp v0.0.0-20260112195511-716be5621a96
+	golang.org/x/net v0.49.0
+	golang.org/x/oauth2 v0.34.0
 	golang.org/x/sync v0.19.0
-	golang.org/x/text v0.32.0
-	google.golang.org/protobuf v1.36.9
+	golang.org/x/text v0.33.0
+	google.golang.org/protobuf v1.36.11
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/apimachinery v0.34.0
+	k8s.io/apimachinery v0.35.0
 	modernc.org/sqlite v1.39.1
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.7 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.39.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
+	github.com/aws/smithy-go v1.24.0 // indirect
 	github.com/bytedance/gopkg v0.1.3 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
+	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
+	github.com/go-openapi/swag/cmdutils v0.25.4 // indirect
+	github.com/go-openapi/swag/conv v0.25.4 // indirect
+	github.com/go-openapi/swag/fileutils v0.25.4 // indirect
+	github.com/go-openapi/swag/jsonname v0.25.4 // indirect
+	github.com/go-openapi/swag/jsonutils v0.25.4 // indirect
+	github.com/go-openapi/swag/loading v0.25.4 // indirect
+	github.com/go-openapi/swag/mangling v0.25.4 // indirect
+	github.com/go-openapi/swag/netutils v0.25.4 // indirect
+	github.com/go-openapi/swag/stringutils v0.25.4 // indirect
+	github.com/go-openapi/swag/typeutils v0.25.4 // indirect
+	github.com/go-openapi/swag/yamlutils v0.25.4 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.27.0 // indirect
-	github.com/goccy/go-yaml v1.18.0 // indirect
+	github.com/goccy/go-yaml v1.19.2 // indirect
+	github.com/hashicorp/go-metrics v0.5.4 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/prometheus/client_golang/exp v0.0.0-20260108101519-fb0838f53562 // indirect
 	github.com/redis/go-redis/extra/rediscmd/v9 v9.15.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/swaggest/refl v1.4.0 // indirect
@@ -108,69 +138,70 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.3.0 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect
-	go.opentelemetry.io/collector/config/configretry v1.34.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.opentelemetry.io/collector/client v1.50.0 // indirect
+	go.opentelemetry.io/collector/config/configoptional v1.50.0 // indirect
+	go.opentelemetry.io/collector/config/configretry v1.50.0 // indirect
+	go.opentelemetry.io/collector/exporter/exporterhelper v0.144.0 // indirect
+	go.opentelemetry.io/collector/internal/componentalias v0.145.0 // indirect
+	go.opentelemetry.io/collector/pdata/xpdata v0.144.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	golang.org/x/arch v0.20.0 // indirect
-	golang.org/x/tools/godoc v0.1.0-deprecated // indirect
 	modernc.org/libc v1.66.10 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 )
 
 require (
-	cel.dev/expr v0.24.0 // indirect
-	cloud.google.com/go/auth v0.16.1 // indirect
+	cel.dev/expr v0.25.1 // indirect
+	cloud.google.com/go/auth v0.18.1 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
-	cloud.google.com/go/compute/metadata v0.8.2 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
 	github.com/ClickHouse/ch-go v0.67.0 // indirect
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Yiling-J/theine-go v0.6.2 // indirect
-	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
+	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aws/aws-sdk-go v1.55.7 // indirect
 	github.com/beevik/etree v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
-	github.com/coder/quartz v0.1.2 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/coder/quartz v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/ebitengine/purego v0.8.4 // indirect
+	github.com/ebitengine/purego v0.9.1 // indirect
 	github.com/edsrzf/mmap-go v1.2.0 // indirect
-	github.com/elastic/lunes v0.1.0 // indirect
+	github.com/elastic/lunes v0.2.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
-	github.com/expr-lang/expr v1.17.5
+	github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
+	github.com/expr-lang/expr v1.17.7
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.7.1 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.1 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
-	github.com/go-openapi/analysis v0.23.0 // indirect
-	github.com/go-openapi/errors v0.22.0 // indirect
-	github.com/go-openapi/jsonpointer v0.21.0 // indirect
-	github.com/go-openapi/jsonreference v0.21.0 // indirect
-	github.com/go-openapi/loads v0.22.0 // indirect
-	github.com/go-openapi/spec v0.21.0 // indirect
-	github.com/go-openapi/swag v0.23.0 // indirect
-	github.com/go-openapi/validate v0.24.0 // indirect
+	github.com/go-openapi/analysis v0.24.2 // indirect
+	github.com/go-openapi/errors v0.22.6 // indirect
+	github.com/go-openapi/jsonpointer v0.22.4 // indirect
+	github.com/go-openapi/jsonreference v0.21.4 // indirect
+	github.com/go-openapi/loads v0.23.2 // indirect
+	github.com/go-openapi/spec v0.22.3 // indirect
+	github.com/go-openapi/swag v0.25.4 // indirect
+	github.com/go-openapi/validate v0.25.1 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
-	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gojek/valkyrie v0.0.0-20180215180059-6aee720afcdf // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
@@ -178,22 +209,22 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/cel-go v0.26.1 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
-	github.com/googleapis/gax-go/v2 v2.14.2 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
+	github.com/googleapis/gax-go/v2 v2.16.0 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
-	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
+	github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.7 // indirect
-	github.com/hashicorp/go-version v1.7.0 // indirect
+	github.com/hashicorp/go-version v1.8.0 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/hashicorp/memberlist v0.5.1 // indirect
+	github.com/hashicorp/memberlist v0.5.4 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -201,26 +232,25 @@ require (
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jessevdk/go-flags v1.6.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jonboulle/clockwork v0.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
-	github.com/leodido/go-syslog/v4 v4.2.0 // indirect
+	github.com/leodido/go-syslog/v4 v4.3.0 // indirect
 	github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
-	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
+	github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
 	github.com/magefile/mage v1.15.0 // indirect
 	github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/mdlayher/socket v0.4.1 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/mdlayher/vsock v1.2.1 // indirect
 	github.com/mfridman/interpolate v0.0.2 // indirect
-	github.com/miekg/dns v1.1.65 // indirect
+	github.com/miekg/dns v1.1.72 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
@@ -229,27 +259,27 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/natefinch/wrap v0.2.0 // indirect
-	github.com/oklog/run v1.1.0 // indirect
+	github.com/oklog/run v1.2.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/oklog/ulid/v2 v2.1.1
 	github.com/open-feature/go-sdk v1.17.0
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.128.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.128.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.128.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.128.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.144.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.145.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.145.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.145.0 // indirect
 	github.com/openfga/openfga v1.10.1
 	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
-	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pierrec/lz4/v4 v4.1.23 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/pressly/goose/v3 v3.25.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/exporter-toolkit v0.14.0 // indirect
-	github.com/prometheus/otlptranslator v0.0.0-20250320144820-d800c8b0eb07 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
-	github.com/prometheus/sigv4 v0.1.2 // indirect
+	github.com/prometheus/exporter-toolkit v0.15.1 // indirect
+	github.com/prometheus/otlptranslator v1.0.0 // indirect
+	github.com/prometheus/procfs v0.19.2 // indirect
+	github.com/prometheus/sigv4 v0.4.1 // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/sagikazarmark/locafero v0.9.0 // indirect
@@ -257,7 +287,7 @@ require (
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/segmentio/backo-go v1.0.1 // indirect
 	github.com/sethvargo/go-retry v0.3.0 // indirect
-	github.com/shirou/gopsutil/v4 v4.25.5 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.12 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 // indirect
@@ -272,94 +302,92 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/swaggest/openapi-go v0.2.60
 	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tklauser/go-sysconf v0.3.15 // indirect
-	github.com/tklauser/numcpus v0.10.0 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
 	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
-	github.com/trivago/tgo v1.0.7 // indirect
-	github.com/valyala/fastjson v1.6.4 // indirect
+	github.com/valyala/fastjson v1.6.7 // indirect
 	github.com/vjeantet/grok v1.0.1 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
-	go.mongodb.org/mongo-driver v1.17.1 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/collector/component v1.34.0 // indirect
-	go.opentelemetry.io/collector/component/componentstatus v0.128.0 // indirect
-	go.opentelemetry.io/collector/component/componenttest v0.128.0 // indirect
-	go.opentelemetry.io/collector/config/configtelemetry v0.128.0 // indirect
-	go.opentelemetry.io/collector/confmap/provider/envprovider v1.34.0 // indirect
-	go.opentelemetry.io/collector/confmap/provider/fileprovider v1.34.0 // indirect
-	go.opentelemetry.io/collector/confmap/xconfmap v0.128.0 // indirect
-	go.opentelemetry.io/collector/connector v0.128.0 // indirect
-	go.opentelemetry.io/collector/connector/connectortest v0.128.0 // indirect
-	go.opentelemetry.io/collector/connector/xconnector v0.128.0 // indirect
-	go.opentelemetry.io/collector/consumer v1.34.0 // indirect
-	go.opentelemetry.io/collector/consumer/consumererror v0.128.0 // indirect
-	go.opentelemetry.io/collector/consumer/consumertest v0.128.0 // indirect
-	go.opentelemetry.io/collector/consumer/xconsumer v0.128.0 // indirect
-	go.opentelemetry.io/collector/exporter v0.128.0 // indirect
-	go.opentelemetry.io/collector/exporter/exportertest v0.128.0 // indirect
-	go.opentelemetry.io/collector/exporter/xexporter v0.128.0 // indirect
-	go.opentelemetry.io/collector/extension v1.34.0 // indirect
-	go.opentelemetry.io/collector/extension/extensioncapabilities v0.128.0 // indirect
-	go.opentelemetry.io/collector/extension/extensiontest v0.128.0 // indirect
-	go.opentelemetry.io/collector/extension/xextension v0.128.0 // indirect
-	go.opentelemetry.io/collector/featuregate v1.34.0 // indirect
-	go.opentelemetry.io/collector/internal/fanoutconsumer v0.128.0 // indirect
-	go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect
-	go.opentelemetry.io/collector/pdata/pprofile v0.128.0 // indirect
-	go.opentelemetry.io/collector/pdata/testdata v0.128.0 // indirect
-	go.opentelemetry.io/collector/pipeline v0.128.0 // indirect
-	go.opentelemetry.io/collector/pipeline/xpipeline v0.128.0 // indirect
-	go.opentelemetry.io/collector/processor v1.34.0 // indirect
-	go.opentelemetry.io/collector/processor/processorhelper v0.128.0 // indirect
-	go.opentelemetry.io/collector/processor/processortest v0.128.0 // indirect
-	go.opentelemetry.io/collector/processor/xprocessor v0.128.0 // indirect
-	go.opentelemetry.io/collector/receiver v1.34.0 // indirect
-	go.opentelemetry.io/collector/receiver/receiverhelper v0.128.0 // indirect
-	go.opentelemetry.io/collector/receiver/receivertest v0.128.0 // indirect
-	go.opentelemetry.io/collector/receiver/xreceiver v0.128.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.128.0
-	go.opentelemetry.io/collector/service v0.128.0 // indirect
-	go.opentelemetry.io/collector/service/hostcapabilities v0.128.0 // indirect
-	go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 // indirect
-	go.opentelemetry.io/contrib/otelconf v0.16.0 // indirect
-	go.opentelemetry.io/contrib/propagators/b3 v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/prometheus v0.58.0
-	go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.12.2 // indirect
-	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 // indirect
-	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 // indirect
-	go.opentelemetry.io/otel/log v0.12.2 // indirect
-	go.opentelemetry.io/otel/sdk/log v0.12.2 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.38.0
-	go.opentelemetry.io/proto/otlp v1.8.0 // indirect
+	go.mongodb.org/mongo-driver v1.17.6 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/collector/component v1.51.0 // indirect
+	go.opentelemetry.io/collector/component/componentstatus v0.145.0 // indirect
+	go.opentelemetry.io/collector/component/componenttest v0.145.0 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.144.0 // indirect
+	go.opentelemetry.io/collector/confmap/provider/envprovider v1.50.0 // indirect
+	go.opentelemetry.io/collector/confmap/provider/fileprovider v1.50.0 // indirect
+	go.opentelemetry.io/collector/confmap/xconfmap v0.145.0 // indirect
+	go.opentelemetry.io/collector/connector v0.144.0 // indirect
+	go.opentelemetry.io/collector/connector/connectortest v0.144.0 // indirect
+	go.opentelemetry.io/collector/connector/xconnector v0.144.0 // indirect
+	go.opentelemetry.io/collector/consumer v1.51.0 // indirect
+	go.opentelemetry.io/collector/consumer/consumererror v0.144.0 // indirect
+	go.opentelemetry.io/collector/consumer/consumertest v0.145.0 // indirect
+	go.opentelemetry.io/collector/consumer/xconsumer v0.145.0 // indirect
+	go.opentelemetry.io/collector/exporter v1.50.0 // indirect
+	go.opentelemetry.io/collector/exporter/exportertest v0.144.0 // indirect
+	go.opentelemetry.io/collector/exporter/xexporter v0.144.0 // indirect
+	go.opentelemetry.io/collector/extension v1.50.0 // indirect
+	go.opentelemetry.io/collector/extension/extensioncapabilities v0.144.0 // indirect
+	go.opentelemetry.io/collector/extension/extensiontest v0.144.0 // indirect
+	go.opentelemetry.io/collector/extension/xextension v0.144.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.51.0 // indirect
+	go.opentelemetry.io/collector/internal/fanoutconsumer v0.144.0 // indirect
+	go.opentelemetry.io/collector/internal/telemetry v0.144.0 // indirect
+	go.opentelemetry.io/collector/pdata/pprofile v0.145.0 // indirect
+	go.opentelemetry.io/collector/pdata/testdata v0.145.0 // indirect
+	go.opentelemetry.io/collector/pipeline v1.51.0 // indirect
+	go.opentelemetry.io/collector/pipeline/xpipeline v0.144.0 // indirect
+	go.opentelemetry.io/collector/processor v1.51.0 // indirect
+	go.opentelemetry.io/collector/processor/processorhelper v0.144.0 // indirect
+	go.opentelemetry.io/collector/processor/processortest v0.145.0 // indirect
+	go.opentelemetry.io/collector/processor/xprocessor v0.145.0 // indirect
+	go.opentelemetry.io/collector/receiver v1.50.0 // indirect
+	go.opentelemetry.io/collector/receiver/receiverhelper v0.144.0 // indirect
+	go.opentelemetry.io/collector/receiver/receivertest v0.144.0 // indirect
+	go.opentelemetry.io/collector/receiver/xreceiver v0.144.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.128.1-0.20250610090210-188191247685
+	go.opentelemetry.io/collector/service v0.144.0 // indirect
+	go.opentelemetry.io/collector/service/hostcapabilities v0.144.0 // indirect
+	go.opentelemetry.io/contrib/bridges/otelzap v0.13.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.65.0 // indirect
+	go.opentelemetry.io/contrib/otelconf v0.18.0 // indirect
+	go.opentelemetry.io/contrib/propagators/b3 v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.60.0
+	go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0 // indirect
+	go.opentelemetry.io/otel/log v0.15.0 // indirect
+	go.opentelemetry.io/otel/sdk/log v0.14.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.40.0
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/mock v0.6.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.30.0 // indirect
-	golang.org/x/sys v0.39.0 // indirect
-	golang.org/x/time v0.11.0 // indirect
-	golang.org/x/tools v0.39.0 // indirect
-	gonum.org/v1/gonum v0.16.0 // indirect
-	google.golang.org/api v0.236.0
-	google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
-	google.golang.org/grpc v1.75.1 // indirect
+	golang.org/x/mod v0.32.0 // indirect
+	golang.org/x/sys v0.40.0 // indirect
+	golang.org/x/time v0.14.0 // indirect
+	golang.org/x/tools v0.41.0 // indirect
+	gonum.org/v1/gonum v0.17.0 // indirect
+	google.golang.org/api v0.265.0
+	google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect
+	google.golang.org/grpc v1.78.0 // indirect
 	gopkg.in/telebot.v3 v3.3.8 // indirect
-	k8s.io/client-go v0.34.0 // indirect
+	k8s.io/client-go v0.35.0 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
-	sigs.k8s.io/yaml v1.6.0 // indirect
+	k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect
 )
 
 replace github.com/expr-lang/expr => github.com/SigNoz/expr v1.17.7-beta

pkg/alertmanager/alertmanagernotify/email/email.go

@@ -0,0 +1,430 @@
+package email
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"log/slog"
+	"math/rand"
+	"mime"
+	"mime/multipart"
+	"mime/quotedprintable"
+	"net"
+	"net/mail"
+	"net/smtp"
+	"net/textproto"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "email"
+)
+
+// Email implements a Notifier for email notifications.
+type Email struct {
+	conf     *config.EmailConfig
+	tmpl     *template.Template
+	logger   *slog.Logger
+	hostname string
+}
+
+var errNoAuthUserNameConfigured = errors.NewInternalf(errors.CodeInternal, "no auth username configured")
+
+// New returns a new Email notifier.
+func New(c *config.EmailConfig, t *template.Template, l *slog.Logger) *Email {
+	if _, ok := c.Headers["Subject"]; !ok {
+		c.Headers["Subject"] = config.DefaultEmailSubject
+	}
+	if _, ok := c.Headers["To"]; !ok {
+		c.Headers["To"] = c.To
+	}
+	if _, ok := c.Headers["From"]; !ok {
+		c.Headers["From"] = c.From
+	}
+
+	h, err := os.Hostname()
+	// If we can't get the hostname, we'll use localhost
+	if err != nil {
+		h = "localhost.localdomain"
+	}
+	return &Email{conf: c, tmpl: t, logger: l, hostname: h}
+}
+
+// auth resolves a string of authentication mechanisms.
+func (n *Email) auth(mechs string) (smtp.Auth, error) {
+	username := n.conf.AuthUsername
+
+	// If no username is set, return custom error which can be ignored if needed.
+	if n.conf.AuthUsername == "" {
+		return nil, errNoAuthUserNameConfigured
+	}
+
+	err := &types.MultiError{}
+	for mech := range strings.SplitSeq(mechs, " ") {
+		switch mech {
+		case "CRAM-MD5":
+			secret, secretErr := n.getAuthSecret()
+			if secretErr != nil {
+				err.Add(secretErr)
+				continue
+			}
+			if secret == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing secret for CRAM-MD5 auth mechanism"))
+				continue
+			}
+			return smtp.CRAMMD5Auth(username, secret), nil
+
+		case "PLAIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				err.Add(passwordErr)
+				continue
+			}
+			if password == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing password for PLAIN auth mechanism"))
+				continue
+			}
+			identity := n.conf.AuthIdentity
+
+			return smtp.PlainAuth(identity, username, password, n.conf.Smarthost.Host), nil
+		case "LOGIN":
+			password, passwordErr := n.getPassword()
+			if passwordErr != nil {
+				err.Add(passwordErr)
+				continue
+			}
+			if password == "" {
+				err.Add(errors.NewInternalf(errors.CodeInternal, "missing password for LOGIN auth mechanism"))
+				continue
+			}
+			return LoginAuth(username, password), nil
+		}
+	}
+	if err.Len() == 0 {
+		err.Add(errors.NewInternalf(errors.CodeInternal, "unknown auth mechanism: %s", mechs))
+	}
+	return nil, err
+}
+
+// Notify implements the Notifier interface.
+func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var (
+		c       *smtp.Client
+		conn    net.Conn
+		err     error
+		success = false
+	)
+	// Determine whether to use Implicit TLS
+	var useImplicitTLS bool
+	if n.conf.ForceImplicitTLS != nil {
+		useImplicitTLS = *n.conf.ForceImplicitTLS
+	} else {
+		// Default logic: port 465 uses implicit TLS (backward compatibility)
+		useImplicitTLS = n.conf.Smarthost.Port == "465"
+	}
+
+	if useImplicitTLS {
+		tlsConfig, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "parse TLS configuration")
+		}
+		if tlsConfig.ServerName == "" {
+			tlsConfig.ServerName = n.conf.Smarthost.Host
+		}
+
+		conn, err = tls.Dial("tcp", n.conf.Smarthost.String(), tlsConfig)
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "establish TLS connection to server")
+		}
+	} else {
+		var (
+			d   = net.Dialer{}
+			err error
+		)
+		conn, err = d.DialContext(ctx, "tcp", n.conf.Smarthost.String())
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "establish connection to server")
+		}
+	}
+	c, err = smtp.NewClient(conn, n.conf.Smarthost.Host)
+	if err != nil {
+		conn.Close()
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "create SMTP client")
+	}
+	defer func() {
+		// Try to clean up after ourselves but don't log anything if something has failed.
+		if err := c.Quit(); success && err != nil {
+			n.logger.WarnContext(ctx, "failed to close SMTP connection", "err", err)
+		}
+	}()
+
+	if n.conf.Hello != "" {
+		err = c.Hello(n.conf.Hello)
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send EHLO command")
+		}
+	}
+
+	// Global Config guarantees RequireTLS is not nil.
+	if *n.conf.RequireTLS && !useImplicitTLS {
+		if ok, _ := c.Extension("STARTTLS"); !ok {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "'require_tls' is true (default) but %q does not advertise the STARTTLS extension", n.conf.Smarthost)
+		}
+
+		tlsConf, err := commoncfg.NewTLSConfig(n.conf.TLSConfig)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "parse TLS configuration")
+		}
+		if tlsConf.ServerName == "" {
+			tlsConf.ServerName = n.conf.Smarthost.Host
+		}
+
+		if err := c.StartTLS(tlsConf); err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send STARTTLS command")
+		}
+	}
+
+	if ok, mech := c.Extension("AUTH"); ok {
+		auth, err := n.auth(mech)
+		if err != nil && err != errNoAuthUserNameConfigured {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "find auth mechanism")
+		} else if err == errNoAuthUserNameConfigured {
+			n.logger.DebugContext(ctx, "no auth username configured. Attempting to send email without authenticating")
+		}
+		if auth != nil {
+			if err := c.Auth(auth); err != nil {
+				return true, errors.WrapInternalf(err, errors.CodeInternal, "%T auth", auth)
+			}
+		}
+	}
+
+	var (
+		tmplErr error
+		data    = notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
+		tmpl    = notify.TmplText(n.tmpl, data, &tmplErr)
+	)
+	from := tmpl(n.conf.From)
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "execute 'from' template")
+	}
+	to := tmpl(n.conf.To)
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "execute 'to' template")
+	}
+
+	addrs, err := mail.ParseAddressList(from)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "parse 'from' addresses")
+	}
+	if len(addrs) != 1 {
+		return false, errors.NewInternalf(errors.CodeInternal, "must be exactly one 'from' address (got: %d)", len(addrs))
+	}
+	if err = c.Mail(addrs[0].Address); err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "send MAIL command")
+	}
+	addrs, err = mail.ParseAddressList(to)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "parse 'to' addresses")
+	}
+	for _, addr := range addrs {
+		if err = c.Rcpt(addr.Address); err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "send RCPT command")
+		}
+	}
+
+	// Send the email headers and body.
+	message, err := c.Data()
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "send DATA command")
+	}
+	closeOnce := sync.OnceValue(func() error {
+		return message.Close()
+	})
+	// Close the message when this method exits in order to not leak resources. Even though we're calling this explicitly
+	// further down, the method may exit before then.
+	defer func() {
+		// If we try close an already-closed writer, it'll send a subsequent request to the server which is invalid.
+		_ = closeOnce()
+	}()
+
+	buffer := &bytes.Buffer{}
+	for header, t := range n.conf.Headers {
+		value, err := n.tmpl.ExecuteTextString(t, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute %q header template", header)
+		}
+		fmt.Fprintf(buffer, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", value))
+	}
+
+	if _, ok := n.conf.Headers["Message-Id"]; !ok {
+		fmt.Fprintf(buffer, "Message-Id: %s\r\n", fmt.Sprintf("<%d.%d@%s>", time.Now().UnixNano(), rand.Uint64(), n.hostname))
+	}
+
+	if n.conf.Threading.Enabled {
+		key, err := notify.ExtractGroupKey(ctx)
+		if err != nil {
+			return false, err
+		}
+		// Add threading headers. All notifications for the same alert group
+		// (identified by key hash) are threaded together.
+		threadBy := ""
+		if n.conf.Threading.ThreadByDate != "none" {
+			// ThreadByDate is 'daily':
+			// Use current date so all mails for this alert today thread together.
+			threadBy = time.Now().Format("2006-01-02")
+		}
+		keyHash := key.Hash()
+		if len(keyHash) > 16 {
+			keyHash = keyHash[:16]
+		}
+		// The thread root ID is a Message-ID that doesn't correspond to
+		// any actual email. Email clients following the (commonly used) JWZ
+		// algorithm will create a dummy container to group these messages.
+		threadRootID := fmt.Sprintf("<alert-%s-%s@alertmanager>", keyHash, threadBy)
+		fmt.Fprintf(buffer, "References: %s\r\n", threadRootID)
+		fmt.Fprintf(buffer, "In-Reply-To: %s\r\n", threadRootID)
+	}
+
+	multipartBuffer := &bytes.Buffer{}
+	multipartWriter := multipart.NewWriter(multipartBuffer)
+
+	fmt.Fprintf(buffer, "Date: %s\r\n", time.Now().Format(time.RFC1123Z))
+	fmt.Fprintf(buffer, "Content-Type: multipart/alternative;  boundary=%s\r\n", multipartWriter.Boundary())
+	fmt.Fprintf(buffer, "MIME-Version: 1.0\r\n\r\n")
+
+	// TODO: Add some useful headers here, such as URL of the alertmanager
+	// and active/resolved.
+	_, err = message.Write(buffer.Bytes())
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "write headers")
+	}
+
+	if len(n.conf.Text) > 0 {
+		// Text template
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/plain; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for text template")
+		}
+		body, err := n.tmpl.ExecuteTextString(n.conf.Text, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute text template")
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "write text part")
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "close text part")
+		}
+	}
+
+	if len(n.conf.HTML) > 0 {
+		// Html template
+		// Preferred alternative placed last per section 5.1.4 of RFC 2046
+		// https://www.ietf.org/rfc/rfc2046.txt
+		w, err := multipartWriter.CreatePart(textproto.MIMEHeader{
+			"Content-Transfer-Encoding": {"quoted-printable"},
+			"Content-Type":              {"text/html; charset=UTF-8"},
+		})
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for html template")
+		}
+		body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute html template")
+		}
+		qw := quotedprintable.NewWriter(w)
+		_, err = qw.Write([]byte(body))
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "write HTML part")
+		}
+		err = qw.Close()
+		if err != nil {
+			return true, errors.WrapInternalf(err, errors.CodeInternal, "close HTML part")
+		}
+	}
+
+	err = multipartWriter.Close()
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "close multipartWriter")
+	}
+
+	_, err = message.Write(multipartBuffer.Bytes())
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "write body buffer")
+	}
+
+	// Complete the message and await response.
+	if err = closeOnce(); err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "delivery failure")
+	}
+
+	success = true
+	return false, nil
+}
+
+type loginAuth struct {
+	username, password string
+}
+
+func LoginAuth(username, password string) smtp.Auth {
+	return &loginAuth{username, password}
+}
+
+func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
+	return "LOGIN", []byte{}, nil
+}
+
+// Used for AUTH LOGIN. (Maybe password should be encrypted).
+func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
+	if more {
+		switch strings.ToLower(string(fromServer)) {
+		case "username:":
+			return []byte(a.username), nil
+		case "password:":
+			return []byte(a.password), nil
+		default:
+			return nil, errors.NewInternalf(errors.CodeInternal, "unexpected server challenge")
+		}
+	}
+	return nil, nil
+}
+
+func (n *Email) getPassword() (string, error) {
+	if len(n.conf.AuthPasswordFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthPasswordFile)
+		if err != nil {
+			return "", errors.NewInternalf(errors.CodeInternal, "could not read %s: %v", n.conf.AuthPasswordFile, err)
+		}
+		return strings.TrimSpace(string(content)), nil
+	}
+	return string(n.conf.AuthPassword), nil
+}
+
+func (n *Email) getAuthSecret() (string, error) {
+	if len(n.conf.AuthSecretFile) > 0 {
+		content, err := os.ReadFile(n.conf.AuthSecretFile)
+		if err != nil {
+			return "", errors.NewInternalf(errors.CodeInternal, "could not read %s: %v", n.conf.AuthSecretFile, err)
+		}
+		return string(content), nil
+	}
+	return string(n.conf.AuthSecret), nil
+}

pkg/alertmanager/alertmanagernotify/email/testdata/auth-local.yml

@@ -0,0 +1,4 @@
+smarthost: 127.0.0.1:1026
+server: http://127.0.0.1:1081/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/auth.yml

@@ -0,0 +1,4 @@
+smarthost: maildev-auth:1025
+server: http://maildev-auth:1080/
+username: user
+password: pass

pkg/alertmanager/alertmanagernotify/email/testdata/noauth-local.yml

@@ -0,0 +1,2 @@
+smarthost: 127.0.0.1:1025
+server: http://127.0.0.1:1080/

pkg/alertmanager/alertmanagernotify/email/testdata/noauth.yml

@@ -0,0 +1,2 @@
+smarthost: maildev-noauth:1025
+server: http://maildev-noauth:1080/

pkg/alertmanager/alertmanagernotify/msteamsv2/msteamsv2.go

@@ -27,6 +27,10 @@ const (
 	colorGrey  = "Warning"
 )
 
+const (
+	Integration = "msteamsv2"
+)
+
 type Notifier struct {
 	conf         *config.MSTeamsV2Config
 	titleLink    string
@@ -87,7 +91,7 @@ type teamsMessage struct {
 
 // New returns a new notifier that uses the Microsoft Teams Power Platform connector.
 func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
-	client, err := commoncfg.NewClientFromConfig(*c.HTTPConfig, "msteamsv2", httpOpts...)
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
 	}

pkg/alertmanager/alertmanagernotify/opsgenie/api_key_file

@@ -0,0 +1,2 @@
+my_secret_api_key
+

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie.go

@@ -0,0 +1,290 @@
+package opsgenie
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"maps"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "opsgenie"
+)
+
+// https://docs.opsgenie.com/docs/alert-api - 130 characters meaning runes.
+const maxMessageLenRunes = 130
+
+// Notifier implements a Notifier for OpsGenie notifications.
+type Notifier struct {
+	conf    *config.OpsGenieConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new OpsGenie notifier.
+func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:    c,
+		tmpl:    t,
+		logger:  l,
+		client:  client,
+		retrier: &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
+	}, nil
+}
+
+type opsGenieCreateMessage struct {
+	Alias       string                           `json:"alias"`
+	Message     string                           `json:"message"`
+	Description string                           `json:"description,omitempty"`
+	Details     map[string]string                `json:"details"`
+	Source      string                           `json:"source"`
+	Responders  []opsGenieCreateMessageResponder `json:"responders,omitempty"`
+	Tags        []string                         `json:"tags,omitempty"`
+	Note        string                           `json:"note,omitempty"`
+	Priority    string                           `json:"priority,omitempty"`
+	Entity      string                           `json:"entity,omitempty"`
+	Actions     []string                         `json:"actions,omitempty"`
+}
+
+type opsGenieCreateMessageResponder struct {
+	ID       string `json:"id,omitempty"`
+	Name     string `json:"name,omitempty"`
+	Username string `json:"username,omitempty"`
+	Type     string `json:"type"` // team, user, escalation, schedule etc.
+}
+
+type opsGenieCloseMessage struct {
+	Source string `json:"source"`
+}
+
+type opsGenieUpdateMessageMessage struct {
+	Message string `json:"message,omitempty"`
+}
+
+type opsGenieUpdateDescriptionMessage struct {
+	Description string `json:"description,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	requests, retry, err := n.createRequests(ctx, as...)
+	if err != nil {
+		return retry, err
+	}
+
+	for _, req := range requests {
+		req.Header.Set("User-Agent", notify.UserAgentHeader)
+		resp, err := n.client.Do(req) //nolint:bodyclose
+		if err != nil {
+			return true, err
+		}
+		shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+		notify.Drain(resp)
+		if err != nil {
+			return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+		}
+	}
+	return true, nil
+}
+
+// Like Split but filter out empty strings.
+func safeSplit(s, sep string) []string {
+	a := strings.Split(strings.TrimSpace(s), sep)
+	b := a[:0]
+	for _, x := range a {
+		if x != "" {
+			b = append(b, x)
+		}
+	}
+	return b
+}
+
+// Create requests for a list of alerts.
+func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return nil, false, err
+	}
+	logger := n.logger.With("group_key", key)
+	logger.DebugContext(ctx, "extracted group key")
+
+	data := notify.GetTemplateData(ctx, n.tmpl, as, logger)
+
+	tmpl := notify.TmplText(n.tmpl, data, &err)
+
+	details := make(map[string]string)
+
+	maps.Copy(details, data.CommonLabels)
+
+	for k, v := range n.conf.Details {
+		details[k] = tmpl(v)
+	}
+
+	requests := []*http.Request{}
+
+	var (
+		alias  = key.Hash()
+		alerts = types.Alerts(as...)
+	)
+	switch alerts.Status() {
+	case model.AlertResolved:
+		resolvedEndpointURL := n.conf.APIURL.Copy()
+		resolvedEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/close", alias)
+		q := resolvedEndpointURL.Query()
+		q.Set("identifierType", "alias")
+		resolvedEndpointURL.RawQuery = q.Encode()
+		msg := &opsGenieCloseMessage{Source: tmpl(n.conf.Source)}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", resolvedEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+	default:
+		message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
+		if truncated {
+			logger.WarnContext(ctx, "Truncated message", "alert", key, "max_runes", maxMessageLenRunes)
+		}
+
+		createEndpointURL := n.conf.APIURL.Copy()
+		createEndpointURL.Path += "v2/alerts"
+
+		var responders []opsGenieCreateMessageResponder
+		for _, r := range n.conf.Responders {
+			responder := opsGenieCreateMessageResponder{
+				ID:       tmpl(r.ID),
+				Name:     tmpl(r.Name),
+				Username: tmpl(r.Username),
+				Type:     tmpl(r.Type),
+			}
+
+			if responder == (opsGenieCreateMessageResponder{}) {
+				// Filter out empty responders. This is useful if you want to fill
+				// responders dynamically from alert's common labels.
+				continue
+			}
+
+			if responder.Type == "teams" {
+				teams := safeSplit(responder.Name, ",")
+				for _, team := range teams {
+					newResponder := opsGenieCreateMessageResponder{
+						Name: tmpl(team),
+						Type: tmpl("team"),
+					}
+					responders = append(responders, newResponder)
+				}
+				continue
+			}
+
+			responders = append(responders, responder)
+		}
+
+		msg := &opsGenieCreateMessage{
+			Alias:       alias,
+			Message:     message,
+			Description: tmpl(n.conf.Description),
+			Details:     details,
+			Source:      tmpl(n.conf.Source),
+			Responders:  responders,
+			Tags:        safeSplit(tmpl(n.conf.Tags), ","),
+			Note:        tmpl(n.conf.Note),
+			Priority:    tmpl(n.conf.Priority),
+			Entity:      tmpl(n.conf.Entity),
+			Actions:     safeSplit(tmpl(n.conf.Actions), ","),
+		}
+		var buf bytes.Buffer
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return nil, false, err
+		}
+		req, err := http.NewRequest("POST", createEndpointURL.String(), &buf)
+		if err != nil {
+			return nil, true, err
+		}
+		requests = append(requests, req.WithContext(ctx))
+
+		if n.conf.UpdateAlerts {
+			updateMessageEndpointURL := n.conf.APIURL.Copy()
+			updateMessageEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/message", alias)
+			q := updateMessageEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateMessageEndpointURL.RawQuery = q.Encode()
+			updateMsgMsg := &opsGenieUpdateMessageMessage{
+				Message: msg.Message,
+			}
+			var updateMessageBuf bytes.Buffer
+			if err := json.NewEncoder(&updateMessageBuf).Encode(updateMsgMsg); err != nil {
+				return nil, false, err
+			}
+			req, err := http.NewRequest("PUT", updateMessageEndpointURL.String(), &updateMessageBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req)
+
+			updateDescriptionEndpointURL := n.conf.APIURL.Copy()
+			updateDescriptionEndpointURL.Path += fmt.Sprintf("v2/alerts/%s/description", alias)
+			q = updateDescriptionEndpointURL.Query()
+			q.Set("identifierType", "alias")
+			updateDescriptionEndpointURL.RawQuery = q.Encode()
+			updateDescMsg := &opsGenieUpdateDescriptionMessage{
+				Description: msg.Description,
+			}
+
+			var updateDescriptionBuf bytes.Buffer
+			if err := json.NewEncoder(&updateDescriptionBuf).Encode(updateDescMsg); err != nil {
+				return nil, false, err
+			}
+			req, err = http.NewRequest("PUT", updateDescriptionEndpointURL.String(), &updateDescriptionBuf)
+			if err != nil {
+				return nil, true, err
+			}
+			requests = append(requests, req.WithContext(ctx))
+		}
+	}
+
+	var apiKey string
+	if n.conf.APIKey != "" {
+		apiKey = tmpl(string(n.conf.APIKey))
+	} else {
+		content, err := os.ReadFile(n.conf.APIKeyFile)
+		if err != nil {
+			return nil, false, errors.WrapInternalf(err, errors.CodeInternal, "read key_file error")
+		}
+		apiKey = tmpl(string(content))
+		apiKey = strings.TrimSpace(string(apiKey))
+	}
+
+	if err != nil {
+		return nil, false, errors.WrapInternalf(err, errors.CodeInternal, "templating error")
+	}
+
+	for _, req := range requests {
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Authorization", fmt.Sprintf("GenieKey %s", apiKey))
+	}
+
+	return requests, true, nil
+}

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie_test.go

@@ -0,0 +1,333 @@
+package opsgenie
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestOpsGenieRetry(t *testing.T) {
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestOpsGenieRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKey:     config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestGettingOpsGegineApikeyFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "key"
+
+	f, err := os.CreateTemp(t.TempDir(), "opsgenie_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.OpsGenieConfig{
+			APIURL:     &config.URL{URL: u},
+			APIKeyFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestOpsGenie(t *testing.T) {
+	u, err := url.Parse("https://opsgenie/api")
+	if err != nil {
+		t.Fatalf("failed to parse URL: %v", err)
+	}
+	logger := promslog.NewNopLogger()
+	tmpl := test.CreateTmpl(t)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.OpsGenieConfig
+
+		expectedEmptyAlertBody string
+		expectedBody           string
+	}{
+		{
+			title: "config without details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with details",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName1 }}`,
+						Type: `{{ .CommonLabels.ResponderType1 }}`,
+					},
+					{
+						Name: `{{ .CommonLabels.ResponderName2 }}`,
+						Type: `{{ .CommonLabels.ResponderType2 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				Entity:     `{{ .CommonLabels.Entity }}`,
+				Actions:    `{{ .CommonLabels.Actions }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"EscalationA","type":"escalation"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1","entity":"test-domain","actions":["doThis","doThat"]}
+`,
+		},
+		{
+			title: "config with multiple teams",
+			cfg: &config.OpsGenieConfig{
+				NotifierConfig: config.NotifierConfig{
+					VSendResolved: true,
+				},
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+				Source:      `{{ .CommonLabels.Source }}`,
+				Details: map[string]string{
+					"Description": `adjusted {{ .CommonLabels.Description }}`,
+				},
+				Responders: []config.OpsGenieConfigResponder{
+					{
+						Name: `{{ .CommonLabels.ResponderName3 }}`,
+						Type: `{{ .CommonLabels.ResponderType3 }}`,
+					},
+				},
+				Tags:       `{{ .CommonLabels.Tags }}`,
+				Note:       `{{ .CommonLabels.Note }}`,
+				Priority:   `{{ .CommonLabels.Priority }}`,
+				APIKey:     `{{ .ExternalURL }}`,
+				APIURL:     &config.URL{URL: u},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			expectedEmptyAlertBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"","details":{"Description":"adjusted "},"source":""}
+`,
+			expectedBody: `{"alias":"6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b","message":"message","description":"description","details":{"Actions":"doThis,doThat","Description":"adjusted description","Entity":"test-domain","Message":"message","Note":"this is a note","Priority":"P1","ResponderName1":"TeamA","ResponderName2":"EscalationA","ResponderName3":"TeamA,TeamB","ResponderType1":"team","ResponderType2":"escalation","ResponderType3":"teams","Source":"http://prometheus","Tags":"tag1,tag2"},"source":"http://prometheus","responders":[{"name":"TeamA","type":"team"},{"name":"TeamB","type":"team"}],"tags":["tag1","tag2"],"note":"this is a note","priority":"P1"}
+`,
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			notifier, err := New(tc.cfg, tmpl, logger)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			expectedURL, _ := url.Parse("https://opsgenie/apiv2/alerts")
+
+			// Empty alert.
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+
+			req, retry, err := notifier.createRequests(ctx, alert1)
+			require.NoError(t, err)
+			require.Len(t, req, 1)
+			require.True(t, retry)
+			require.Equal(t, expectedURL, req[0].URL)
+			require.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization"))
+			require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0]))
+
+			// Fully defined alert.
+			alert2 := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"Message":        "message",
+						"Description":    "description",
+						"Source":         "http://prometheus",
+						"ResponderName1": "TeamA",
+						"ResponderType1": "team",
+						"ResponderName2": "EscalationA",
+						"ResponderType2": "escalation",
+						"ResponderName3": "TeamA,TeamB",
+						"ResponderType3": "teams",
+						"Tags":           "tag1,tag2",
+						"Note":           "this is a note",
+						"Priority":       "P1",
+						"Entity":         "test-domain",
+						"Actions":        "doThis,doThat",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			req, retry, err = notifier.createRequests(ctx, alert2)
+			require.NoError(t, err)
+			require.True(t, retry)
+			require.Len(t, req, 1)
+			require.Equal(t, tc.expectedBody, readBody(t, req[0]))
+
+			// Broken API Key Template.
+			tc.cfg.APIKey = "{{ kaput "
+			_, _, err = notifier.createRequests(ctx, alert2)
+			require.Error(t, err)
+			require.Equal(t, "template: :1: function \"kaput\" not defined", err.Error())
+		})
+	}
+}
+
+func TestOpsGenieWithUpdate(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		Message:      `{{ .CommonLabels.Message }}`,
+		Description:  `{{ .CommonLabels.Description }}`,
+		UpdateAlerts: true,
+		APIKey:       "test-api-key",
+		APIURL:       &config.URL{URL: u},
+		HTTPConfig:   &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+	alert := &types.Alert{
+		Alert: model.Alert{
+			StartsAt: time.Now(),
+			EndsAt:   time.Now().Add(time.Hour),
+			Labels: model.LabelSet{
+				"Message":     "new message",
+				"Description": "new description",
+			},
+		},
+	}
+	require.NoError(t, err)
+	requests, retry, err := notifierWithUpdate.createRequests(ctx, alert)
+	require.NoError(t, err)
+	require.True(t, retry)
+	require.Len(t, requests, 3)
+
+	body0 := readBody(t, requests[0])
+	body1 := readBody(t, requests[1])
+	body2 := readBody(t, requests[2])
+	key, _ := notify.ExtractGroupKey(ctx)
+	alias := key.Hash()
+
+	require.Equal(t, "https://test-opsgenie-url/v2/alerts", requests[0].URL.String())
+	require.NotEmpty(t, body0)
+
+	require.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias))
+	require.JSONEq(t, `{"message":"new message"}`, body1)
+	require.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias))
+	require.JSONEq(t, `{"description":"new description"}`, body2)
+}
+
+func TestOpsGenieApiKeyFile(t *testing.T) {
+	u, err := url.Parse("https://test-opsgenie-url")
+	require.NoError(t, err)
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+	opsGenieConfigWithUpdate := config.OpsGenieConfig{
+		APIKeyFile: `./api_key_file`,
+		APIURL:     &config.URL{URL: u},
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+	}
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+
+	require.NoError(t, err)
+	requests, _, err := notifierWithUpdate.createRequests(ctx)
+	require.NoError(t, err)
+	require.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
+}
+
+func readBody(t *testing.T, r *http.Request) string {
+	t.Helper()
+	body, err := io.ReadAll(r.Body)
+	require.NoError(t, err)
+	return string(body)
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty.go

@@ -0,0 +1,374 @@
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/alecthomas/units"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "pagerduty"
+)
+
+const (
+	maxEventSize int = 512000
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTc4-send-a-v1-event - 1024 characters or runes.
+	maxV1DescriptionLenRunes = 1024
+	// https://developer.pagerduty.com/docs/ZG9jOjExMDI5NTgx-send-an-alert-event - 1024 characters or runes.
+	maxV2SummaryLenRunes = 1024
+)
+
+// Notifier implements a Notifier for PagerDuty notifications.
+type Notifier struct {
+	conf    *config.PagerdutyConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	apiV1   string // for tests.
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new PagerDuty notifier.
+func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	n := &Notifier{conf: c, tmpl: t, logger: l, client: client}
+	if c.ServiceKey != "" || c.ServiceKeyFile != "" {
+		n.apiV1 = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+		// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v1-overview#api-response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusForbidden}, CustomDetailsFunc: errDetails}
+	} else {
+		// Retrying can solve the issue on 429 (rate limiting) and 5xx response codes.
+		// https://developer.pagerduty.com/docs/events-api-v2-overview#response-codes--retry-logic
+		n.retrier = &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}, CustomDetailsFunc: errDetails}
+	}
+	return n, nil
+}
+
+const (
+	pagerDutyEventTrigger = "trigger"
+	pagerDutyEventResolve = "resolve"
+)
+
+type pagerDutyMessage struct {
+	RoutingKey  string            `json:"routing_key,omitempty"`
+	ServiceKey  string            `json:"service_key,omitempty"`
+	DedupKey    string            `json:"dedup_key,omitempty"`
+	IncidentKey string            `json:"incident_key,omitempty"`
+	EventType   string            `json:"event_type,omitempty"`
+	Description string            `json:"description,omitempty"`
+	EventAction string            `json:"event_action"`
+	Payload     *pagerDutyPayload `json:"payload"`
+	Client      string            `json:"client,omitempty"`
+	ClientURL   string            `json:"client_url,omitempty"`
+	Details     map[string]any    `json:"details,omitempty"`
+	Images      []pagerDutyImage  `json:"images,omitempty"`
+	Links       []pagerDutyLink   `json:"links,omitempty"`
+}
+
+type pagerDutyLink struct {
+	HRef string `json:"href"`
+	Text string `json:"text"`
+}
+
+type pagerDutyImage struct {
+	Src  string `json:"src"`
+	Alt  string `json:"alt"`
+	Href string `json:"href"`
+}
+
+type pagerDutyPayload struct {
+	Summary       string         `json:"summary"`
+	Source        string         `json:"source"`
+	Severity      string         `json:"severity"`
+	Timestamp     string         `json:"timestamp,omitempty"`
+	Class         string         `json:"class,omitempty"`
+	Component     string         `json:"component,omitempty"`
+	Group         string         `json:"group,omitempty"`
+	CustomDetails map[string]any `json:"custom_details,omitempty"`
+}
+
+func (n *Notifier) encodeMessage(ctx context.Context, msg *pagerDutyMessage) (bytes.Buffer, error) {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return buf, errors.WrapInternalf(err, errors.CodeInternal, "failed to encode PagerDuty message")
+	}
+
+	if buf.Len() > maxEventSize {
+		truncatedMsg := fmt.Sprintf("Custom details have been removed because the original event exceeds the maximum size of %s", units.MetricBytes(maxEventSize).String())
+
+		if n.apiV1 != "" {
+			msg.Details = map[string]any{"error": truncatedMsg}
+		} else {
+			msg.Payload.CustomDetails = map[string]any{"error": truncatedMsg}
+		}
+
+		n.logger.WarnContext(ctx, "Truncated Details because message of size exceeds limit", "message_size", units.MetricBytes(buf.Len()).String(), "max_size", units.MetricBytes(maxEventSize).String())
+
+		buf.Reset()
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return buf, errors.WrapInternalf(err, errors.CodeInternal, "failed to encode PagerDuty message")
+		}
+	}
+
+	return buf, nil
+}
+
+func (n *Notifier) notifyV1(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated description", "key", key, "max_runes", maxV1DescriptionLenRunes)
+	}
+
+	serviceKey := string(n.conf.ServiceKey)
+	if serviceKey == "" {
+		content, fileErr := os.ReadFile(n.conf.ServiceKeyFile)
+		if fileErr != nil {
+			return false, errors.WrapInternalf(fileErr, errors.CodeInternal, "failed to read service key from file")
+		}
+		serviceKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		ServiceKey:  tmpl(serviceKey),
+		EventType:   eventType,
+		IncidentKey: key.Hash(),
+		Description: description,
+		Details:     details,
+	}
+
+	if eventType == pagerDutyEventTrigger {
+		msg.Client = tmpl(n.conf.Client)
+		msg.ClientURL = tmpl(n.conf.ClientURL)
+	}
+
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "failed to template PagerDuty v1 message")
+	}
+
+	// Ensure that the service key isn't empty after templating.
+	if msg.ServiceKey == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "service key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(ctx, msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &encodedMsg) //nolint:bodyclose
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "failed to post message to PagerDuty v1")
+	}
+	defer notify.Drain(resp)
+
+	return n.retrier.Check(resp.StatusCode, resp.Body)
+}
+
+func (n *Notifier) notifyV2(
+	ctx context.Context,
+	eventType string,
+	key notify.Key,
+	data *template.Data,
+	details map[string]any,
+) (bool, error) {
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.Severity == "" {
+		n.conf.Severity = "error"
+	}
+
+	summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV2SummaryLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated summary", "key", key, "max_runes", maxV2SummaryLenRunes)
+	}
+
+	routingKey := string(n.conf.RoutingKey)
+	if routingKey == "" {
+		content, fileErr := os.ReadFile(n.conf.RoutingKeyFile)
+		if fileErr != nil {
+			return false, errors.WrapInternalf(fileErr, errors.CodeInternal, "failed to read routing key from file")
+		}
+		routingKey = strings.TrimSpace(string(content))
+	}
+
+	msg := &pagerDutyMessage{
+		Client:      tmpl(n.conf.Client),
+		ClientURL:   tmpl(n.conf.ClientURL),
+		RoutingKey:  tmpl(routingKey),
+		EventAction: eventType,
+		DedupKey:    key.Hash(),
+		Images:      make([]pagerDutyImage, 0, len(n.conf.Images)),
+		Links:       make([]pagerDutyLink, 0, len(n.conf.Links)),
+		Payload: &pagerDutyPayload{
+			Summary:       summary,
+			Source:        tmpl(n.conf.Source),
+			Severity:      tmpl(n.conf.Severity),
+			CustomDetails: details,
+			Class:         tmpl(n.conf.Class),
+			Component:     tmpl(n.conf.Component),
+			Group:         tmpl(n.conf.Group),
+		},
+	}
+
+	for _, item := range n.conf.Images {
+		image := pagerDutyImage{
+			Src:  tmpl(item.Src),
+			Alt:  tmpl(item.Alt),
+			Href: tmpl(item.Href),
+		}
+
+		if image.Src != "" {
+			msg.Images = append(msg.Images, image)
+		}
+	}
+
+	for _, item := range n.conf.Links {
+		link := pagerDutyLink{
+			HRef: tmpl(item.Href),
+			Text: tmpl(item.Text),
+		}
+
+		if link.HRef != "" {
+			msg.Links = append(msg.Links, link)
+		}
+	}
+
+	if tmplErr != nil {
+		return false, errors.WrapInternalf(tmplErr, errors.CodeInternal, "failed to template PagerDuty v2 message")
+	}
+
+	// Ensure that the routing key isn't empty after templating.
+	if msg.RoutingKey == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "routing key cannot be empty")
+	}
+
+	encodedMsg, err := n.encodeMessage(ctx, msg)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &encodedMsg) //nolint:bodyclose
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "failed to post message to PagerDuty")
+	}
+	defer notify.Drain(resp)
+
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return retry, err
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With("group_key", key)
+
+	var (
+		alerts    = types.Alerts(as...)
+		data      = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		eventType = pagerDutyEventTrigger
+	)
+
+	if alerts.Status() == model.AlertResolved {
+		eventType = pagerDutyEventResolve
+	}
+
+	logger.DebugContext(ctx, "extracted group key", "event_type", eventType)
+
+	details, err := n.renderDetails(data)
+	if err != nil {
+		return false, errors.WrapInternalf(err, errors.CodeInternal, "failed to render details: %v", err)
+	}
+
+	if n.conf.Timeout > 0 {
+		nfCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeInternal, "configured pagerduty timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = nfCtx
+	}
+
+	nf := n.notifyV2
+	if n.apiV1 != "" {
+		nf = n.notifyV1
+	}
+	retry, err := nf(ctx, eventType, key, data, details)
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.WrapInternalf(err, errors.CodeInternal, "failed to notify PagerDuty: %v", context.Cause(ctx))
+		}
+		return retry, err
+	}
+	return retry, nil
+}
+
+func errDetails(status int, body io.Reader) string {
+	// See https://v2.developer.pagerduty.com/docs/trigger-events for the v1 events API.
+	// See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 for the v2 events API.
+	if status != http.StatusBadRequest || body == nil {
+		return ""
+	}
+	var pgr struct {
+		Status  string   `json:"status"`
+		Message string   `json:"message"`
+		Errors  []string `json:"errors"`
+	}
+	if err := json.NewDecoder(body).Decode(&pgr); err != nil {
+		return ""
+	}
+	return fmt.Sprintf("%s: %s", pgr.Message, strings.Join(pgr.Errors, ","))
+}
+
+func (n *Notifier) renderDetails(
+	data *template.Data,
+) (map[string]any, error) {
+	var (
+		tmplTextErr  error
+		tmplText     = notify.TmplText(n.tmpl, data, &tmplTextErr)
+		tmplTextFunc = func(tmpl string) (string, error) {
+			return tmplText(tmpl), tmplTextErr
+		}
+	)
+	var err error
+	rendered := make(map[string]any, len(n.conf.Details))
+	for k, v := range n.conf.Details {
+		rendered[k], err = template.DeepCopyWithTemplate(v, tmplTextFunc)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return rendered, nil
+}

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty_test.go

@@ -0,0 +1,879 @@
+package pagerduty
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestPagerDutyRetryV1(t *testing.T) {
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusForbidden)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "retryv1 - error on status %d", statusCode)
+	}
+}
+
+func TestPagerDutyRetryV2(t *testing.T) {
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
+	for statusCode, expected := range test.RetryTests(retryCodes) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "retryv2 - error on status %d", statusCode)
+	}
+}
+
+func TestPagerDutyRedactedURLV1(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "01234567890123456789012345678901"
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+	notifier.apiV1 = u.String()
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyRedactedURLV2(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	key := "01234567890123456789012345678901"
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			URL:        &config.URL{URL: u},
+			RoutingKey: config.Secret(key),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyV1ServiceKeyFromFile(t *testing.T) {
+	key := "01234567890123456789012345678901"
+	f, err := os.CreateTemp(t.TempDir(), "pagerduty_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			ServiceKeyFile: f.Name(),
+			HTTPConfig:     &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+	notifier.apiV1 = u.String()
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyV2RoutingKeyFromFile(t *testing.T) {
+	key := "01234567890123456789012345678901"
+	f, err := os.CreateTemp(t.TempDir(), "pagerduty_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(key)
+	require.NoError(t, err, "writing to temp file failed")
+
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.PagerdutyConfig{
+			URL:            &config.URL{URL: u},
+			RoutingKeyFile: f.Name(),
+			HTTPConfig:     &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key)
+}
+
+func TestPagerDutyTemplating(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		dec := json.NewDecoder(r.Body)
+		out := make(map[string]any)
+		err := dec.Decode(&out)
+		if err != nil {
+			panic(err)
+		}
+	}))
+	defer srv.Close()
+	u, _ := url.Parse(srv.URL)
+
+	for _, tc := range []struct {
+		title string
+		cfg   *config.PagerdutyConfig
+
+		retry  bool
+		errMsg string
+	}{
+		{
+			title: "full-blown legacy message",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Images: []config.PagerdutyImage{
+					{
+						Src:  "{{ .Status }}",
+						Alt:  "{{ .Status }}",
+						Href: "{{ .Status }}",
+					},
+				},
+				Links: []config.PagerdutyLink{
+					{
+						Href: "{{ .Status }}",
+						Text: "{{ .Status }}",
+					},
+				},
+				Details: map[string]any{
+					"firing":       `{{ .Alerts.Firing | toJson }}`,
+					"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+		},
+		{
+			title: "full-blown legacy message",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Images: []config.PagerdutyImage{
+					{
+						Src:  "{{ .Status }}",
+						Alt:  "{{ .Status }}",
+						Href: "{{ .Status }}",
+					},
+				},
+				Links: []config.PagerdutyLink{
+					{
+						Href: "{{ .Status }}",
+						Text: "{{ .Status }}",
+					},
+				},
+				Details: map[string]any{
+					"firing":       `{{ template "pagerduty.default.instances" .Alerts.Firing }}`,
+					"resolved":     `{{ template "pagerduty.default.instances" .Alerts.Resolved }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+		},
+		{
+			title: "nested details",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": map[string]any{
+								"firing":       `{{ .Alerts.Firing | toJson }}`,
+								"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+								"num_firing":   `{{ .Alerts.Firing | len }}`,
+								"num_resolved": `{{ .Alerts.Resolved | len }}`,
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			title: "nested details with template error",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": map[string]any{
+								"firing": `{{ template "pagerduty.default.instances" .Alerts.Firing`,
+							},
+						},
+					},
+				},
+			},
+			errMsg: "failed to render details: template: :1: unclosed action",
+		},
+		{
+			title: "details with templating errors",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Details: map[string]any{
+					"firing":       `{{ .Alerts.Firing | toJson`,
+					"resolved":     `{{ .Alerts.Resolved | toJson }}`,
+					"num_firing":   `{{ .Alerts.Firing | len }}`,
+					"num_resolved": `{{ .Alerts.Resolved | len }}`,
+				},
+			},
+			errMsg: "failed to render details: template: :1: unclosed action",
+		},
+		{
+			title: "v2 message with templating errors",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				Severity:   "{{ ",
+			},
+			errMsg: "failed to template",
+		},
+		{
+			title: "v1 message with templating errors",
+			cfg: &config.PagerdutyConfig{
+				ServiceKey: config.Secret("01234567890123456789012345678901"),
+				Client:     "{{ ",
+			},
+			errMsg: "failed to template",
+		},
+		{
+			title: "routing key cannot be empty",
+			cfg: &config.PagerdutyConfig{
+				RoutingKey: config.Secret(`{{ "" }}`),
+			},
+			errMsg: "routing key cannot be empty",
+		},
+		{
+			title: "service_key cannot be empty",
+			cfg: &config.PagerdutyConfig{
+				ServiceKey: config.Secret(`{{ "" }}`),
+			},
+			errMsg: "service key cannot be empty",
+		},
+	} {
+		t.Run(tc.title, func(t *testing.T) {
+			tc.cfg.URL = &config.URL{URL: u}
+			tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
+			pd, err := New(tc.cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+			if pd.apiV1 != "" {
+				pd.apiV1 = u.String()
+			}
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			ok, err := pd.Notify(ctx, []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels: model.LabelSet{
+							"lbl1": "val1",
+						},
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}...)
+			if tc.errMsg == "" {
+				require.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				if errors.Asc(err, errors.CodeInternal) {
+					_, _, errMsg, _, _, _ := errors.Unwrapb(err)
+					require.Contains(t, errMsg, tc.errMsg)
+				} else {
+					require.Contains(t, err.Error(), tc.errMsg)
+				}
+			}
+			require.Equal(t, tc.retry, ok)
+		})
+	}
+}
+
+func TestErrDetails(t *testing.T) {
+	for _, tc := range []struct {
+		status int
+		body   io.Reader
+
+		exp string
+	}{
+		{
+			status: http.StatusBadRequest,
+			body: bytes.NewBuffer([]byte(
+				`{"status":"invalid event","message":"Event object is invalid","errors":["Length of 'routing_key' is incorrect (should be 32 characters)"]}`,
+			)),
+
+			exp: "Length of 'routing_key' is incorrect",
+		},
+		{
+			status: http.StatusBadRequest,
+			body:   bytes.NewBuffer([]byte(`{"status"}`)),
+
+			exp: "",
+		},
+		{
+			status: http.StatusBadRequest,
+
+			exp: "",
+		},
+		{
+			status: http.StatusTooManyRequests,
+
+			exp: "",
+		},
+	} {
+		t.Run("", func(t *testing.T) {
+			err := errDetails(tc.status, tc.body)
+			require.Contains(t, err, tc.exp)
+		})
+	}
+}
+
+func TestEventSizeEnforcement(t *testing.T) {
+	bigDetailsV1 := map[string]any{
+		"firing": strings.Repeat("a", 513000),
+	}
+	bigDetailsV2 := map[string]any{
+		"firing": strings.Repeat("a", 513000),
+	}
+
+	// V1 Messages
+	msgV1 := &pagerDutyMessage{
+		ServiceKey: "01234567890123456789012345678901",
+		EventType:  "trigger",
+		Details:    bigDetailsV1,
+	}
+
+	notifierV1, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV1, err := notifierV1.encodeMessage(context.Background(), msgV1)
+	require.NoError(t, err)
+	require.Contains(t, encodedV1.String(), `"details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+
+	// V2 Messages
+	msgV2 := &pagerDutyMessage{
+		RoutingKey:  "01234567890123456789012345678901",
+		EventAction: "trigger",
+		Payload: &pagerDutyPayload{
+			CustomDetails: bigDetailsV2,
+		},
+	}
+
+	notifierV2, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV2, err := notifierV2.encodeMessage(context.Background(), msgV2)
+	require.NoError(t, err)
+	require.Contains(t, encodedV2.String(), `"custom_details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+}
+
+func TestPagerDutyEmptySrcHref(t *testing.T) {
+	type pagerDutyEvent struct {
+		RoutingKey  string           `json:"routing_key"`
+		EventAction string           `json:"event_action"`
+		DedupKey    string           `json:"dedup_key"`
+		Payload     pagerDutyPayload `json:"payload"`
+		Images      []pagerDutyImage
+		Links       []pagerDutyLink
+	}
+
+	images := []config.PagerdutyImage{
+		{
+			Src:  "",
+			Alt:  "Empty src",
+			Href: "https://example.com/",
+		},
+		{
+			Src:  "https://example.com/cat.jpg",
+			Alt:  "Empty href",
+			Href: "",
+		},
+		{
+			Src:  "https://example.com/cat.jpg",
+			Alt:  "",
+			Href: "https://example.com/",
+		},
+	}
+
+	links := []config.PagerdutyLink{
+		{
+			Href: "",
+			Text: "Empty href",
+		},
+		{
+			Href: "https://example.com/",
+			Text: "",
+		},
+	}
+
+	expectedImages := make([]pagerDutyImage, 0, len(images))
+	for _, image := range images {
+		if image.Src == "" {
+			continue
+		}
+		expectedImages = append(expectedImages, pagerDutyImage{
+			Src:  image.Src,
+			Alt:  image.Alt,
+			Href: image.Href,
+		})
+	}
+
+	expectedLinks := make([]pagerDutyLink, 0, len(links))
+	for _, link := range links {
+		if link.Href == "" {
+			continue
+		}
+		expectedLinks = append(expectedLinks, pagerDutyLink{
+			HRef: link.Href,
+			Text: link.Text,
+		})
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			decoder := json.NewDecoder(r.Body)
+			var event pagerDutyEvent
+			if err := decoder.Decode(&event); err != nil {
+				panic(err)
+			}
+
+			if event.RoutingKey == "" || event.EventAction == "" {
+				http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+				return
+			}
+
+			for _, image := range event.Images {
+				if image.Src == "" {
+					http.Error(w, "Event object is invalid: 'image src' is missing or blank", http.StatusBadRequest)
+					return
+				}
+			}
+
+			for _, link := range event.Links {
+				if link.HRef == "" {
+					http.Error(w, "Event object is invalid: 'link href' is missing or blank", http.StatusBadRequest)
+					return
+				}
+			}
+
+			require.Equal(t, expectedImages, event.Images)
+			require.Equal(t, expectedLinks, event.Links)
+		},
+	))
+	defer server.Close()
+
+	url, err := url.Parse(server.URL)
+	require.NoError(t, err)
+
+	pagerDutyConfig := config.PagerdutyConfig{
+		HTTPConfig: &commoncfg.HTTPClientConfig{},
+		RoutingKey: config.Secret("01234567890123456789012345678901"),
+		URL:        &config.URL{URL: url},
+		Images:     images,
+		Links:      links,
+	}
+
+	pagerDuty, err := New(&pagerDutyConfig, test.CreateTmpl(t), promslog.NewNopLogger())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "1")
+
+	_, err = pagerDuty.Notify(ctx, []*types.Alert{
+		{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"lbl1": "val1",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		},
+	}...)
+	require.NoError(t, err)
+}
+
+func TestPagerDutyTimeout(t *testing.T) {
+	type pagerDutyEvent struct {
+		RoutingKey  string           `json:"routing_key"`
+		EventAction string           `json:"event_action"`
+		DedupKey    string           `json:"dedup_key"`
+		Payload     pagerDutyPayload `json:"payload"`
+		Images      []pagerDutyImage
+		Links       []pagerDutyLink
+	}
+
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(
+				func(w http.ResponseWriter, r *http.Request) {
+					decoder := json.NewDecoder(r.Body)
+					var event pagerDutyEvent
+					if err := decoder.Decode(&event); err != nil {
+						panic(err)
+					}
+
+					if event.RoutingKey == "" || event.EventAction == "" {
+						http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+						return
+					}
+					time.Sleep(tt.latency)
+				},
+			))
+			defer srv.Close()
+			u, err := url.Parse(srv.URL)
+			require.NoError(t, err)
+
+			cfg := config.PagerdutyConfig{
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				URL:        &config.URL{URL: u},
+				Timeout:    tt.timeout,
+			}
+
+			pd, err := New(&cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+			alert := &types.Alert{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"lbl1": "val1",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = pd.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+func TestRenderDetails(t *testing.T) {
+	type args struct {
+		details map[string]any
+		data    *template.Data
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    map[string]any
+		wantErr bool
+	}{
+		{
+			name: "flat",
+			args: args{
+				details: map[string]any{
+					"a": "{{ .Status }}",
+					"b": "String",
+				},
+				data: &template.Data{
+					Status: "Flat",
+				},
+			},
+			want: map[string]any{
+				"a": "Flat",
+				"b": "String",
+			},
+			wantErr: false,
+		},
+		{
+			name: "flat error",
+			args: args{
+				details: map[string]any{
+					"a": "{{ .Status",
+				},
+				data: &template.Data{
+					Status: "Error",
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "nested",
+			args: args{
+				details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": "{{ .Status }}",
+							"d": "String",
+						},
+					},
+				},
+				data: &template.Data{
+					Status: "Nested",
+				},
+			},
+			want: map[string]any{
+				"a": map[string]any{
+					"b": map[string]any{
+						"c": "Nested",
+						"d": "String",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "nested error",
+			args: args{
+				details: map[string]any{
+					"a": map[string]any{
+						"b": map[string]any{
+							"c": "{{ .Status",
+						},
+					},
+				},
+				data: &template.Data{
+					Status: "Error",
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "alerts",
+			args: args{
+				details: map[string]any{
+					"alerts": map[string]any{
+						"firing":       "{{ .Alerts.Firing | toJson }}",
+						"resolved":     "{{ .Alerts.Resolved | toJson }}",
+						"num_firing":   "{{ len .Alerts.Firing }}",
+						"num_resolved": "{{ len .Alerts.Resolved }}",
+					},
+				},
+				data: &template.Data{
+					Alerts: template.Alerts{
+						{
+							Status: "firing",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Firing1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint1",
+							GeneratorURL: "http://generator1",
+							StartsAt:     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "firing",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Firing2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint2",
+							GeneratorURL: "http://generator2",
+							StartsAt:     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "resolved",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Resolved1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint3",
+							GeneratorURL: "http://generator3",
+							StartsAt:     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+						{
+							Status: "resolved",
+							Annotations: template.KV{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							Labels: template.KV{
+								"alertname": "Resolved2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							Fingerprint:  "fingerprint4",
+							GeneratorURL: "http://generator4",
+							StartsAt:     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC),
+							EndsAt:       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC),
+						},
+					},
+				},
+			},
+			want: map[string]any{
+				"alerts": map[string]any{
+					"firing": []any{
+						map[string]any{
+							"status": "firing",
+							"labels": map[string]any{
+								"alertname": "Firing1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint1",
+							"generatorURL": "http://generator1",
+						},
+						map[string]any{
+							"status": "firing",
+							"labels": map[string]any{
+								"alertname": "Firing2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint2",
+							"generatorURL": "http://generator2",
+						},
+					},
+					"resolved": []any{
+						map[string]any{
+							"status": "resolved",
+							"labels": map[string]any{
+								"alertname": "Resolved1",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2001, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint3",
+							"generatorURL": "http://generator3",
+						},
+						map[string]any{
+							"status": "resolved",
+							"labels": map[string]any{
+								"alertname": "Resolved2",
+								"label1":    "value1",
+								"label2":    "value2",
+							},
+							"annotations": map[string]any{
+								"annotation1": "value1",
+								"annotation2": "value2",
+							},
+							"startsAt":     time.Date(2002, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"endsAt":       time.Date(2002, time.January, 1, 1, 0, 0, 0, time.UTC).Format(time.RFC3339),
+							"fingerprint":  "fingerprint4",
+							"generatorURL": "http://generator4",
+						},
+					},
+					"num_firing":   2,
+					"num_resolved": 2,
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			n := &Notifier{
+				conf: &config.PagerdutyConfig{
+					Details: tt.args.details,
+				},
+				tmpl: test.CreateTmpl(t),
+			}
+			got, err := n.renderDetails(tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("renderDetails() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			require.Equal(t, tt.want, got)
+		})
+	}
+}

pkg/alertmanager/alertmanagernotify/receiver.go

@@ -2,8 +2,14 @@ package alertmanagernotify
 
 import (
 	"log/slog"
+	"slices"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/email"
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/msteamsv2"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/opsgenie"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/pagerduty"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/slack"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/webhook"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/prometheus/alertmanager/config/receiver"
 	"github.com/prometheus/alertmanager/notify"
@@ -11,6 +17,15 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+var customNotifierIntegrations = []string{
+	webhook.Integration,
+	email.Integration,
+	pagerduty.Integration,
+	opsgenie.Integration,
+	slack.Integration,
+	msteamsv2.Integration,
+}
+
 func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error) {
 	upstreamIntegrations, err := receiver.BuildReceiverIntegrations(nc, tmpl, logger)
 	if err != nil {
@@ -31,14 +46,29 @@ func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Templ
 	)
 
 	for _, integration := range upstreamIntegrations {
-		// skip upstream msteamsv2 integration
-		if integration.Name() != "msteamsv2" {
+		// skip upstream integration if we support custom integration for it
+		if !slices.Contains(customNotifierIntegrations, integration.Name()) {
 			integrations = append(integrations, integration)
 		}
 	}
 
+	for i, c := range nc.WebhookConfigs {
+		add(webhook.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) })
+	}
+	for i, c := range nc.EmailConfigs {
+		add(email.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil })
+	}
+	for i, c := range nc.PagerdutyConfigs {
+		add(pagerduty.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) })
+	}
+	for i, c := range nc.OpsGenieConfigs {
+		add(opsgenie.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) })
+	}
+	for i, c := range nc.SlackConfigs {
+		add(slack.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) })
+	}
 	for i, c := range nc.MSTeamsV2Configs {
-		add("msteamsv2", i, c, func(l *slog.Logger) (notify.Notifier, error) {
+		add(msteamsv2.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
 			return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l)
 		})
 	}

pkg/alertmanager/alertmanagernotify/slack/slack.go

@@ -0,0 +1,291 @@
+package slack
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "slack"
+)
+
+// https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
+const maxTitleLenRunes = 1024
+
+// Notifier implements a Notifier for Slack notifications.
+type Notifier struct {
+	conf    *config.SlackConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+
+	postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
+}
+
+// New returns a new Slack notification handler.
+func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Notifier{
+		conf:         c,
+		tmpl:         t,
+		logger:       l,
+		client:       client,
+		retrier:      &notify.Retrier{},
+		postJSONFunc: notify.PostJSON,
+	}, nil
+}
+
+// request is the request for sending a slack notification.
+type request struct {
+	Channel     string       `json:"channel,omitempty"`
+	Username    string       `json:"username,omitempty"`
+	IconEmoji   string       `json:"icon_emoji,omitempty"`
+	IconURL     string       `json:"icon_url,omitempty"`
+	LinkNames   bool         `json:"link_names,omitempty"`
+	Text        string       `json:"text,omitempty"`
+	Attachments []attachment `json:"attachments"`
+}
+
+// attachment is used to display a richly-formatted message block.
+type attachment struct {
+	Title      string               `json:"title,omitempty"`
+	TitleLink  string               `json:"title_link,omitempty"`
+	Pretext    string               `json:"pretext,omitempty"`
+	Text       string               `json:"text"`
+	Fallback   string               `json:"fallback"`
+	CallbackID string               `json:"callback_id"`
+	Fields     []config.SlackField  `json:"fields,omitempty"`
+	Actions    []config.SlackAction `json:"actions,omitempty"`
+	ImageURL   string               `json:"image_url,omitempty"`
+	ThumbURL   string               `json:"thumb_url,omitempty"`
+	Footer     string               `json:"footer"`
+	Color      string               `json:"color,omitempty"`
+	MrkdwnIn   []string             `json:"mrkdwn_in,omitempty"`
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
+	var err error
+
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+	logger := n.logger.With("group_key", key)
+	logger.DebugContext(ctx, "extracted group key")
+
+	var (
+		data     = notify.GetTemplateData(ctx, n.tmpl, as, logger)
+		tmplText = notify.TmplText(n.tmpl, data, &err)
+	)
+
+	titleTmpl, bodyTmpl := alertmanagertemplate.ExtractTemplatesFromAnnotations(as)
+	expanded, expandErr := alertmanagertemplate.ExpandAlertTemplates(ctx, n.tmpl, alertmanagertemplate.TemplateInput{
+		TitleTemplate:        titleTmpl,
+		BodyTemplate:         bodyTmpl,
+		DefaultTitleTemplate: n.conf.Title,
+		DefaultBodyTemplate:  n.conf.Text,
+	}, as, logger)
+	if expandErr != nil {
+		return false, expandErr
+	}
+
+	var markdownIn []string
+
+	if len(n.conf.MrkdwnIn) == 0 {
+		markdownIn = []string{"fallback", "pretext", "text"}
+	} else {
+		markdownIn = n.conf.MrkdwnIn
+	}
+
+	expandedTitle, truncated := notify.TruncateInRunes(expanded.Title, maxTitleLenRunes)
+	if truncated {
+		logger.WarnContext(ctx, "Truncated title", "max_runes", maxTitleLenRunes)
+	}
+	att := &attachment{
+		Title:      expandedTitle,
+		TitleLink:  tmplText(n.conf.TitleLink),
+		Pretext:    tmplText(n.conf.Pretext),
+		Text:       expanded.Body,
+		Fallback:   tmplText(n.conf.Fallback),
+		CallbackID: tmplText(n.conf.CallbackID),
+		ImageURL:   tmplText(n.conf.ImageURL),
+		ThumbURL:   tmplText(n.conf.ThumbURL),
+		Footer:     tmplText(n.conf.Footer),
+		Color:      tmplText(n.conf.Color),
+		MrkdwnIn:   markdownIn,
+	}
+
+	numFields := len(n.conf.Fields)
+	if numFields > 0 {
+		fields := make([]config.SlackField, numFields)
+		for index, field := range n.conf.Fields {
+			// Check if short was defined for the field otherwise fallback to the global setting
+			var short bool
+			if field.Short != nil {
+				short = *field.Short
+			} else {
+				short = n.conf.ShortFields
+			}
+
+			// Rebuild the field by executing any templates and setting the new value for short
+			fields[index] = config.SlackField{
+				Title: tmplText(field.Title),
+				Value: tmplText(field.Value),
+				Short: &short,
+			}
+		}
+		att.Fields = fields
+	}
+
+	numActions := len(n.conf.Actions)
+	if numActions > 0 {
+		actions := make([]config.SlackAction, numActions)
+		for index, action := range n.conf.Actions {
+			slackAction := config.SlackAction{
+				Type:  tmplText(action.Type),
+				Text:  tmplText(action.Text),
+				URL:   tmplText(action.URL),
+				Style: tmplText(action.Style),
+				Name:  tmplText(action.Name),
+				Value: tmplText(action.Value),
+			}
+
+			if action.ConfirmField != nil {
+				slackAction.ConfirmField = &config.SlackConfirmationField{
+					Title:       tmplText(action.ConfirmField.Title),
+					Text:        tmplText(action.ConfirmField.Text),
+					OkText:      tmplText(action.ConfirmField.OkText),
+					DismissText: tmplText(action.ConfirmField.DismissText),
+				}
+			}
+
+			actions[index] = slackAction
+		}
+		att.Actions = actions
+	}
+
+	req := &request{
+		Channel:     tmplText(n.conf.Channel),
+		Username:    tmplText(n.conf.Username),
+		IconEmoji:   tmplText(n.conf.IconEmoji),
+		IconURL:     tmplText(n.conf.IconURL),
+		LinkNames:   n.conf.LinkNames,
+		Text:        tmplText(n.conf.MessageText),
+		Attachments: []attachment{*att},
+	}
+	if err != nil {
+		return false, err
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(req); err != nil {
+		return false, err
+	}
+
+	var u string
+	if n.conf.APIURL != nil {
+		u = n.conf.APIURL.String()
+	} else {
+		content, err := os.ReadFile(n.conf.APIURLFile)
+		if err != nil {
+			return false, err
+		}
+		u = strings.TrimSpace(string(content))
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeInternal, "configured slack timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := n.postJSONFunc(ctx, n.client, u, &buf) //nolint:bodyclose
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.NewInternalf(errors.CodeInternal, "failed to post JSON to slack: %v", context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	// Use a retrier to generate an error message for non-200 responses and
+	// classify them as retriable or not.
+	retry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		err = errors.NewInternalf(errors.CodeInternal, "channel %q: %v", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+
+	// Slack web API might return errors with a 200 response code.
+	// https://slack.dev/node-slack-sdk/web-api#handle-errors
+	retry, err = checkResponseError(resp)
+	if err != nil {
+		err = errors.NewInternalf(errors.CodeInternal, "channel %q: %v", req.Channel, err)
+		return retry, notify.NewErrorWithReason(notify.ClientErrorReason, err)
+	}
+
+	return retry, nil
+}
+
+// checkResponseError parses out the error message from Slack API response.
+func checkResponseError(resp *http.Response) (bool, error) {
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return true, errors.WrapInternalf(err, errors.CodeInternal, "could not read response body")
+	}
+
+	if strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") {
+		return checkJSONResponseError(body)
+	}
+	return checkTextResponseError(body)
+}
+
+// checkTextResponseError classifies plaintext responses from Slack.
+// A plaintext (non-JSON) response is successful if it's a string "ok".
+// This is typically a response for an Incoming Webhook
+// (https://api.slack.com/messaging/webhooks#handling_errors)
+func checkTextResponseError(body []byte) (bool, error) {
+	if !bytes.Equal(body, []byte("ok")) {
+		return false, errors.NewInternalf(errors.CodeInternal, "received an error response from Slack: %s", string(body))
+	}
+	return false, nil
+}
+
+// checkJSONResponseError classifies JSON responses from Slack.
+func checkJSONResponseError(body []byte) (bool, error) {
+	// response is for parsing out errors from the JSON response.
+	type response struct {
+		OK    bool   `json:"ok"`
+		Error string `json:"error"`
+	}
+
+	var data response
+	if err := json.Unmarshal(body, &data); err != nil {
+		return true, errors.NewInternalf(errors.CodeInternal, "could not unmarshal JSON response %q: %v", string(body), err)
+	}
+	if !data.OK {
+		return false, errors.NewInternalf(errors.CodeInternal, "error response from Slack: %s", data.Error)
+	}
+	return false, nil
+}

pkg/alertmanager/alertmanagernotify/slack/slack_test.go

@@ -0,0 +1,339 @@
+package slack
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestSlackRetry(t *testing.T) {
+	notifier, err := New(
+		&config.SlackConfig{
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+		actual, _ := notifier.retrier.Check(statusCode, nil)
+		require.Equal(t, expected, actual, "error on status %d", statusCode)
+	}
+}
+
+func TestSlackRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURL:     &config.SecretURL{URL: u},
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestGettingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String())
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestTrimmingSlackURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "slack_test_newline")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.SlackConfig{
+			APIURLFile: f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestNotifier_Notify_WithReason(t *testing.T) {
+	tests := []struct {
+		name           string
+		statusCode     int
+		responseBody   string
+		expectedReason notify.Reason
+		expectedErr    string
+		expectedRetry  bool
+		noError        bool
+	}{
+		{
+			name:           "with a 4xx status code",
+			statusCode:     http.StatusUnauthorized,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 401",
+		},
+		{
+			name:           "with a 5xx status code",
+			statusCode:     http.StatusInternalServerError,
+			expectedReason: notify.ServerErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "unexpected status code 500",
+		},
+		{
+			name:           "with a 3xx status code",
+			statusCode:     http.StatusTemporaryRedirect,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 307",
+		},
+		{
+			name:           "with a 1xx status code",
+			statusCode:     http.StatusSwitchingProtocols,
+			expectedReason: notify.DefaultReason,
+			expectedRetry:  false,
+			expectedErr:    "unexpected status code 101",
+		},
+		{
+			name:           "2xx response with invalid JSON",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"not valid json"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  true,
+			expectedErr:    "could not unmarshal",
+		},
+		{
+			name:           "2xx response with a JSON error",
+			statusCode:     http.StatusOK,
+			responseBody:   `{"ok":false,"error":"error_message"}`,
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: error_message",
+		},
+		{
+			name:           "2xx response with a plaintext error",
+			statusCode:     http.StatusOK,
+			responseBody:   "no_channel",
+			expectedReason: notify.ClientErrorReason,
+			expectedRetry:  false,
+			expectedErr:    "error response from Slack: no_channel",
+		},
+		{
+			name:         "successful JSON response",
+			statusCode:   http.StatusOK,
+			responseBody: `{"ok":true}`,
+			noError:      true,
+		},
+		{
+			name:         "successful plaintext response",
+			statusCode:   http.StatusOK,
+			responseBody: "ok",
+			noError:      true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			apiurl, _ := url.Parse("https://slack.com/post.Message")
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: apiurl},
+					Channel:        "channelname",
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				resp := httptest.NewRecorder()
+				if strings.HasPrefix(tt.responseBody, "{") {
+					resp.Header().Add("Content-Type", "application/json; charset=utf-8")
+				}
+				resp.WriteHeader(tt.statusCode)
+				_, _ = resp.WriteString(tt.responseBody)
+				return resp.Result(), nil
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert1 := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			retry, err := notifier.Notify(ctx, alert1)
+			require.Equal(t, tt.expectedRetry, retry)
+			if tt.noError {
+				require.NoError(t, err)
+			} else {
+				var reasonError *notify.ErrorWithReason
+				require.ErrorAs(t, err, &reasonError)
+				require.Equal(t, tt.expectedReason, reasonError.Reason)
+				require.Contains(t, err.Error(), tt.expectedErr)
+				require.Contains(t, err.Error(), "channelname")
+			}
+		})
+	}
+}
+
+func TestSlackTimeout(t *testing.T) {
+	tests := map[string]struct {
+		latency time.Duration
+		timeout time.Duration
+		wantErr bool
+	}{
+		"success": {latency: 100 * time.Millisecond, timeout: 120 * time.Millisecond, wantErr: false},
+		"error":   {latency: 100 * time.Millisecond, timeout: 80 * time.Millisecond, wantErr: true},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			u, _ := url.Parse("https://slack.com/post.Message")
+			notifier, err := New(
+				&config.SlackConfig{
+					NotifierConfig: config.NotifierConfig{},
+					HTTPConfig:     &commoncfg.HTTPClientConfig{},
+					APIURL:         &config.SecretURL{URL: u},
+					Channel:        "channelname",
+					Timeout:        tt.timeout,
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
+				select {
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				case <-time.After(tt.latency):
+					resp := httptest.NewRecorder()
+					resp.Header().Set("Content-Type", "application/json; charset=utf-8")
+					resp.WriteHeader(http.StatusOK)
+					_, _ = resp.WriteString(`{"ok":true}`)
+
+					return resp.Result(), nil
+				}
+			}
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "1")
+
+			alert := &types.Alert{
+				Alert: model.Alert{
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			}
+			_, err = notifier.Notify(ctx, alert)
+			require.Equal(t, tt.wantErr, err != nil)
+		})
+	}
+}
+
+func TestSlackMessageField(t *testing.T) {
+	// 1. Setup a fake Slack server
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatal(err)
+		}
+
+		// 2. VERIFY: Top-level text exists
+		if body["text"] != "My Top Level Message" {
+			t.Errorf("Expected top-level 'text' to be 'My Top Level Message', got %v", body["text"])
+		}
+
+		// 3. VERIFY: Old attachments still exist
+		attachments, ok := body["attachments"].([]any)
+		if !ok || len(attachments) == 0 {
+			t.Errorf("Expected attachments to exist")
+		} else {
+			first := attachments[0].(map[string]any)
+			if first["title"] != "Old Attachment Title" {
+				t.Errorf("Expected attachment title 'Old Attachment Title', got %v", first["title"])
+			}
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"ok": true}`))
+	}))
+	defer server.Close()
+
+	// 4. Configure Notifier with BOTH new and old fields
+	u, _ := url.Parse(server.URL)
+	conf := &config.SlackConfig{
+		APIURL:      &config.SecretURL{URL: u},
+		MessageText: "My Top Level Message", // Your NEW field
+		Title:       "Old Attachment Title", // An OLD field
+		Channel:     "#test-channel",
+		HTTPConfig:  &commoncfg.HTTPClientConfig{},
+	}
+
+	tmpl, err := template.FromGlobs([]string{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpl.ExternalURL = u
+
+	logger := slog.New(slog.DiscardHandler)
+	notifier, err := New(conf, tmpl, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group-key")
+
+	if _, err := notifier.Notify(ctx); err != nil {
+		t.Fatal("Notify failed:", err)
+	}
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook.go

@@ -0,0 +1,136 @@
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	commoncfg "github.com/prometheus/common/config"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+const (
+	Integration = "webhook"
+)
+
+// Notifier implements a Notifier for generic webhooks.
+type Notifier struct {
+	conf    *config.WebhookConfig
+	tmpl    *template.Template
+	logger  *slog.Logger
+	client  *http.Client
+	retrier *notify.Retrier
+}
+
+// New returns a new Webhook.
+func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+	client, err := notify.NewClientWithTracing(*conf.HTTPConfig, Integration, httpOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return &Notifier{
+		conf:   conf,
+		tmpl:   t,
+		logger: l,
+		client: client,
+		// Webhooks are assumed to respond with 2xx response codes on a successful
+		// request and 5xx response codes are assumed to be recoverable.
+		retrier: &notify.Retrier{},
+	}, nil
+}
+
+// Message defines the JSON object send to webhook endpoints.
+type Message struct {
+	*template.Data
+
+	// The protocol version.
+	Version         string `json:"version"`
+	GroupKey        string `json:"groupKey"`
+	TruncatedAlerts uint64 `json:"truncatedAlerts"`
+}
+
+func truncateAlerts(maxAlerts uint64, alerts []*types.Alert) ([]*types.Alert, uint64) {
+	if maxAlerts != 0 && uint64(len(alerts)) > maxAlerts {
+		return alerts[:maxAlerts], uint64(len(alerts)) - maxAlerts
+	}
+
+	return alerts, 0
+}
+
+// Notify implements the Notifier interface.
+func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
+	alerts, numTruncated := truncateAlerts(n.conf.MaxAlerts, alerts)
+	data := notify.GetTemplateData(ctx, n.tmpl, alerts, n.logger)
+
+	groupKey, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	logger := n.logger.With("group_key", groupKey)
+	logger.DebugContext(ctx, "extracted group key")
+
+	msg := &Message{
+		Version:         "4",
+		Data:            data,
+		GroupKey:        groupKey.String(),
+		TruncatedAlerts: numTruncated,
+	}
+
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return false, err
+	}
+
+	var url string
+	var tmplErr error
+	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
+
+	if n.conf.URL != "" {
+		url = tmpl(string(n.conf.URL))
+	} else {
+		content, err := os.ReadFile(n.conf.URLFile)
+		if err != nil {
+			return false, errors.WrapInternalf(err, errors.CodeInternal, "read url_file")
+		}
+		url = tmpl(strings.TrimSpace(string(content)))
+	}
+
+	if tmplErr != nil {
+		return false, errors.NewInternalf(errors.CodeInternal, "failed to template webhook URL: %v", tmplErr)
+	}
+
+	if url == "" {
+		return false, errors.NewInternalf(errors.CodeInternal, "webhook URL is empty after templating")
+	}
+
+	if n.conf.Timeout > 0 {
+		postCtx, cancel := context.WithTimeoutCause(ctx, n.conf.Timeout, errors.NewInternalf(errors.CodeInternal, "configured webhook timeout reached (%s)", n.conf.Timeout))
+		defer cancel()
+		ctx = postCtx
+	}
+
+	resp, err := notify.PostJSON(ctx, n.client, url, &buf) //nolint:bodyclose
+	if err != nil {
+		if ctx.Err() != nil {
+			err = errors.NewInternalf(errors.CodeInternal, "failed to post JSON to webhook: %v", context.Cause(ctx))
+		}
+		return true, notify.RedactURL(err)
+	}
+	defer notify.Drain(resp)
+
+	shouldRetry, err := n.retrier.Check(resp.StatusCode, resp.Body)
+	if err != nil {
+		return shouldRetry, notify.NewErrorWithReason(notify.GetFailureReasonFromStatusCode(resp.StatusCode), err)
+	}
+	return shouldRetry, err
+}

pkg/alertmanager/alertmanagernotify/webhook/webhook_test.go

@@ -0,0 +1,214 @@
+package webhook
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+	"time"
+
+	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/config"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/types"
+)
+
+func TestWebhookRetry(t *testing.T) {
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL("http://example.com"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	if err != nil {
+		require.NoError(t, err)
+	}
+
+	t.Run("test retry status code", func(t *testing.T) {
+		for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
+			actual, _ := notifier.retrier.Check(statusCode, nil)
+			require.Equal(t, expected, actual, "error on status %d", statusCode)
+		}
+	})
+
+	t.Run("test retry error details", func(t *testing.T) {
+		for _, tc := range []struct {
+			status int
+			body   io.Reader
+
+			exp string
+		}{
+			{
+				status: http.StatusBadRequest,
+				body: bytes.NewBuffer([]byte(
+					`{"status":"invalid event"}`,
+				)),
+
+				exp: fmt.Sprintf(`unexpected status code %d: {"status":"invalid event"}`, http.StatusBadRequest),
+			},
+			{
+				status: http.StatusBadRequest,
+
+				exp: fmt.Sprintf(`unexpected status code %d`, http.StatusBadRequest),
+			},
+		} {
+			t.Run("", func(t *testing.T) {
+				_, err = notifier.retrier.Check(tc.status, tc.body)
+				require.Equal(t, tc.exp, err.Error())
+			})
+		}
+	})
+}
+
+func TestWebhookTruncateAlerts(t *testing.T) {
+	alerts := make([]*types.Alert, 10)
+
+	truncatedAlerts, numTruncated := truncateAlerts(0, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(4, alerts)
+	require.Len(t, truncatedAlerts, 4)
+	require.EqualValues(t, 6, numTruncated)
+
+	truncatedAlerts, numTruncated = truncateAlerts(100, alerts)
+	require.Len(t, truncatedAlerts, 10)
+	require.EqualValues(t, 0, numTruncated)
+}
+
+func TestWebhookRedactedURL(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	secret := "secret"
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL(u.String()),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, secret)
+}
+
+func TestWebhookReadingURLFromFile(t *testing.T) {
+	ctx, u, fn := test.GetContextWithCancelingURL()
+	defer fn()
+
+	f, err := os.CreateTemp(t.TempDir(), "webhook_url")
+	require.NoError(t, err, "creating temp file failed")
+	_, err = f.WriteString(u.String() + "\n")
+	require.NoError(t, err, "writing to temp file failed")
+
+	notifier, err := New(
+		&config.WebhookConfig{
+			URLFile:    f.Name(),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		promslog.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, u.String())
+}
+
+func TestWebhookURLTemplating(t *testing.T) {
+	var calledURL string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calledURL = r.URL.Path
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	tests := []struct {
+		name           string
+		url            string
+		groupLabels    model.LabelSet
+		alertLabels    model.LabelSet
+		expectError    bool
+		expectedErrMsg string
+		expectedPath   string
+	}{
+		{
+			name:         "templating with alert labels",
+			url:          srv.URL + "/{{ .GroupLabels.alertname }}/{{ .CommonLabels.severity }}",
+			groupLabels:  model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:  model.LabelSet{"alertname": "TestAlert", "severity": "critical"},
+			expectError:  false,
+			expectedPath: "/TestAlert/critical",
+		},
+		{
+			name:           "invalid template field",
+			url:            srv.URL + "/{{ .InvalidField }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "failed to template webhook URL",
+		},
+		{
+			name:           "template renders to empty string",
+			url:            "{{ if .CommonLabels.nonexistent }}http://example.com{{ end }}",
+			groupLabels:    model.LabelSet{"alertname": "TestAlert"},
+			alertLabels:    model.LabelSet{"alertname": "TestAlert"},
+			expectError:    true,
+			expectedErrMsg: "webhook URL is empty after templating",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			calledURL = "" // Reset for each test
+
+			notifier, err := New(
+				&config.WebhookConfig{
+					URL:        config.SecretTemplateURL(tc.url),
+					HTTPConfig: &commoncfg.HTTPClientConfig{},
+				},
+				test.CreateTmpl(t),
+				promslog.NewNopLogger(),
+			)
+			require.NoError(t, err)
+
+			ctx := context.Background()
+			ctx = notify.WithGroupKey(ctx, "test-group")
+			if tc.groupLabels != nil {
+				ctx = notify.WithGroupLabels(ctx, tc.groupLabels)
+			}
+
+			alerts := []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels:   tc.alertLabels,
+						StartsAt: time.Now(),
+						EndsAt:   time.Now().Add(time.Hour),
+					},
+				},
+			}
+
+			_, err = notifier.Notify(ctx, alerts...)
+
+			if tc.expectError {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.expectedErrMsg)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedPath, calledURL)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagerserver/dispatcher.go

@@ -95,7 +95,7 @@ func (d *Dispatcher) Run() {
 	d.ctx, d.cancel = context.WithCancel(context.Background())
 	d.mtx.Unlock()
 
-	d.run(d.alerts.Subscribe())
+	d.run(d.alerts.Subscribe(fmt.Sprintf("dispatcher-%s", d.orgID)))
 	close(d.done)
 }
 
@@ -107,14 +107,15 @@ func (d *Dispatcher) run(it provider.AlertIterator) {
 
 	for {
 		select {
-		case alert, ok := <-it.Next():
-			if !ok {
+		case alertWrapper, ok := <-it.Next():
+			if !ok || alertWrapper == nil {
 				// Iterator exhausted for some reason.
 				if err := it.Err(); err != nil {
 					d.logger.ErrorContext(d.ctx, "Error on alert update", "err", err)
 				}
 				return
 			}
+			alert := alertWrapper.Data
 
 			d.logger.DebugContext(d.ctx, "SigNoz Custom Dispatcher: Received alert", "alert", alert)
 

pkg/alertmanager/alertmanagerserver/distpatcher_test.go

@@ -365,7 +365,7 @@ route:
 	logger := providerSettings.Logger
 	route := dispatch.NewRoute(conf.Route, nil)
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
-	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, nil, logger, nil)
+	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -496,7 +496,7 @@ route:
 		err := nfManager.SetNotificationConfig(orgId, ruleID, &config)
 		require.NoError(t, err)
 	}
-	err = alerts.Put(inputAlerts...)
+	err = alerts.Put(ctx, inputAlerts...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -638,7 +638,7 @@ route:
 	logger := providerSettings.Logger
 	route := dispatch.NewRoute(conf.Route, nil)
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
-	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, nil, logger, nil)
+	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -798,7 +798,7 @@ route:
 		err := nfManager.SetNotificationConfig(orgId, ruleID, &config)
 		require.NoError(t, err)
 	}
-	err = alerts.Put(inputAlerts...)
+	err = alerts.Put(ctx, inputAlerts...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -897,7 +897,7 @@ route:
 	logger := providerSettings.Logger
 	route := dispatch.NewRoute(conf.Route, nil)
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
-	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, nil, logger, nil)
+	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1028,7 +1028,7 @@ route:
 		err := nfManager.SetNotificationConfig(orgId, ruleID, &config)
 		require.NoError(t, err)
 	}
-	err = alerts.Put(inputAlerts...)
+	err = alerts.Put(ctx, inputAlerts...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1159,7 +1159,7 @@ func newAlert(labels model.LabelSet) *alertmanagertypes.Alert {
 func TestDispatcherRace(t *testing.T) {
 	logger := promslog.NewNopLogger()
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
-	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, nil, logger, nil)
+	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1175,6 +1175,7 @@ func TestDispatcherRace(t *testing.T) {
 }
 
 func TestDispatcherRaceOnFirstAlertNotDeliveredWhenGroupWaitIsZero(t *testing.T) {
+	ctx := context.Background()
 	const numAlerts = 5000
 	confData := `receivers:
 - name: 'slack'
@@ -1194,7 +1195,7 @@ route:
 	providerSettings := createTestProviderSettings()
 	logger := providerSettings.Logger
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
-	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, nil, logger, nil)
+	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1247,7 +1248,7 @@ route:
 	for i := 0; i < numAlerts; i++ {
 		ruleId := fmt.Sprintf("Alert_%d", i)
 		alert := newAlert(model.LabelSet{"ruleId": model.LabelValue(ruleId)})
-		require.NoError(t, alerts.Put(alert))
+		require.NoError(t, alerts.Put(ctx, alert))
 	}
 
 	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
@@ -1265,7 +1266,7 @@ func TestDispatcher_DoMaintenance(t *testing.T) {
 	r := prometheus.NewRegistry()
 	marker := alertmanagertypes.NewMarker(r)
 
-	alerts, err := mem.NewAlerts(context.Background(), marker, time.Minute, nil, promslog.NewNopLogger(), nil)
+	alerts, err := mem.NewAlerts(context.Background(), marker, time.Minute, 0, nil, promslog.NewNopLogger(), prometheus.NewRegistry(), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1370,7 +1371,7 @@ route:
 			logger := providerSettings.Logger
 			route := dispatch.NewRoute(conf.Route, nil)
 			marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
-			alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, nil, logger, nil)
+			alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
 			if err != nil {
 				t.Fatal(err)
 			}

pkg/alertmanager/alertmanagerserver/server.go

@@ -190,7 +190,7 @@ func New(ctx context.Context, logger *slog.Logger, registry prometheus.Registere
 		})
 	}()
 
-	server.alerts, err = mem.NewAlerts(ctx, server.marker, server.srvConfig.Alerts.GCInterval, nil, server.logger, signozRegisterer)
+	server.alerts, err = mem.NewAlerts(ctx, server.marker, server.srvConfig.Alerts.GCInterval, 0, nil, server.logger, signozRegisterer, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -203,15 +203,15 @@ func New(ctx context.Context, logger *slog.Logger, registry prometheus.Registere
 
 func (server *Server) GetAlerts(ctx context.Context, params alertmanagertypes.GettableAlertsParams) (alertmanagertypes.GettableAlerts, error) {
 	return alertmanagertypes.NewGettableAlertsFromAlertProvider(server.alerts, server.alertmanagerConfig, server.marker.Status, func(labels model.LabelSet) {
-		server.inhibitor.Mutes(labels)
-		server.silencer.Mutes(labels)
+		server.inhibitor.Mutes(ctx, labels)
+		server.silencer.Mutes(ctx, labels)
 	}, params)
 }
 
 func (server *Server) PutAlerts(ctx context.Context, postableAlerts alertmanagertypes.PostableAlerts) error {
-	alerts, err := alertmanagertypes.NewAlertsFromPostableAlerts(postableAlerts, time.Duration(server.srvConfig.Global.ResolveTimeout), time.Now())
+	alerts, err := alertmanagertypes.NewAlertsFromPostableAlerts(ctx, postableAlerts, time.Duration(server.srvConfig.Global.ResolveTimeout), time.Now())
 	// Notification sending alert takes precedence over validation errors.
-	if err := server.alerts.Put(alerts...); err != nil {
+	if err := server.alerts.Put(ctx, alerts...); err != nil {
 		return err
 	}
 
@@ -340,6 +340,7 @@ func (server *Server) TestAlert(ctx context.Context, receiversMap map[*alertmana
 	}
 
 	alerts, err := alertmanagertypes.NewAlertsFromPostableAlerts(
+		ctx,
 		postableAlerts,
 		time.Duration(server.srvConfig.Global.ResolveTimeout),
 		time.Now(),

pkg/alertmanager/alertmanagerserver/server_test.go

@@ -70,7 +70,7 @@ func TestServerTestReceiverTypeWebhook(t *testing.T) {
 		WebhookConfigs: []*config.WebhookConfig{
 			{
 				HTTPConfig: &commoncfg.HTTPClientConfig{},
-				URL:        &config.SecretURL{URL: webhookURL},
+				URL:        config.SecretTemplateURL(webhookURL.String()),
 			},
 		},
 	})
@@ -96,7 +96,7 @@ func TestServerPutAlerts(t *testing.T) {
 		WebhookConfigs: []*config.WebhookConfig{
 			{
 				HTTPConfig: &commoncfg.HTTPClientConfig{},
-				URL:        &config.SecretURL{URL: &url.URL{Host: "localhost", Path: "/test-receiver"}},
+				URL:        config.SecretTemplateURL("http://localhost/test-receiver"),
 			},
 		},
 	}))
@@ -176,7 +176,7 @@ func TestServerTestAlert(t *testing.T) {
 		WebhookConfigs: []*config.WebhookConfig{
 			{
 				HTTPConfig: &commoncfg.HTTPClientConfig{},
-				URL:        &config.SecretURL{URL: webhook1URL},
+				URL:        config.SecretTemplateURL(webhook1URL.String()),
 			},
 		},
 	}))
@@ -186,7 +186,7 @@ func TestServerTestAlert(t *testing.T) {
 		WebhookConfigs: []*config.WebhookConfig{
 			{
 				HTTPConfig: &commoncfg.HTTPClientConfig{},
-				URL:        &config.SecretURL{URL: webhook2URL},
+				URL:        config.SecretTemplateURL(webhook2URL.String()),
 			},
 		},
 	}))
@@ -268,7 +268,7 @@ func TestServerTestAlertContinuesOnFailure(t *testing.T) {
 		WebhookConfigs: []*config.WebhookConfig{
 			{
 				HTTPConfig: &commoncfg.HTTPClientConfig{},
-				URL:        &config.SecretURL{URL: webhookURL},
+				URL:        config.SecretTemplateURL(webhookURL.String()),
 			},
 		},
 	}))
@@ -278,7 +278,7 @@ func TestServerTestAlertContinuesOnFailure(t *testing.T) {
 		WebhookConfigs: []*config.WebhookConfig{
 			{
 				HTTPConfig: &commoncfg.HTTPClientConfig{},
-				URL:        &config.SecretURL{URL: &url.URL{Scheme: "http", Host: "localhost:1", Path: "/webhook"}},
+				URL:        config.SecretTemplateURL("http://localhost:1/webhook"),
 			},
 		},
 	}))

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate.go

@@ -0,0 +1,336 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+)
+
+// TemplateInput carries the title/body templates
+// and their defaults to apply in case the custom templates
+// are result in empty strings.
+type TemplateInput struct {
+	TitleTemplate        string
+	BodyTemplate         string
+	DefaultTitleTemplate string
+	DefaultBodyTemplate  string
+}
+
+// ExpandedTemplates is the result of ExpandAlertTemplates.
+type ExpandedTemplates struct {
+	Title string
+	Body  string
+}
+
+// AlertData holds per-alert data used when expanding body templates.
+type AlertData struct {
+	Status       string      `json:"status"`
+	Labels       template.KV `json:"labels"`
+	Annotations  template.KV `json:"annotations"`
+	StartsAt     time.Time   `json:"starts_at"`
+	EndsAt       time.Time   `json:"ends_at"`
+	GeneratorURL string      `json:"generator_url"`
+	Fingerprint  string      `json:"fingerprint"`
+
+	// Convenience fields extracted from well-known labels/annotations.
+	AlertName string `json:"rule_name"`
+	RuleID    string `json:"rule_id"`
+	RuleLink  string `json:"rule_link"`
+	Severity  string `json:"severity"`
+
+	// Link annotations added by the rule evaluator.
+	LogLink   string `json:"log_link"`
+	TraceLink string `json:"trace_link"`
+
+	// Status booleans for easy conditional templating.
+	IsFiring      bool `json:"is_firing"`
+	IsResolved    bool `json:"is_resolved"`
+	IsMissingData bool `json:"is_missing_data"`
+	IsRecovering  bool `json:"is_recovering"`
+}
+
+// NotificationTemplateData is the top-level data struct provided to custom templates.
+type NotificationTemplateData struct {
+	Receiver string `json:"receiver"`
+	Status   string `json:"status"`
+
+	// Convenience fields for title templates.
+	AlertName     string `json:"rule_name"`
+	RuleID        string `json:"rule_id"`
+	RuleLink      string `json:"rule_link"`
+	TotalFiring   int    `json:"total_firing"`
+	TotalResolved int    `json:"total_resolved"`
+
+	// Per-alert data, also available as filtered sub-slices.
+	Alerts []AlertData `json:"alerts"`
+
+	// Cross-alert aggregates, computed as intersection across all alerts.
+	GroupLabels       template.KV `json:"group_labels"`
+	CommonLabels      template.KV `json:"common_labels"`
+	CommonAnnotations template.KV `json:"common_annotations"`
+	ExternalURL       string      `json:"external_url"`
+}
+
+// ExtractTemplatesFromAnnotations computes the common annotations across all alerts
+// and returns the values for the title_template and body_template annotation keys.
+func ExtractTemplatesFromAnnotations(alerts []*types.Alert) (titleTemplate, bodyTemplate string) {
+	if len(alerts) == 0 {
+		return "", ""
+	}
+
+	commonAnnotations := computeCommonAnnotations(alerts)
+	return commonAnnotations[ruletypes.AnnotationTitleTemplate], commonAnnotations[ruletypes.AnnotationBodyTemplate]
+}
+
+// ExpandAlertTemplates expands the title and body templates from input
+// against the provided alerts and returns the expanded templates.
+func ExpandAlertTemplates(
+	ctx context.Context,
+	tmpl *template.Template,
+	input TemplateInput,
+	alerts []*types.Alert,
+	logger *slog.Logger,
+) (*ExpandedTemplates, error) {
+	ntd := buildNotificationTemplateData(ctx, tmpl, alerts, logger)
+
+	title, err := expandTitle(ctx, tmpl, input, alerts, ntd, logger)
+	if err != nil {
+		return nil, err
+	}
+
+	body, err := expandBody(ctx, tmpl, input, alerts, ntd, logger)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ExpandedTemplates{Title: title, Body: body}, nil
+}
+
+// expandTitle expands the title template. Falls back to the default if the custom template
+// result in empty string.
+func expandTitle(
+	ctx context.Context,
+	tmpl *template.Template,
+	input TemplateInput,
+	alerts []*types.Alert,
+	ntd *NotificationTemplateData,
+	logger *slog.Logger,
+) (string, error) {
+	if input.TitleTemplate != "" {
+		result, err := tmpl.ExecuteTextString(input.TitleTemplate, ntd)
+		if err != nil {
+			return "", err
+		}
+		if strings.TrimSpace(result) != "" {
+			return result, nil
+		}
+	}
+
+	// Fall back to the notifier's default title template using standard template.Data.
+	if input.DefaultTitleTemplate == "" {
+		return "", nil
+	}
+	data := notify.GetTemplateData(ctx, tmpl, alerts, logger)
+	return tmpl.ExecuteTextString(input.DefaultTitleTemplate, data)
+}
+
+// expandBody expands the body template once per alert, concatenates the results
+// and falls back to the default if the custom template result in empty string.
+func expandBody(
+	ctx context.Context,
+	tmpl *template.Template,
+	input TemplateInput,
+	alerts []*types.Alert,
+	ntd *NotificationTemplateData,
+	logger *slog.Logger,
+) (string, error) {
+	if input.BodyTemplate != "" && len(ntd.Alerts) > 0 {
+		var parts []string
+		for i := range ntd.Alerts {
+			part, err := tmpl.ExecuteTextString(input.BodyTemplate, &ntd.Alerts[i])
+			if err != nil {
+				return "", err
+			}
+			parts = append(parts, part)
+		}
+		result := strings.Join(parts, "<br><br>") // markdown uses html for line breaks
+		if strings.TrimSpace(result) != "" {
+			return result, nil
+		}
+	}
+
+	// Fall back to the notifier's default body template using standard template.Data.
+	if input.DefaultBodyTemplate == "" {
+		return "", nil
+	}
+	data := notify.GetTemplateData(ctx, tmpl, alerts, logger)
+	return tmpl.ExecuteTextString(input.DefaultBodyTemplate, data)
+}
+
+// buildNotificationTemplateData derives a NotificationTemplateData from
+// the context, template, and the raw alerts.
+func buildNotificationTemplateData(
+	ctx context.Context,
+	tmpl *template.Template,
+	alerts []*types.Alert,
+	logger *slog.Logger,
+) *NotificationTemplateData {
+	// extract the required data from the context
+	receiver, ok := notify.ReceiverName(ctx)
+	if !ok {
+		logger.WarnContext(ctx, "missing receiver name in context")
+	}
+
+	groupLabels, ok := notify.GroupLabels(ctx)
+	if !ok {
+		logger.WarnContext(ctx, "missing group labels in context")
+	}
+
+	// extract the external URL from the template
+	externalURL := ""
+	if tmpl.ExternalURL != nil {
+		externalURL = tmpl.ExternalURL.String()
+	}
+
+	commonAnnotations := computeCommonAnnotations(alerts)
+	commonLabels := computeCommonLabels(alerts)
+
+	// build the alert data slice
+	alertDataSlice := make([]AlertData, 0, len(alerts))
+	for _, a := range alerts {
+		ad := buildAlertData(a)
+		alertDataSlice = append(alertDataSlice, ad)
+	}
+
+	// count the number of firing and resolved alerts
+	var firing, resolved int
+	for _, ad := range alertDataSlice {
+		if ad.IsFiring {
+			firing++
+		} else if ad.IsResolved {
+			resolved++
+		}
+	}
+
+	// extract the rule-level convenience fields from common labels
+	alertName := commonLabels[ruletypes.LabelAlertName]
+	ruleID := commonLabels[ruletypes.LabelRuleId]
+	ruleLink := commonLabels[ruletypes.LabelRuleSource]
+
+	// build the group labels
+	gl := make(template.KV, len(groupLabels))
+	for k, v := range groupLabels {
+		gl[string(k)] = string(v)
+	}
+
+	// build the notification template data
+	return &NotificationTemplateData{
+		Receiver:          receiver,
+		Status:            string(types.Alerts(alerts...).Status()),
+		AlertName:         alertName,
+		RuleID:            ruleID,
+		RuleLink:          ruleLink,
+		TotalFiring:       firing,
+		TotalResolved:     resolved,
+		Alerts:            alertDataSlice,
+		GroupLabels:       gl,
+		CommonLabels:      commonLabels,
+		CommonAnnotations: commonAnnotations,
+		ExternalURL:       externalURL,
+	}
+}
+
+// buildAlertData converts a single *types.Alert into an AlertData.
+func buildAlertData(a *types.Alert) AlertData {
+	labels := make(template.KV, len(a.Labels))
+	for k, v := range a.Labels {
+		labels[string(k)] = string(v)
+	}
+
+	annotations := make(template.KV, len(a.Annotations))
+	for k, v := range a.Annotations {
+		annotations[string(k)] = string(v)
+	}
+
+	status := string(a.Status())
+	isFiring := a.Status() == model.AlertFiring
+	isResolved := a.Status() == model.AlertResolved
+	isMissingData := labels[ruletypes.LabelNoData] == "true"
+
+	return AlertData{
+		Status:        status,
+		Labels:        labels,
+		Annotations:   annotations,
+		StartsAt:      a.StartsAt,
+		EndsAt:        a.EndsAt,
+		GeneratorURL:  a.GeneratorURL,
+		Fingerprint:   a.Fingerprint().String(),
+		AlertName:     labels[ruletypes.LabelAlertName],
+		RuleID:        labels[ruletypes.LabelRuleId],
+		RuleLink:      labels[ruletypes.LabelRuleSource],
+		Severity:      labels[ruletypes.LabelSeverityName],
+		LogLink:       annotations[ruletypes.AnnotationRelatedLogs],
+		TraceLink:     annotations[ruletypes.AnnotationRelatedTraces],
+		IsFiring:      isFiring,
+		IsResolved:    isResolved,
+		IsMissingData: isMissingData,
+	}
+}
+
+// computeCommonAnnotations returns the intersection of annotations across all alerts as a template.KV.
+// An annotation key/value pair is included only if it appears identically on every alert.
+func computeCommonAnnotations(alerts []*types.Alert) template.KV {
+	if len(alerts) == 0 {
+		return template.KV{}
+	}
+
+	common := make(template.KV, len(alerts[0].Annotations))
+	for k, v := range alerts[0].Annotations {
+		common[string(k)] = string(v)
+	}
+
+	for _, a := range alerts[1:] {
+		for k := range common {
+			if string(a.Annotations[model.LabelName(k)]) != common[k] {
+				delete(common, k)
+			}
+		}
+		if len(common) == 0 {
+			break
+		}
+	}
+
+	return common
+}
+
+// computeCommonLabels returns the intersection of labels across all alerts as a template.KV.
+func computeCommonLabels(alerts []*types.Alert) template.KV {
+	if len(alerts) == 0 {
+		return template.KV{}
+	}
+
+	common := make(template.KV, len(alerts[0].Labels))
+	for k, v := range alerts[0].Labels {
+		common[string(k)] = string(v)
+	}
+
+	for _, a := range alerts[1:] {
+		for k := range common {
+			if string(a.Labels[model.LabelName(k)]) != common[k] {
+				delete(common, k)
+			}
+		}
+		if len(common) == 0 {
+			break
+		}
+	}
+
+	return common
+}

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate_test.go

@@ -0,0 +1,147 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+
+	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// testSetup returns template, context, and logger for tests.
+func testSetup(t *testing.T) (*template.Template, context.Context, *slog.Logger) {
+	t.Helper()
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "slack")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": "HighCPU", "severity": "critical"})
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	return tmpl, ctx, logger
+}
+
+// firingAlert returns a firing alert with the given labels/annotations.
+func firingAlert(labels, annotations map[string]string) *types.Alert {
+	ls := model.LabelSet{}
+	for k, v := range labels {
+		ls[model.LabelName(k)] = model.LabelValue(v)
+	}
+	ann := model.LabelSet{}
+	for k, v := range annotations {
+		ann[model.LabelName(k)] = model.LabelValue(v)
+	}
+	now := time.Now()
+	return &types.Alert{
+		Alert: model.Alert{
+			Labels:      ls,
+			Annotations: ann,
+			StartsAt:    now,
+			EndsAt:      now.Add(time.Hour),
+		},
+	}
+}
+
+// TestExpandAlertTemplates_BothCustomTitleAndBody verifies that when both custom
+// title and body templates are provided, both expand correctly (main happy path).
+func TestExpandAlertTemplates_BothCustomTitleAndBody(t *testing.T) {
+	tmpl, ctx, logger := testSetup(t)
+	alerts := []*types.Alert{
+		firingAlert(
+			map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelRuleId: "rule-1", ruletypes.LabelSeverityName: "critical"},
+			nil,
+		),
+	}
+	input := TemplateInput{
+		TitleTemplate:        "[{{.Status}}] {{.AlertName}}",
+		BodyTemplate:         "Alert {{.AlertName}} ({{.Status}})",
+		DefaultTitleTemplate: "{{ .CommonLabels.alertname }}",
+		DefaultBodyTemplate:  "{{ .Status }}",
+	}
+	got, err := ExpandAlertTemplates(ctx, tmpl, input, alerts, logger)
+	require.NoError(t, err)
+	require.Equal(t, "[firing] HighCPU", got.Title)
+	require.Equal(t, "Alert HighCPU (firing)", got.Body)
+}
+
+// TestExpandAlertTemplates_CustomTitleExpands verifies that a custom title
+// template expands against NotificationTemplateData (rule-level fields).
+func TestExpandAlertTemplates_CustomTitleExpands(t *testing.T) {
+	tmpl, ctx, logger := testSetup(t)
+	alerts := []*types.Alert{
+		firingAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelRuleId: "r1"}, nil),
+		firingAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelRuleId: "r1"}, nil),
+	}
+	input := TemplateInput{
+		TitleTemplate:        "{{.AlertName}}: {{.TotalFiring}} firing",
+		DefaultTitleTemplate: "fallback",
+	}
+	got, err := ExpandAlertTemplates(ctx, tmpl, input, alerts, logger)
+	require.NoError(t, err)
+	require.Equal(t, "HighCPU: 2 firing", got.Title)
+	require.Equal(t, "", got.Body)
+}
+
+// TestExpandAlertTemplates_CustomBodySingleAlert verifies that a custom body
+// template is expanded once per alert; with one alert, body is that expansion.
+func TestExpandAlertTemplates_CustomBodySingleAlert(t *testing.T) {
+	tmpl, ctx, logger := testSetup(t)
+	alerts := []*types.Alert{
+		firingAlert(
+			map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"},
+			map[string]string{"description": "CPU > 80%"},
+		),
+	}
+	input := TemplateInput{
+		BodyTemplate: "{{.AlertName}} ({{.Severity}}): {{.Annotations.description}}",
+	}
+	got, err := ExpandAlertTemplates(ctx, tmpl, input, alerts, logger)
+	require.NoError(t, err)
+	require.Equal(t, "", got.Title)
+	require.Equal(t, "HighCPU (critical): CPU > 80%", got.Body)
+}
+
+// TestExpandAlertTemplates_CustomBodyMultipleAlerts verifies that body template
+// is expanded per alert and results are concatenated with "<br><br>".
+func TestExpandAlertTemplates_CustomBodyMultipleAlerts(t *testing.T) {
+	tmpl, ctx, logger := testSetup(t)
+	alerts := []*types.Alert{
+		firingAlert(map[string]string{ruletypes.LabelAlertName: "A"}, nil),
+		firingAlert(map[string]string{ruletypes.LabelAlertName: "B"}, nil),
+		firingAlert(map[string]string{ruletypes.LabelAlertName: "C"}, nil),
+	}
+	input := TemplateInput{
+		BodyTemplate: "{{.AlertName}}",
+	}
+	got, err := ExpandAlertTemplates(ctx, tmpl, input, alerts, logger)
+	require.NoError(t, err)
+	require.Equal(t, "A<br><br>B<br><br>C", got.Body)
+}
+
+// TestExpandAlertTemplates_BothDefaultTitleAndBody verifies that when no custom
+// templates are set, both title and body fall back to default templates
+// (executed against Prometheus template.Data).
+func TestExpandAlertTemplates_BothDefaultTitleAndBody(t *testing.T) {
+	tmpl, ctx, logger := testSetup(t)
+	alerts := []*types.Alert{
+		firingAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil),
+	}
+	input := TemplateInput{
+		TitleTemplate:        "",
+		BodyTemplate:         "",
+		DefaultTitleTemplate: "{{ .CommonLabels.alertname }}",
+		DefaultBodyTemplate:  "{{ .Status }}: {{ .CommonLabels.alertname }}",
+	}
+	got, err := ExpandAlertTemplates(ctx, tmpl, input, alerts, logger)
+	require.NoError(t, err)
+	require.Equal(t, "HighCPU", got.Title)
+	require.Equal(t, "firing: HighCPU", got.Body)
+}

pkg/errors/errors.go

@@ -147,7 +147,7 @@ func Ast(cause error, typ typ) bool {
 	return t == typ
 }
 
-// Ast checks if the provided error matches the specified custom error code.
+// Asc checks if the provided error matches the specified custom error code.
 func Asc(cause error, code Code) bool {
 	_, c, _, _, _, _ := Unwrapb(cause)
 

pkg/instrumentation/sdk.go

@@ -15,7 +15,7 @@ import (
 	sdkmetric "go.opentelemetry.io/otel/metric"
 	sdkmetricnoop "go.opentelemetry.io/otel/metric/noop"
 	sdkresource "go.opentelemetry.io/otel/sdk/resource"
-	semconv "go.opentelemetry.io/otel/semconv/v1.37.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.39.0"
 	sdktrace "go.opentelemetry.io/otel/trace"
 )
 

pkg/prometheus/clickhouseprometheus/client.go

@@ -6,6 +6,7 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
@@ -90,6 +91,41 @@ func (client *client) Read(ctx context.Context, query *prompb.Query, sortSeries
 	return remote.FromQueryResult(sortSeries, res), nil
 }
 
+func (c *client) ReadMultiple(ctx context.Context, queries []*prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
+	if len(queries) == 0 {
+		return storage.EmptySeriesSet(), nil
+	}
+	if len(queries) == 1 {
+		return c.Read(ctx, queries[0], sortSeries)
+	}
+
+	type result struct {
+		ss  storage.SeriesSet
+		err error
+	}
+
+	results := make([]result, len(queries))
+	var wg sync.WaitGroup
+	wg.Add(len(queries))
+	for i, q := range queries {
+		go func(i int, q *prompb.Query) {
+			defer wg.Done()
+			ss, err := c.Read(ctx, q, sortSeries)
+			results[i] = result{ss, err}
+		}(i, q)
+	}
+	wg.Wait()
+
+	sets := make([]storage.SeriesSet, 0, len(queries))
+	for _, r := range results {
+		if r.err != nil {
+			return nil, r.err
+		}
+		sets = append(sets, r.ss)
+	}
+	return storage.NewMergeSeriesSet(sets, 0, storage.ChainedSeriesMerge), nil
+}
+
 func (client *client) queryToClickhouseQuery(_ context.Context, query *prompb.Query, metricName string, subQuery bool) (string, []any, error) {
 	var clickHouseQuery string
 	var conditions []string

pkg/prometheus/utils.go

@@ -2,6 +2,7 @@ package prometheus
 
 import (
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
 )
 
@@ -10,35 +11,22 @@ func RemoveExtraLabels(res *promql.Result, labelsToRemove ...string) error {
 		return nil
 	}
 
-	toRemove := make(map[string]struct{}, len(labelsToRemove))
-	for _, l := range labelsToRemove {
-		toRemove[l] = struct{}{}
-	}
-
 	switch res.Value.(type) {
 	case promql.Vector:
 		value := res.Value.(promql.Vector)
 		for i := range value {
-			series := &(value)[i]
-			dst := series.Metric[:0]
-			for _, lbl := range series.Metric {
-				if _, drop := toRemove[lbl.Name]; !drop {
-					dst = append(dst, lbl)
-				}
-			}
-			series.Metric = dst
+			b := labels.NewBuilder(value[i].Metric)
+			b.Del(labelsToRemove...)
+			newLabels := b.Labels()
+			value[i].Metric = newLabels
 		}
 	case promql.Matrix:
 		value := res.Value.(promql.Matrix)
 		for i := range value {
-			series := &(value)[i]
-			dst := series.Metric[:0]
-			for _, lbl := range series.Metric {
-				if _, drop := toRemove[lbl.Name]; !drop {
-					dst = append(dst, lbl)
-				}
-			}
-			series.Metric = dst
+			b := labels.NewBuilder(value[i].Metric)
+			b.Del(labelsToRemove...)
+			newLabels := b.Labels()
+			value[i].Metric = newLabels
 		}
 	case promql.Scalar:
 		return nil

pkg/querier/promql_query.go

@@ -18,6 +18,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
 	qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
 )
@@ -254,17 +255,16 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 	var series []*qbv5.TimeSeries
 	for _, v := range matrix {
 		var s qbv5.TimeSeries
-		lbls := make([]*qbv5.Label, 0, len(v.Metric))
-		for name, value := range v.Metric.Copy().Map() {
-			if excludeLabel(name) {
-				continue
+		lbls := make([]*qbv5.Label, 0, v.Metric.Len())
+		v.Metric.Range(func(l labels.Label) {
+			if excludeLabel(l.Name) {
+				return
 			}
 			lbls = append(lbls, &qbv5.Label{
-				Key:   telemetrytypes.TelemetryFieldKey{Name: name},
-				Value: value,
+				Key:   telemetrytypes.TelemetryFieldKey{Name: l.Name},
+				Value: l.Value,
 			})
-		}
-
+		})
 		s.Labels = lbls
 
 		for idx := range v.Floats {

pkg/query-service/rules/prom_rule.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/prometheus"
-	"github.com/SigNoz/signoz/pkg/units"
 	"github.com/SigNoz/signoz/pkg/query-service/interfaces"
 	"github.com/SigNoz/signoz/pkg/query-service/model"
 	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
@@ -18,7 +17,9 @@ import (
 	"github.com/SigNoz/signoz/pkg/query-service/utils/timestamp"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/SigNoz/signoz/pkg/units"
 	"github.com/SigNoz/signoz/pkg/valuer"
+	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
 )
 
@@ -461,12 +462,12 @@ func toCommonSeries(series promql.Series) v3.Series {
 		Points:      make([]v3.Point, 0),
 	}
 
-	for _, lbl := range series.Metric {
+	series.Metric.Range(func(lbl labels.Label) {
 		commonSeries.Labels[lbl.Name] = lbl.Value
 		commonSeries.LabelsArray = append(commonSeries.LabelsArray, map[string]string{
 			lbl.Name: lbl.Value,
 		})
-	}
+	})
 
 	for _, f := range series.Floats {
 		commonSeries.Points = append(commonSeries.Points, v3.Point{

pkg/telemetrylogs/condition_builder.go

@@ -35,7 +35,7 @@ func (c *conditionBuilder) conditionFor(
 		return "", err
 	}
 
-	if column.IsJSONColumn() && querybuilder.BodyJSONQueryEnabled {
+	if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled {
 		valueType, value := InferDataType(value, operator, key)
 		cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
 		if err != nil {

pkg/telemetrylogs/const.go

@@ -17,7 +17,7 @@ const (
 	LogsV2TimestampColumn         = "timestamp"
 	LogsV2ObservedTimestampColumn = "observed_timestamp"
 	LogsV2BodyColumn              = "body"
-	LogsV2BodyJSONColumn          = constants.BodyJSONColumn
+	LogsV2BodyJSONColumn          = constants.BodyV2Column
 	LogsV2BodyPromotedColumn      = constants.BodyPromotedColumn
 	LogsV2TraceIDColumn           = "trace_id"
 	LogsV2SpanIDColumn            = "span_id"
@@ -34,7 +34,7 @@ const (
 	LogsV2ResourcesStringColumn  = "resources_string"
 	LogsV2ScopeStringColumn      = "scope_string"
 
-	BodyJSONColumnPrefix     = constants.BodyJSONColumnPrefix
+	BodyJSONColumnPrefix     = constants.BodyV2ColumnPrefix
 	BodyPromotedColumnPrefix = constants.BodyPromotedColumnPrefix
 )
 

pkg/telemetrylogs/field_mapper.go

@@ -61,7 +61,7 @@ var (
 	}
 )
 
-type fieldMapper struct {}
+type fieldMapper struct{}
 
 func NewFieldMapper() qbtypes.FieldMapper {
 	return &fieldMapper{}

pkg/telemetrymetadata/body_json_metadata.go

@@ -260,7 +260,7 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
 	sb.Where(sb.Equal("database", telemetrylogs.DBName))
 	sb.Where(sb.Equal("table", telemetrylogs.LogsV2LocalTableName))
 	sb.Where(sb.Or(
-		sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyJSONColumnPrefix))),
+		sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix))),
 		sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix))),
 	))
 

pkg/telemetrymetadata/body_json_metadata_test.go

@@ -117,7 +117,7 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
 			expectedArgs: []any{
 				telemetrylogs.DBName,
 				telemetrylogs.LogsV2LocalTableName,
-				fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyJSONColumnPrefix)),
+				fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
 				fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
 			},
 		},
@@ -130,7 +130,7 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
 			expectedArgs: []any{
 				telemetrylogs.DBName,
 				telemetrylogs.LogsV2LocalTableName,
-				fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyJSONColumnPrefix)),
+				fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
 				fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
 				fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains("foo")),
 				fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains("bar")),

pkg/types/alertmanagertypes/alert.go

@@ -1,6 +1,7 @@
 package alertmanagertypes
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -90,8 +91,8 @@ func NewDeprecatedGettableAlertsFromGettableAlerts(gettableAlerts GettableAlerts
 }
 
 // Converts a slice of PostableAlert to a slice of Alert.
-func NewAlertsFromPostableAlerts(postableAlerts PostableAlerts, resolveTimeout time.Duration, now time.Time) ([]*types.Alert, []error) {
-	alerts := v2.OpenAPIAlertsToAlerts(postableAlerts)
+func NewAlertsFromPostableAlerts(ctx context.Context, postableAlerts PostableAlerts, resolveTimeout time.Duration, now time.Time) ([]*types.Alert, []error) {
+	alerts := v2.OpenAPIAlertsToAlerts(ctx, postableAlerts)
 
 	for _, alert := range alerts {
 		alert.UpdatedAt = now
@@ -196,8 +197,15 @@ func NewGettableAlertsFromAlertProvider(
 		if err = iterator.Err(); err != nil {
 			break
 		}
+		if a == nil {
+			break
+		}
+		alertData := a.Data
+		if alertData == nil {
+			continue
+		}
 
-		routes := dispatch.NewRoute(cfg.alertmanagerConfig.Route, nil).Match(a.Labels)
+		routes := dispatch.NewRoute(cfg.alertmanagerConfig.Route, nil).Match(alertData.Labels)
 		receivers := make([]string, 0, len(routes))
 		for _, r := range routes {
 			receivers = append(receivers, r.RouteOpts.Receiver)
@@ -207,11 +215,11 @@ func NewGettableAlertsFromAlertProvider(
 			continue
 		}
 
-		if !alertFilter(a, now) {
+		if !alertFilter(alertData, now) {
 			continue
 		}
 
-		alert := v2.AlertToOpenAPIAlert(a, getAlertStatusFunc(a.Fingerprint()), receivers, nil)
+		alert := v2.AlertToOpenAPIAlert(alertData, getAlertStatusFunc(alertData.Fingerprint()), receivers, nil)
 
 		res = append(res, alert)
 	}

pkg/types/alertmanagertypes/channel_test.go

@@ -41,6 +41,7 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"require_tls":   true,
 						"html":          "{{ template \"email.default.html\" . }}",
 						"tls_config":    map[string]any{"insecure_skip_verify": false},
+						"threading":     map[string]any{},
 					}},
 				},
 			},
@@ -62,6 +63,7 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"slack_configs": []any{map[string]any{
 						"send_resolved": true,
 						"api_url":       "https://slack.com/api/test",
+						"app_url":       "https://slack.com/api/chat.postMessage",
 						"channel":       "#alerts",
 						"callback_id":   "{{ template \"slack.default.callbackid\" . }}",
 						"color":         "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",
@@ -71,6 +73,7 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"icon_url":      "{{ template \"slack.default.iconurl\" . }}",
 						"pretext":       "{{ template \"slack.default.pretext\" . }}",
 						"text":          "{{ template \"slack.default.text\" . }}",
+						"timeout":       float64(0),
 						"title":         "{{ template \"slack.default.title\" . }}",
 						"title_link":    "{{ template \"slack.default.titlelink\" . }}",
 						"username":      "{{ template \"slack.default.username\" . }}",
@@ -106,11 +109,12 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"client_url":    "{{ template \"pagerduty.default.clientURL\" . }}",
 						"description":   "{{ template \"pagerduty.default.description\" .}}",
 						"source":        "{{ template \"pagerduty.default.client\" . }}",
+						"timeout":       float64(0),
 						"details": map[string]any{
-							"firing":       "{{ template \"pagerduty.default.instances\" .Alerts.Firing }}",
+							"firing":       "{{ .Alerts.Firing | toJson }}",
 							"num_firing":   "{{ .Alerts.Firing | len }}",
 							"num_resolved": "{{ .Alerts.Resolved | len }}",
-							"resolved":     "{{ template \"pagerduty.default.instances\" .Alerts.Resolved }}",
+							"resolved":     "{{ .Alerts.Resolved | toJson }}",
 						},
 						"http_config": map[string]any{
 							"tls_config":       map[string]any{"insecure_skip_verify": false},
@@ -149,11 +153,12 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"client_url":    "{{ template \"pagerduty.default.clientURL\" . }}",
 						"description":   "{{ template \"pagerduty.default.description\" .}}",
 						"source":        "{{ template \"pagerduty.default.client\" . }}",
+						"timeout":       float64(0),
 						"details": map[string]any{
-							"firing":       "{{ template \"pagerduty.default.instances\" .Alerts.Firing }}",
+							"firing":       "{{ .Alerts.Firing | toJson }}",
 							"num_firing":   "{{ .Alerts.Firing | len }}",
 							"num_resolved": "{{ .Alerts.Resolved | len }}",
-							"resolved":     "{{ template \"pagerduty.default.instances\" .Alerts.Resolved }}",
+							"resolved":     "{{ .Alerts.Resolved | toJson }}",
 						},
 						"http_config": map[string]any{
 							"tls_config":       map[string]any{"insecure_skip_verify": false},
@@ -168,6 +173,7 @@ func TestNewConfigFromChannels(t *testing.T) {
 					"slack_configs": []any{map[string]any{
 						"send_resolved": true,
 						"api_url":       "https://slack.com/api/test",
+						"app_url":       "https://slack.com/api/chat.postMessage",
 						"channel":       "#alerts",
 						"callback_id":   "{{ template \"slack.default.callbackid\" . }}",
 						"color":         "{{ if eq .Status \"firing\" }}danger{{ else }}good{{ end }}",
@@ -177,6 +183,7 @@ func TestNewConfigFromChannels(t *testing.T) {
 						"icon_url":      "{{ template \"slack.default.iconurl\" . }}",
 						"pretext":       "{{ template \"slack.default.pretext\" . }}",
 						"text":          "{{ template \"slack.default.text\" . }}",
+						"timeout":       float64(0),
 						"title":         "{{ template \"slack.default.title\" . }}",
 						"title_link":    "{{ template \"slack.default.titlelink\" . }}",
 						"username":      "{{ template \"slack.default.username\" . }}",
@@ -270,7 +277,7 @@ func TestNewChannelFromReceiver(t *testing.T) {
 			expected: &Channel{
 				Name: "test-receiver",
 				Type: "slack",
-				Data: `{"name":"test-receiver","slack_configs":[{"send_resolved":true,"api_url":"https://slack.com/api/test","channel":"#alerts"}]}`,
+				Data: `{"name":"test-receiver","slack_configs":[{"send_resolved":true,"api_url":"https://slack.com/api/test","channel":"#alerts","timeout":0}]}`,
 			},
 			pass: true,
 		},

pkg/types/promotetypes/types.go

@@ -36,8 +36,8 @@ func (i *PromotePath) ValidateAndSetDefaults() error {
 		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "array paths can not be promoted or indexed")
 	}
 
-	if strings.HasPrefix(i.Path, constants.BodyJSONColumnPrefix) || strings.HasPrefix(i.Path, constants.BodyPromotedColumnPrefix) {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", constants.BodyJSONColumnPrefix, constants.BodyPromotedColumnPrefix)
+	if strings.HasPrefix(i.Path, constants.BodyV2ColumnPrefix) || strings.HasPrefix(i.Path, constants.BodyPromotedColumnPrefix) {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", constants.BodyV2ColumnPrefix, constants.BodyPromotedColumnPrefix)
 	}
 
 	if !strings.HasPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix) {

pkg/types/ruletypes/constants.go

@@ -9,4 +9,15 @@ const (
 	LabelSeverityName     = "severity"
 	LabelLastSeen         = "lastSeen"
 	LabelRuleId           = "ruleId"
+	LabelRuleSource       = "ruleSource"
+	LabelNoData           = "nodata"
+	LabelTestAlert        = "testalert"
+	LabelAlertName        = "alertname"
+)
+
+const (
+	AnnotationRelatedLogs   = "related_logs"
+	AnnotationRelatedTraces = "related_traces"
+	AnnotationTitleTemplate = "title_template"
+	AnnotationBodyTemplate  = "body_template"
 )