chore: increased alert time to avoid flaky tests

* refactor: moving types to cloud provider specific namespace/pkg

* refactor: separating cloud provider types

* refactor: using upper case key for AWS

* feat: adding cloud integration azure types

* feat: adding azure services

* refactor: updating omitempty tags

* refactor: updating azure integration config

* feat: completing azure types

* refactor: lint issues

* refactor: updating command key

* fix: handle optional connection URL in AWS integration

* refactor: updating strategy struct

* refactor: updating azure blob storage service name

* refactor: update Azure service identifiers

* refactor: updating types

.github/workflows/integrationci.yaml

@@ -39,6 +39,7 @@ jobs:
       matrix:
         suite:
           - alerts
+          - alertmanager
           - callbackauthn
           - cloudintegrations
           - dashboard

cmd/community/server.go

@@ -92,7 +92,7 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
 		func(ctx context.Context, providerSettings factory.ProviderSettings, store authtypes.AuthNStore, licensing licensing.Licensing) (map[authtypes.AuthNProvider]authn.AuthN, error) {
 			return signoz.NewAuthNs(ctx, providerSettings, store, licensing)
 		},
-		func(ctx context.Context, sqlstore sqlstore.SQLStore, _ licensing.Licensing, _ dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error) {
+		func(ctx context.Context, sqlstore sqlstore.SQLStore, _ licensing.Licensing, _ []authz.OnBeforeRoleDelete, _ dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error) {
 			openfgaDataStore, err := openfgaserver.NewSQLStore(sqlstore)
 			if err != nil {
 				return nil, err

cmd/enterprise/server.go

@@ -137,12 +137,12 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
 
 			return authNs, nil
 		},
-		func(ctx context.Context, sqlstore sqlstore.SQLStore, licensing licensing.Licensing, dashboardModule dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error) {
+		func(ctx context.Context, sqlstore sqlstore.SQLStore, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete, dashboardModule dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error) {
 			openfgaDataStore, err := openfgaserver.NewSQLStore(sqlstore)
 			if err != nil {
 				return nil, err
 			}
-			return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, licensing, dashboardModule), nil
+			return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, licensing, onBeforeRoleDelete, dashboardModule), nil
 		},
 		func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing) dashboard.Module {
 			return impldashboard.NewModule(pkgimpldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, querier, licensing)

conf/example.yaml

@@ -407,3 +407,11 @@ cloudintegration:
   agent:
     # The version of the cloud integration agent.
     version: v0.0.8
+
+##################### Authz #################################
+authz:
+  # Specifies the authz provider to use.
+  provider: openfga
+  openfga:
+    # maximum tuples allowed per openfga write operation.
+    max_tuples_per_write: 100

docs/api/openapi.yml

@@ -668,8 +668,8 @@ components:
       properties:
         aws:
           $ref: '#/components/schemas/CloudintegrationtypesAWSAccountConfig'
-      required:
-      - aws
+        azure:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureAccountConfig'
       type: object
     CloudintegrationtypesAgentReport:
       nullable: true
@@ -693,6 +693,90 @@ components:
           nullable: true
           type: array
       type: object
+    CloudintegrationtypesAzureAccountConfig:
+      properties:
+        deploymentRegion:
+          type: string
+        resourceGroups:
+          items:
+            type: string
+          type: array
+      required:
+      - deploymentRegion
+      - resourceGroups
+      type: object
+    CloudintegrationtypesAzureConnectionArtifact:
+      properties:
+        cliCommand:
+          type: string
+        cloudPowerShellCommand:
+          type: string
+      required:
+      - cliCommand
+      - cloudPowerShellCommand
+      type: object
+    CloudintegrationtypesAzureIntegrationConfig:
+      properties:
+        deploymentRegion:
+          type: string
+        resourceGroups:
+          items:
+            type: string
+          type: array
+        telemetryCollectionStrategy:
+          items:
+            $ref: '#/components/schemas/CloudintegrationtypesAzureTelemetryCollectionStrategy'
+          type: array
+      required:
+      - deploymentRegion
+      - resourceGroups
+      - telemetryCollectionStrategy
+      type: object
+    CloudintegrationtypesAzureLogsCollectionStrategy:
+      properties:
+        categoryGroups:
+          items:
+            type: string
+          type: array
+      required:
+      - categoryGroups
+      type: object
+    CloudintegrationtypesAzureMetricsCollectionStrategy:
+      type: object
+    CloudintegrationtypesAzureServiceConfig:
+      properties:
+        logs:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureServiceLogsConfig'
+        metrics:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureServiceMetricsConfig'
+      required:
+      - logs
+      - metrics
+      type: object
+    CloudintegrationtypesAzureServiceLogsConfig:
+      properties:
+        enabled:
+          type: boolean
+      type: object
+    CloudintegrationtypesAzureServiceMetricsConfig:
+      properties:
+        enabled:
+          type: boolean
+      type: object
+    CloudintegrationtypesAzureTelemetryCollectionStrategy:
+      properties:
+        logs:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureLogsCollectionStrategy'
+        metrics:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureMetricsCollectionStrategy'
+        resourceProvider:
+          type: string
+        resourceType:
+          type: string
+      required:
+      - resourceProvider
+      - resourceType
+      type: object
     CloudintegrationtypesCloudIntegrationService:
       nullable: true
       properties:
@@ -737,8 +821,8 @@ components:
       properties:
         aws:
           $ref: '#/components/schemas/CloudintegrationtypesAWSConnectionArtifact'
-      required:
-      - aws
+        azure:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureConnectionArtifact'
       type: object
     CloudintegrationtypesCredentials:
       properties:
@@ -910,8 +994,8 @@ components:
       properties:
         aws:
           $ref: '#/components/schemas/CloudintegrationtypesAWSPostableAccountConfig'
-      required:
-      - aws
+        azure:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureAccountConfig'
       type: object
     CloudintegrationtypesPostableAgentCheckIn:
       properties:
@@ -934,8 +1018,8 @@ components:
       properties:
         aws:
           $ref: '#/components/schemas/CloudintegrationtypesAWSIntegrationConfig'
-      required:
-      - aws
+        azure:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureIntegrationConfig'
       type: object
     CloudintegrationtypesService:
       properties:
@@ -972,8 +1056,8 @@ components:
       properties:
         aws:
           $ref: '#/components/schemas/CloudintegrationtypesAWSServiceConfig'
-      required:
-      - aws
+        azure:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureServiceConfig'
       type: object
     CloudintegrationtypesServiceID:
       enum:
@@ -990,6 +1074,8 @@ components:
       - s3sync
       - sns
       - sqs
+      - storageaccountsblob
+      - cdnprofile
       type: string
     CloudintegrationtypesServiceMetadata:
       properties:
@@ -1018,16 +1104,32 @@ components:
       properties:
         aws:
           $ref: '#/components/schemas/CloudintegrationtypesAWSTelemetryCollectionStrategy'
-      required:
-      - aws
+        azure:
+          $ref: '#/components/schemas/CloudintegrationtypesAzureTelemetryCollectionStrategy'
       type: object
     CloudintegrationtypesUpdatableAccount:
       properties:
         config:
-          $ref: '#/components/schemas/CloudintegrationtypesAccountConfig'
+          $ref: '#/components/schemas/CloudintegrationtypesUpdatableAccountConfig'
       required:
       - config
       type: object
+    CloudintegrationtypesUpdatableAccountConfig:
+      properties:
+        aws:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSAccountConfig'
+        azure:
+          $ref: '#/components/schemas/CloudintegrationtypesUpdatableAzureAccountConfig'
+      type: object
+    CloudintegrationtypesUpdatableAzureAccountConfig:
+      properties:
+        resourceGroups:
+          items:
+            type: string
+          type: array
+      required:
+      - resourceGroups
+      type: object
     CloudintegrationtypesUpdatableService:
       properties:
         config:

ee/authz/openfgaauthz/provider.go

@@ -20,20 +20,23 @@ import (
 )
 
 type provider struct {
-	pkgAuthzService authz.AuthZ
-	openfgaServer   *openfgaserver.Server
-	licensing       licensing.Licensing
-	store           authtypes.RoleStore
-	registry        []authz.RegisterTypeable
+	config             authz.Config
+	pkgAuthzService    authz.AuthZ
+	openfgaServer      *openfgaserver.Server
+	licensing          licensing.Licensing
+	store              authtypes.RoleStore
+	registry           []authz.RegisterTypeable
+	settings           factory.ScopedProviderSettings
+	onBeforeRoleDelete []authz.OnBeforeRoleDelete
 }
 
-func NewProviderFactory(sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, licensing licensing.Licensing, registry ...authz.RegisterTypeable) factory.ProviderFactory[authz.AuthZ, authz.Config] {
+func NewProviderFactory(sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete, registry ...authz.RegisterTypeable) factory.ProviderFactory[authz.AuthZ, authz.Config] {
 	return factory.NewProviderFactory(factory.MustNewName("openfga"), func(ctx context.Context, ps factory.ProviderSettings, config authz.Config) (authz.AuthZ, error) {
-		return newOpenfgaProvider(ctx, ps, config, sqlstore, openfgaSchema, openfgaDataStore, licensing, registry)
+		return newOpenfgaProvider(ctx, ps, config, sqlstore, openfgaSchema, openfgaDataStore, licensing, onBeforeRoleDelete, registry)
 	})
 }
 
-func newOpenfgaProvider(ctx context.Context, settings factory.ProviderSettings, config authz.Config, sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, licensing licensing.Licensing, registry []authz.RegisterTypeable) (authz.AuthZ, error) {
+func newOpenfgaProvider(ctx context.Context, settings factory.ProviderSettings, config authz.Config, sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete, registry []authz.RegisterTypeable) (authz.AuthZ, error) {
 	pkgOpenfgaAuthzProvider := pkgopenfgaauthz.NewProviderFactory(sqlstore, openfgaSchema, openfgaDataStore)
 	pkgAuthzService, err := pkgOpenfgaAuthzProvider.New(ctx, settings, config)
 	if err != nil {
@@ -45,12 +48,17 @@ func newOpenfgaProvider(ctx context.Context, settings factory.ProviderSettings,
 		return nil, err
 	}
 
+	scopedSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/ee/authz/openfgaauthz")
+
 	return &provider{
-		pkgAuthzService: pkgAuthzService,
-		openfgaServer:   openfgaServer,
-		licensing:       licensing,
-		store:           sqlauthzstore.NewSqlAuthzStore(sqlstore),
-		registry:        registry,
+		config:             config,
+		pkgAuthzService:    pkgAuthzService,
+		openfgaServer:      openfgaServer,
+		licensing:          licensing,
+		store:              sqlauthzstore.NewSqlAuthzStore(sqlstore),
+		registry:           registry,
+		settings:           scopedSettings,
+		onBeforeRoleDelete: onBeforeRoleDelete,
 	}, nil
 }
 
@@ -78,14 +86,18 @@ func (provider *provider) BatchCheck(ctx context.Context, tupleReq map[string]*o
 	return provider.openfgaServer.BatchCheck(ctx, tupleReq)
 }
 
-func (provider *provider) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, typeable authtypes.Typeable) ([]*authtypes.Object, error) {
-	return provider.openfgaServer.ListObjects(ctx, subject, relation, typeable)
+func (provider *provider) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType authtypes.Type) ([]*authtypes.Object, error) {
+	return provider.openfgaServer.ListObjects(ctx, subject, relation, objectType)
 }
 
 func (provider *provider) Write(ctx context.Context, additions []*openfgav1.TupleKey, deletions []*openfgav1.TupleKey) error {
 	return provider.openfgaServer.Write(ctx, additions, deletions)
 }
 
+func (provider *provider) ReadTuples(ctx context.Context, tupleKey *openfgav1.ReadRequestTupleKey) ([]*openfgav1.TupleKey, error) {
+	return provider.openfgaServer.ReadTuples(ctx, tupleKey)
+}
+
 func (provider *provider) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID) (*authtypes.Role, error) {
 	return provider.pkgAuthzService.Get(ctx, orgID, id)
 }
@@ -146,7 +158,7 @@ func (provider *provider) Create(ctx context.Context, orgID valuer.UUID, role *a
 		return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
 	}
 
-	return provider.store.Create(ctx, authtypes.NewStorableRoleFromRole(role))
+	return provider.store.Create(ctx, role)
 }
 
 func (provider *provider) GetOrCreate(ctx context.Context, orgID valuer.UUID, role *authtypes.Role) (*authtypes.Role, error) {
@@ -163,10 +175,10 @@ func (provider *provider) GetOrCreate(ctx context.Context, orgID valuer.UUID, ro
 	}
 
 	if existingRole != nil {
-		return authtypes.NewRoleFromStorableRole(existingRole), nil
+		return existingRole, nil
 	}
 
-	err = provider.store.Create(ctx, authtypes.NewStorableRoleFromRole(role))
+	err = provider.store.Create(ctx, role)
 	if err != nil {
 		return nil, err
 	}
@@ -175,14 +187,13 @@ func (provider *provider) GetOrCreate(ctx context.Context, orgID valuer.UUID, ro
 }
 
 func (provider *provider) GetResources(_ context.Context) []*authtypes.Resource {
-	typeables := make([]authtypes.Typeable, 0)
-	for _, register := range provider.registry {
-		typeables = append(typeables, register.MustGetTypeables()...)
-	}
-
-	typeables = append(typeables, provider.MustGetTypeables()...)
 	resources := make([]*authtypes.Resource, 0)
-	for _, typeable := range typeables {
+	for _, register := range provider.registry {
+		for _, typeable := range register.MustGetTypeables() {
+			resources = append(resources, &authtypes.Resource{Name: typeable.Name(), Type: typeable.Type()})
+		}
+	}
+	for _, typeable := range provider.MustGetTypeables() {
 		resources = append(resources, &authtypes.Resource{Name: typeable.Name(), Type: typeable.Type()})
 	}
 
@@ -201,21 +212,23 @@ func (provider *provider) GetObjects(ctx context.Context, orgID valuer.UUID, id
 	}
 
 	objects := make([]*authtypes.Object, 0)
-	for _, resource := range provider.GetResources(ctx) {
-		if slices.Contains(authtypes.TypeableRelations[resource.Type], relation) {
-			resourceObjects, err := provider.
-				ListObjects(
-					ctx,
-					authtypes.MustNewSubject(authtypes.TypeableRole, storableRole.Name, orgID, &authtypes.RelationAssignee),
-					relation,
-					authtypes.MustNewTypeableFromType(resource.Type, resource.Name),
-				)
-			if err != nil {
-				return nil, err
-			}
-
-			objects = append(objects, resourceObjects...)
+	for _, objectType := range provider.getUniqueTypes() {
+		if !slices.Contains(authtypes.TypeableRelations[objectType], relation) {
+			continue
 		}
+
+		resourceObjects, err := provider.
+			ListObjects(
+				ctx,
+				authtypes.MustNewSubject(authtypes.TypeableRole, storableRole.Name, orgID, &authtypes.RelationAssignee),
+				relation,
+				objectType,
+			)
+		if err != nil {
+			return nil, err
+		}
+
+		objects = append(objects, resourceObjects...)
 	}
 
 	return objects, nil
@@ -227,7 +240,7 @@ func (provider *provider) Patch(ctx context.Context, orgID valuer.UUID, role *au
 		return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
 	}
 
-	return provider.store.Update(ctx, orgID, authtypes.NewStorableRoleFromRole(role))
+	return provider.store.Update(ctx, orgID, role)
 }
 
 func (provider *provider) PatchObjects(ctx context.Context, orgID valuer.UUID, name string, relation authtypes.Relation, additions, deletions []*authtypes.Object) error {
@@ -260,17 +273,26 @@ func (provider *provider) Delete(ctx context.Context, orgID valuer.UUID, id valu
 		return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
 	}
 
-	storableRole, err := provider.store.Get(ctx, orgID, id)
+	role, err := provider.store.Get(ctx, orgID, id)
 	if err != nil {
 		return err
 	}
 
-	role := authtypes.NewRoleFromStorableRole(storableRole)
 	err = role.ErrIfManaged()
 	if err != nil {
 		return err
 	}
 
+	for _, cb := range provider.onBeforeRoleDelete {
+		if err := cb(ctx, orgID, id); err != nil {
+			return err
+		}
+	}
+
+	if err := provider.deleteTuples(ctx, role.Name, orgID); err != nil {
+		return errors.WithAdditionalf(err, "failed to delete tuples for the role: %s", role.Name)
+	}
+
 	return provider.store.Delete(ctx, orgID, id)
 }
 
@@ -346,3 +368,62 @@ func (provider *provider) getManagedRoleTransactionTuples(orgID valuer.UUID) ([]
 
 	return tuples, nil
 }
+
+func (provider *provider) deleteTuples(ctx context.Context, roleName string, orgID valuer.UUID) error {
+	subject := authtypes.MustNewSubject(authtypes.TypeableRole, roleName, orgID, &authtypes.RelationAssignee)
+
+	tuples := make([]*openfgav1.TupleKey, 0)
+	for _, objectType := range provider.getUniqueTypes() {
+		typeTuples, err := provider.ReadTuples(ctx, &openfgav1.ReadRequestTupleKey{
+			User:   subject,
+			Object: objectType.StringValue() + ":",
+		})
+		if err != nil {
+			return err
+		}
+		tuples = append(tuples, typeTuples...)
+	}
+
+	if len(tuples) == 0 {
+		return nil
+	}
+
+	for idx := 0; idx < len(tuples); idx += provider.config.OpenFGA.MaxTuplesPerWrite {
+		end := idx + provider.config.OpenFGA.MaxTuplesPerWrite
+		if end > len(tuples) {
+			end = len(tuples)
+		}
+
+		err := provider.Write(ctx, nil, tuples[idx:end])
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (provider *provider) getUniqueTypes() []authtypes.Type {
+	seen := make(map[string]struct{})
+	uniqueTypes := make([]authtypes.Type, 0)
+	for _, register := range provider.registry {
+		for _, typeable := range register.MustGetTypeables() {
+			typeKey := typeable.Type().StringValue()
+			if _, ok := seen[typeKey]; ok {
+				continue
+			}
+			seen[typeKey] = struct{}{}
+			uniqueTypes = append(uniqueTypes, typeable.Type())
+		}
+	}
+	for _, typeable := range provider.MustGetTypeables() {
+		typeKey := typeable.Type().StringValue()
+		if _, ok := seen[typeKey]; ok {
+			continue
+		}
+		seen[typeKey] = struct{}{}
+		uniqueTypes = append(uniqueTypes, typeable.Type())
+	}
+
+	return uniqueTypes
+}

ee/authz/openfgaserver/server.go

@@ -110,10 +110,14 @@ func (server *Server) BatchCheck(ctx context.Context, tupleReq map[string]*openf
 	return server.pkgAuthzService.BatchCheck(ctx, tupleReq)
 }
 
-func (server *Server) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, typeable authtypes.Typeable) ([]*authtypes.Object, error) {
-	return server.pkgAuthzService.ListObjects(ctx, subject, relation, typeable)
+func (server *Server) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType authtypes.Type) ([]*authtypes.Object, error) {
+	return server.pkgAuthzService.ListObjects(ctx, subject, relation, objectType)
 }
 
 func (server *Server) Write(ctx context.Context, additions []*openfgav1.TupleKey, deletions []*openfgav1.TupleKey) error {
 	return server.pkgAuthzService.Write(ctx, additions, deletions)
 }
+
+func (server *Server) ReadTuples(ctx context.Context, tupleKey *openfgav1.ReadRequestTupleKey) ([]*openfgav1.TupleKey, error) {
+	return server.pkgAuthzService.ReadTuples(ctx, tupleKey)
+}

ee/query-service/rules/anomaly.go

@@ -277,8 +277,23 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 
 		annotations := make(ruletypes.Labels, 0, len(r.Annotations().Map()))
 		for name, value := range r.Annotations().Map() {
+			// no need to expand custom templating annotations — they get expanded in the notifier layer
+			if ruletypes.IsCustomTemplatingAnnotation(name) {
+				annotations = append(annotations, ruletypes.Label{Name: name, Value: value})
+				continue
+			}
 			annotations = append(annotations, ruletypes.Label{Name: name, Value: expand(value)})
 		}
+		// Add values to be used in notifier layer for notification templates
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationValue, Value: value})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationThresholdValue, Value: threshold})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationCompareOp, Value: smpl.CompareOperator.Literal()})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationMatchType, Value: smpl.MatchType.Literal()})
+
+		if smpl.IsRecovering {
+			lb.Set(ruletypes.LabelIsRecovering, "true")
+		}
+
 		if smpl.IsMissing {
 			lb.Set(ruletypes.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(ruletypes.NoDataLabel, "true")

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -795,7 +795,8 @@ export interface CloudintegrationtypesAccountDTO {
 }
 
 export interface CloudintegrationtypesAccountConfigDTO {
-	aws: CloudintegrationtypesAWSAccountConfigDTO;
+	aws?: CloudintegrationtypesAWSAccountConfigDTO;
+	azure?: CloudintegrationtypesAzureAccountConfigDTO;
 }
 
 /**
@@ -829,6 +830,86 @@ export interface CloudintegrationtypesAssetsDTO {
 	dashboards?: CloudintegrationtypesDashboardDTO[] | null;
 }
 
+export interface CloudintegrationtypesAzureAccountConfigDTO {
+	/**
+	 * @type string
+	 */
+	deploymentRegion: string;
+	/**
+	 * @type array
+	 */
+	resourceGroups: string[];
+}
+
+export interface CloudintegrationtypesAzureConnectionArtifactDTO {
+	/**
+	 * @type string
+	 */
+	cliCommand: string;
+	/**
+	 * @type string
+	 */
+	cloudPowerShellCommand: string;
+}
+
+export interface CloudintegrationtypesAzureIntegrationConfigDTO {
+	/**
+	 * @type string
+	 */
+	deploymentRegion: string;
+	/**
+	 * @type array
+	 */
+	resourceGroups: string[];
+	/**
+	 * @type array
+	 */
+	telemetryCollectionStrategy: CloudintegrationtypesAzureTelemetryCollectionStrategyDTO[];
+}
+
+export interface CloudintegrationtypesAzureLogsCollectionStrategyDTO {
+	/**
+	 * @type array
+	 */
+	categoryGroups: string[];
+}
+
+export interface CloudintegrationtypesAzureMetricsCollectionStrategyDTO {
+	[key: string]: unknown;
+}
+
+export interface CloudintegrationtypesAzureServiceConfigDTO {
+	logs: CloudintegrationtypesAzureServiceLogsConfigDTO;
+	metrics: CloudintegrationtypesAzureServiceMetricsConfigDTO;
+}
+
+export interface CloudintegrationtypesAzureServiceLogsConfigDTO {
+	/**
+	 * @type boolean
+	 */
+	enabled?: boolean;
+}
+
+export interface CloudintegrationtypesAzureServiceMetricsConfigDTO {
+	/**
+	 * @type boolean
+	 */
+	enabled?: boolean;
+}
+
+export interface CloudintegrationtypesAzureTelemetryCollectionStrategyDTO {
+	logs?: CloudintegrationtypesAzureLogsCollectionStrategyDTO;
+	metrics?: CloudintegrationtypesAzureMetricsCollectionStrategyDTO;
+	/**
+	 * @type string
+	 */
+	resourceProvider: string;
+	/**
+	 * @type string
+	 */
+	resourceType: string;
+}
+
 /**
  * @nullable
  */
@@ -890,7 +971,8 @@ export interface CloudintegrationtypesCollectedMetricDTO {
 }
 
 export interface CloudintegrationtypesConnectionArtifactDTO {
-	aws: CloudintegrationtypesAWSConnectionArtifactDTO;
+	aws?: CloudintegrationtypesAWSConnectionArtifactDTO;
+	azure?: CloudintegrationtypesAzureConnectionArtifactDTO;
 }
 
 export interface CloudintegrationtypesCredentialsDTO {
@@ -1074,7 +1156,8 @@ export interface CloudintegrationtypesPostableAccountDTO {
 }
 
 export interface CloudintegrationtypesPostableAccountConfigDTO {
-	aws: CloudintegrationtypesAWSPostableAccountConfigDTO;
+	aws?: CloudintegrationtypesAWSPostableAccountConfigDTO;
+	azure?: CloudintegrationtypesAzureAccountConfigDTO;
 }
 
 /**
@@ -1109,7 +1192,8 @@ export interface CloudintegrationtypesPostableAgentCheckInDTO {
 }
 
 export interface CloudintegrationtypesProviderIntegrationConfigDTO {
-	aws: CloudintegrationtypesAWSIntegrationConfigDTO;
+	aws?: CloudintegrationtypesAWSIntegrationConfigDTO;
+	azure?: CloudintegrationtypesAzureIntegrationConfigDTO;
 }
 
 export interface CloudintegrationtypesServiceDTO {
@@ -1137,7 +1221,8 @@ export interface CloudintegrationtypesServiceDTO {
 }
 
 export interface CloudintegrationtypesServiceConfigDTO {
-	aws: CloudintegrationtypesAWSServiceConfigDTO;
+	aws?: CloudintegrationtypesAWSServiceConfigDTO;
+	azure?: CloudintegrationtypesAzureServiceConfigDTO;
 }
 
 export enum CloudintegrationtypesServiceIDDTO {
@@ -1154,6 +1239,8 @@ export enum CloudintegrationtypesServiceIDDTO {
 	s3sync = 's3sync',
 	sns = 'sns',
 	sqs = 'sqs',
+	storageaccountsblob = 'storageaccountsblob',
+	cdnprofile = 'cdnprofile',
 }
 export interface CloudintegrationtypesServiceMetadataDTO {
 	/**
@@ -1186,11 +1273,24 @@ export interface CloudintegrationtypesSupportedSignalsDTO {
 }
 
 export interface CloudintegrationtypesTelemetryCollectionStrategyDTO {
-	aws: CloudintegrationtypesAWSTelemetryCollectionStrategyDTO;
+	aws?: CloudintegrationtypesAWSTelemetryCollectionStrategyDTO;
+	azure?: CloudintegrationtypesAzureTelemetryCollectionStrategyDTO;
 }
 
 export interface CloudintegrationtypesUpdatableAccountDTO {
-	config: CloudintegrationtypesAccountConfigDTO;
+	config: CloudintegrationtypesUpdatableAccountConfigDTO;
+}
+
+export interface CloudintegrationtypesUpdatableAccountConfigDTO {
+	aws?: CloudintegrationtypesAWSAccountConfigDTO;
+	azure?: CloudintegrationtypesUpdatableAzureAccountConfigDTO;
+}
+
+export interface CloudintegrationtypesUpdatableAzureAccountConfigDTO {
+	/**
+	 * @type array
+	 */
+	resourceGroups: string[];
 }
 
 export interface CloudintegrationtypesUpdatableServiceDTO {

frontend/src/container/FormAlertRules/ChartPreview/index.tsx

@@ -138,8 +138,8 @@ function ChartPreview({
 		if (startTime && endTime && startTime !== endTime) {
 			dispatch(
 				UpdateTimeInterval('custom', [
-					parseInt(getTimeString(startTime), 10),
-					parseInt(getTimeString(endTime), 10),
+					Number.parseInt(getTimeString(startTime), 10),
+					Number.parseInt(getTimeString(endTime), 10),
 				]),
 			);
 		}

frontend/src/container/FormAlertRules/index.tsx

@@ -369,7 +369,7 @@ function FormAlertRules({
 	// onQueryCategoryChange handles changes to query category
 	// in state as well as sets additional defaults
 	const onQueryCategoryChange = (val: EQueryType): void => {
-		const element = document.getElementById('top');
+		const element = document.querySelector('#top');
 		if (element) {
 			element.scrollIntoView({ behavior: 'smooth' });
 		}

frontend/src/container/MetricsExplorer/Explorer/Explorer.tsx

@@ -280,7 +280,7 @@ function Explorer(): JSX.Element {
 		[],
 	);
 
-	const [warning, setWarning] = useState<Warning | undefined>(undefined);
+	const [warning, setWarning] = useState<Warning | undefined>();
 
 	const oneChartPerQueryDisabledTooltip = useMemo(() => {
 		if (splitedQueries.length <= 1) {
@@ -292,7 +292,7 @@ function Explorer(): JSX.Element {
 		if (disableOneChartPerQuery) {
 			return 'One chart per query cannot be disabled for multiple queries with different units.';
 		}
-		return undefined;
+		return;
 	}, [disableOneChartPerQuery, splitedQueries.length, units.length]);
 
 	// Show the y axis unit selector if -

frontend/src/container/MetricsExplorer/Inspect/Inspect.tsx

@@ -221,7 +221,7 @@ function Inspect({
 			);
 		}
 
-		if (!inspectMetricsTimeSeries.length) {
+		if (inspectMetricsTimeSeries.length === 0) {
 			return renderFallback(
 				'inspect-metrics-empty',
 				<Empty description="No time series found for this metric to inspect." />,

frontend/src/container/MetricsExplorer/Inspect/useInspectMetrics.ts

@@ -256,10 +256,10 @@ export function useInspectMetrics(
 			const valuesMap = new Map<number, number>();
 
 			series.values.forEach(({ timestamp, value }) => {
-				valuesMap.set(timestamp, parseFloat(value));
+				valuesMap.set(timestamp, Number.parseFloat(value));
 			});
 
-			return timestamps.map((timestamp) => valuesMap.get(timestamp) ?? NaN);
+			return timestamps.map((timestamp) => valuesMap.get(timestamp) ?? Number.NaN);
 		});
 
 		const rawData = [timestamps, ...timeseriesArray];
@@ -273,7 +273,7 @@ export function useInspectMetrics(
 				labels.add(label);
 			});
 		});
-		return Array.from(labels);
+		return [...labels];
 	}, [inspectMetricsData]);
 
 	const reset = useCallback(() => {

frontend/src/container/QueryBuilder/components/RunQueryBtn/__test__/RunQueryBtn.test.tsx

@@ -18,7 +18,7 @@ describe('RunQueryBtn', () => {
 		);
 	});
 
-	test('renders run state and triggers on click', async () => {
+	it('renders run state and triggers on click', async () => {
 		const user = userEvent.setup();
 		const onRun = jest.fn();
 		const onCancel = jest.fn();
@@ -35,7 +35,7 @@ describe('RunQueryBtn', () => {
 		expect(onRun).toHaveBeenCalledTimes(1);
 	});
 
-	test('shows cancel state and calls handleCancelQuery', async () => {
+	it('shows cancel state and calls handleCancelQuery', async () => {
 		const user = userEvent.setup();
 		const onRun = jest.fn();
 		const onCancel = jest.fn();
@@ -51,19 +51,19 @@ describe('RunQueryBtn', () => {
 		expect(onCancel).toHaveBeenCalledTimes(1);
 	});
 
-	test('disabled when disabled prop is true', () => {
+	it('disabled when disabled prop is true', () => {
 		render(<RunQueryBtn disabled />);
 		expect(screen.getByRole('button', { name: /run query/i })).toBeDisabled();
 	});
 
-	test('disabled when no props provided', () => {
+	it('disabled when no props provided', () => {
 		render(<RunQueryBtn />);
 		expect(
 			screen.getByRole('button', { name: /run query/i }),
 		).toBeInTheDocument();
 	});
 
-	test('shows Command + CornerDownLeft on mac', () => {
+	it('shows Command + CornerDownLeft on mac', () => {
 		const { container } = render(
 			<RunQueryBtn
 				onStageRunQuery={jest.fn()}
@@ -77,7 +77,7 @@ describe('RunQueryBtn', () => {
 		).toBeInTheDocument();
 	});
 
-	test('shows ChevronUp + CornerDownLeft on non-mac', () => {
+	it('shows ChevronUp + CornerDownLeft on non-mac', () => {
 		(getUserOperatingSystem as jest.Mock).mockReturnValue(
 			UserOperatingSystem.WINDOWS,
 		);
@@ -95,7 +95,7 @@ describe('RunQueryBtn', () => {
 		).toBeInTheDocument();
 	});
 
-	test('renders custom label when provided', () => {
+	it('renders custom label when provided', () => {
 		render(
 			<RunQueryBtn
 				onStageRunQuery={jest.fn()}

frontend/src/hooks/integration/aws/useIntegrationModal.ts

@@ -164,7 +164,8 @@ export function useIntegrationModal({
 				{
 					onSuccess: (response: CreateAccountMutationResult) => {
 						const accountId = response.data.id;
-						const connectionUrl = response.data.connectionArtifact.aws.connectionUrl;
+						const connectionUrl =
+							response.data.connectionArtifact.aws?.connectionUrl ?? '';
 
 						logEvent(
 							'AWS Integration: Account connection attempt redirected to AWS',

frontend/src/hooks/queryBuilder/useGetQueryRange.ts

@@ -115,8 +115,8 @@ export const useGetQueryRange: UseGetQueryRange = (
 
 			const updatedQuery = updateBarStepInterval(
 				requestData.query,
-				requestData.start ? requestData.start * 1e3 : parseInt(start, 10) * 1e3,
-				requestData.end ? requestData.end * 1e3 : parseInt(end, 10) * 1e3,
+				requestData.start ? requestData.start * 1e3 : Number.parseInt(start, 10) * 1e3,
+				requestData.end ? requestData.end * 1e3 : Number.parseInt(end, 10) * 1e3,
 			);
 
 			return {

frontend/src/lib/uPlotV2/components/Tooltip/components/TooltipHeader/TooltipHeader.module.scss

@@ -1,21 +1,23 @@
 .headerContainer {
-	padding: 1rem 1rem 0 1rem;
 	display: flex;
 	flex-direction: column;
-	gap: var(--spacing-4);
 
-	&:last-child {
-		padding-bottom: var(--spacing-4);
-	}
 }
 
+
 .headerRow {
+	padding: var(--spacing-8) var(--spacing-8) 0 var(--spacing-8);
 	font-size: var(--font-size-sm);
 	font-weight: 500;
 	display: flex;
 	align-items: center;
 	justify-content: space-between;
 	gap: var(--spacing-2);
+	margin-bottom: var(--spacing-2);
+}
+
+.pinnedItem {
+	padding: var(--spacing-4) var(--spacing-4) 0 var(--spacing-4);
 }
 
 .status {

frontend/src/lib/uPlotV2/components/Tooltip/components/TooltipHeader/TooltipHeader.tsx

@@ -72,13 +72,15 @@ export default function TooltipHeader({
 			)}
 
 			{activeItem && (
-				<TooltipItem
-					item={activeItem}
-					isItemActive={true}
-					containerTestId="uplot-tooltip-pinned"
-					markerTestId="uplot-tooltip-pinned-marker"
-					contentTestId="uplot-tooltip-pinned-content"
-				/>
+				<div className={Styles.pinnedItem} data-testid="uplot-tooltip-pinned-item">
+					<TooltipItem
+						item={activeItem}
+						isItemActive={true}
+						containerTestId="uplot-tooltip-pinned"
+						markerTestId="uplot-tooltip-pinned-marker"
+						contentTestId="uplot-tooltip-pinned-content"
+					/>
+				</div>
 			)}
 		</div>
 	);

frontend/src/lib/uPlotV2/components/Tooltip/components/TooltipItem/TooltipItem.module.scss

@@ -1,36 +1,51 @@
-.uplot-tooltip-item {
+.uplotTooltipItem {
 	display: flex;
 	align-items: center;
-	gap: 8px;
-	padding: 4px 0;
+	gap: var(--spacing-2);
+	padding: 6px;
+	cursor: pointer;
+	opacity: 0.7;
+	font-weight: 400;
 
-	.uplot-tooltip-item-marker {
-		border-radius: 50%;
-		border-style: solid;
-		border-width: 2px;
-		width: 12px;
-		height: 12px;
-		flex-shrink: 0;
+	&[data-is-active='true'] {
+		opacity: 1;
+		font-weight: 700;
 	}
 
-	.uplot-tooltip-item-content {
-		width: 100%;
-		display: flex;
-		align-items: center;
-		gap: 8px;
-		justify-content: space-between;
-
-		.uplot-tooltip-item-label {
-			white-space: normal;
-			overflow-wrap: anywhere;
-		}
-
-		&-separator {
-			flex: 1;
-			border-width: 0.5px;
-			border-style: dashed;
-			min-width: 24px;
-			opacity: 0.5;
-		}
+	&:hover {
+		opacity: 1 !important;
+		background-color: var(--l2-border);
+		border-radius: 4px;
 	}
 }
+
+.uplotTooltipItemMarker {
+	border-radius: 50%;
+	border-style: solid;
+	border-width: 2px;
+	width: 12px;
+	height: 12px;
+	box-sizing: border-box;
+	flex-shrink: 0;
+}
+
+.uplotTooltipItemContent {
+	width: 100%;
+	display: flex;
+	align-items: center;
+	gap: var(--spacing-2);
+	justify-content: space-between;
+}
+
+.uplotTooltipItemLabel {
+	white-space: normal;
+	overflow-wrap: anywhere;
+}
+
+.uplotTooltipItemContentSeparator {
+	flex: 1;
+	border-width: 0.5px;
+	border-style: dashed;
+	min-width: 24px;
+	opacity: 0.5;
+}

frontend/src/lib/uPlotV2/components/Tooltip/components/TooltipItem/TooltipItem.tsx

@@ -20,10 +20,7 @@ export default function TooltipItem({
 	return (
 		<div
 			className={Styles.uplotTooltipItem}
-			style={{
-				opacity: isItemActive ? 1 : 0.7,
-				fontWeight: isItemActive ? 700 : 400,
-			}}
+			data-is-active={isItemActive}
 			data-testid={containerTestId}
 		>
 			<div

frontend/src/lib/uPlotV2/components/Tooltip/components/TooltipList/TooltipList.module.scss

@@ -3,7 +3,11 @@
 	:global(div[data-viewport-type='element']) {
 		left: 0;
 		box-sizing: border-box;
-		padding: 4px 12px 4px 16px;
+		padding: 0px var(--spacing-2) 0 var(--spacing-4);
+
+		[data-test-id='virtuoso-item-list'] > * + * {
+			margin-top: var(--spacing-2);
+		}
 	}
 
 	&::-webkit-scrollbar {

frontend/src/pages/LogsExplorer/index.tsx

@@ -98,7 +98,7 @@ function LogsExplorer(): JSX.Element {
 		setIsLoadingQueries(false);
 	}, [queryClient]);
 
-	const [warning, setWarning] = useState<Warning | undefined>(undefined);
+	const [warning, setWarning] = useState<Warning | undefined>();
 
 	const handleChangeSelectedView = useCallback(
 		(view: ExplorerViews, querySearchParameters?: ICurrentQueryData): void => {

frontend/src/pages/TracesExplorer/index.tsx

@@ -101,7 +101,7 @@ function TracesExplorer(): JSX.Element {
 		getExplorerViewFromUrl(searchParams, panelTypesFromUrl),
 	);
 
-	const [warning, setWarning] = useState<Warning | undefined>(undefined);
+	const [warning, setWarning] = useState<Warning | undefined>();
 	const [isOpen, setOpen] = useState<boolean>(true);
 
 	const defaultQuery = useMemo(

pkg/alertmanager/alertmanagernotify/email/email.go

@@ -23,7 +23,14 @@ import (
 	"sync"
 	"time"
 
+	htmltemplate "html/template"
+
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	commoncfg "github.com/prometheus/common/config"
 
 	"github.com/prometheus/alertmanager/config"
@@ -38,16 +45,30 @@ const (
 
 // Email implements a Notifier for email notifications.
 type Email struct {
-	conf     *config.EmailConfig
-	tmpl     *template.Template
-	logger   *slog.Logger
-	hostname string
+	conf          *config.EmailConfig
+	tmpl          *template.Template
+	logger        *slog.Logger
+	hostname      string
+	templater     alertmanagertypes.Templater
+	templateStore emailtypes.TemplateStore
+}
+
+// layoutData is the value passed to the alert_email_notification layout
+// template. It embeds NotificationTemplateData so templates can reference
+// `.Alert.Status`, `.Alert.TotalFiring`, `.Alert.TotalResolved`,
+// `.NotificationTemplateData.ExternalURL`, etc. alongside the rendered
+// Title and per-alert Bodies.
+type layoutData struct {
+	alertmanagertypes.NotificationTemplateData
+	Title  string
+	Bodies []htmltemplate.HTML
 }
 
 var errNoAuthUsernameConfigured = errors.NewInternalf(errors.CodeInternal, "no auth username configured")
 
-// New returns a new Email notifier.
-func New(c *config.EmailConfig, t *template.Template, l *slog.Logger) *Email {
+// New returns a new Email notifier. templateStore may be nil; in that case
+// custom-body alerts fall back to plain <div>-wrapped HTML.
+func New(c *config.EmailConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater, templateStore emailtypes.TemplateStore) *Email {
 	if _, ok := c.Headers["Subject"]; !ok {
 		c.Headers["Subject"] = config.DefaultEmailSubject
 	}
@@ -63,7 +84,7 @@ func New(c *config.EmailConfig, t *template.Template, l *slog.Logger) *Email {
 	if err != nil {
 		h = "localhost.localdomain"
 	}
-	return &Email{conf: c, tmpl: t, logger: l, hostname: h}
+	return &Email{conf: c, tmpl: t, logger: l, hostname: h, templater: templater, templateStore: templateStore}
 }
 
 // auth resolves a string of authentication mechanisms.
@@ -245,6 +266,16 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 		}
 	}
 
+	// Prepare the content for the email. customSubject, when non-empty, overrides
+	// the configured Subject header for this notification only. We deliberately
+	// do not mutate n.conf.Headers here: the config map is shared across
+	// concurrent notifications to the same receiver.
+	customSubject, htmlBody, err := n.prepareContent(ctx, as, data)
+	if err != nil {
+		n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
+		return false, err
+	}
+
 	// Send the email headers and body.
 	message, err := c.Data()
 	if err != nil {
@@ -262,6 +293,10 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 
 	buffer := &bytes.Buffer{}
 	for header, t := range n.conf.Headers {
+		if header == "Subject" && customSubject != "" {
+			fmt.Fprintf(buffer, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", customSubject))
+			continue
+		}
 		value, err := n.tmpl.ExecuteTextString(t, data)
 		if err != nil {
 			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute %q header template", header)
@@ -336,7 +371,7 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 		}
 	}
 
-	if len(n.conf.HTML) > 0 {
+	if htmlBody != "" {
 		// Html template
 		// Preferred alternative placed last per section 5.1.4 of RFC 2046
 		// https://www.ietf.org/rfc/rfc2046.txt
@@ -347,12 +382,8 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 		if err != nil {
 			return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for html template")
 		}
-		body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
-		if err != nil {
-			return false, errors.WrapInternalf(err, errors.CodeInternal, "execute html template")
-		}
 		qw := quotedprintable.NewWriter(w)
-		_, err = qw.Write([]byte(body))
+		_, err = qw.Write([]byte(htmlBody))
 		if err != nil {
 			return true, errors.WrapInternalf(err, errors.CodeInternal, "write HTML part")
 		}
@@ -381,6 +412,146 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 	return false, nil
 }
 
+// prepareContent returns a subject override (empty when the default config
+// Subject should be used) and the HTML body for the email. Callers must treat
+// the subject as local state and never write it back to n.conf.Headers.
+func (n *Email) prepareContent(ctx context.Context, alerts []*types.Alert, data *template.Data) (string, string, error) {
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: n.conf.Headers["Subject"],
+		// The email body's default path uses the configured HTML template
+		// verbatim (n.conf.HTML below). Leave DefaultBodyTemplate empty so
+		// the templater produces a single empty default body we ignore.
+		DefaultBodyTemplate: "",
+	}, alerts)
+	if err != nil {
+		return "", "", err
+	}
+
+	// subject == "" signals "use the configured Subject header"; Notify then
+	// runs it through the normal header template pipeline.
+	subject := ""
+	if customTitle != "" {
+		subject = result.Title
+	}
+
+	if !result.IsDefaultBody {
+		// Custom-body path: render each expanded markdown body to HTML, then
+		// wrap the whole thing in the alert_email_notification layout (or fall
+		// back to plain <div> wrapping when the template store is absent).
+		for i, body := range result.Body {
+			if body == "" {
+				continue
+			}
+			rendered, err := markdownrenderer.RenderHTML(body)
+			if err != nil {
+				return "", "", err
+			}
+			result.Body[i] = rendered
+		}
+		appendRelatedLinkButtons(alerts, result.Body)
+		html, err := n.renderLayout(ctx, result)
+		if err != nil {
+			n.logger.WarnContext(ctx, "custom email template rendering failed, falling back to plain <div> wrap", errors.Attr(err))
+			return subject, wrapBodiesAsDivs(result.Body), nil
+		}
+		return subject, html, nil
+	}
+
+	// Default-body path: use the HTML config template if available.
+	if len(n.conf.HTML) > 0 {
+		body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
+		if err != nil {
+			return "", "", errors.WrapInternalf(err, errors.CodeInternal, "execute html template")
+		}
+		return subject, body, nil
+	}
+	return subject, "", nil
+}
+
+// renderLayout wraps result in the alert_email_notification HTML layout.
+// Returns an error when the store has no such template (e.g. in tests using
+// an empty store) so prepareContent can fall back to plain <div> wrapping.
+func (n *Email) renderLayout(ctx context.Context, result *alertmanagertypes.ExpandResult) (string, error) {
+	if n.templateStore == nil {
+		return "", errors.NewInternalf(errors.CodeInternal, "no email template store configured")
+	}
+	layout, err := n.templateStore.Get(ctx, emailtypes.TemplateNameAlertEmailNotification)
+	if err != nil {
+		return "", err
+	}
+	bodies := make([]htmltemplate.HTML, 0, len(result.Body))
+	for _, b := range result.Body {
+		bodies = append(bodies, htmltemplate.HTML(b))
+	}
+	data := layoutData{Title: result.Title, Bodies: bodies}
+	if result.NotificationData != nil {
+		data.NotificationTemplateData = *result.NotificationData
+	}
+	var buf bytes.Buffer
+	if err := layout.Execute(&buf, data); err != nil {
+		return "", errors.WrapInternalf(err, errors.CodeInternal, "failed to render email layout")
+	}
+	return buf.String(), nil
+}
+
+// appendRelatedLinkButtons appends "View Related Logs/Traces" buttons to each
+// per-alert body when the rule manager attached the corresponding annotation.
+// bodies is positionally aligned with alerts (see alertmanagertemplate.Prepare);
+// empty bodies are skipped so we never attach a button to an alert that produced
+// no visible content.
+func appendRelatedLinkButtons(alerts []*types.Alert, bodies []string) {
+	for i := range bodies {
+		if i >= len(alerts) || bodies[i] == "" {
+			continue
+		}
+		if link := alerts[i].Annotations[ruletypes.AnnotationRelatedLogs]; link != "" {
+			bodies[i] += htmlButton("View Related Logs", string(link))
+		}
+		if link := alerts[i].Annotations[ruletypes.AnnotationRelatedTraces]; link != "" {
+			bodies[i] += htmlButton("View Related Traces", string(link))
+		}
+	}
+}
+
+func wrapBodiesAsDivs(bodies []string) string {
+	var b strings.Builder
+	for _, part := range bodies {
+		if part == "" {
+			continue
+		}
+		b.WriteString("<div>")
+		b.WriteString(part)
+		b.WriteString("</div>")
+	}
+	return b.String()
+}
+
+func htmlButton(text, url string) string {
+	return fmt.Sprintf(`
+	<a href="%s" target="_blank" style="text-decoration: none;">
+<button style="
+    padding: 6px 16px;
+    /* Default System Font */
+    font-family: sans-serif;
+    font-size: 14px;
+    font-weight: 500;
+    line-height: 1.5;
+    /* Light Theme & Dynamic Background (Solid) */
+    color: #111827;
+    background-color: #f9fafb;
+    /* Static Outline */
+    border: 1px solid #d1d5db;
+    border-radius: 4px;
+    cursor: pointer;
+">
+  %s
+</button>
+</a>`, url, text)
+}
+
 type loginAuth struct {
 	username, password string
 }

pkg/alertmanager/alertmanagernotify/email/email_test.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"log/slog"
 	"net"
 	"net/http"
 	"net/url"
@@ -17,7 +18,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/emailing/templatestore/filetemplatestore"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	"github.com/emersion/go-smtp"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
@@ -42,6 +48,18 @@ const (
 	emailFrom = "alertmanager@example.com"
 )
 
+// testTemplater returns a Templater bound to tmpl with a discard logger.
+func testTemplater(tmpl *template.Template) alertmanagertypes.Templater {
+	return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
+}
+
+// testEmptyStore returns an email template store with no templates. When the
+// email notifier tries to render the alert_email_notification layout against
+// this, the Get call fails and prepareContent falls back to plain <div> wrap.
+func testEmptyStore() emailtypes.TemplateStore {
+	return filetemplatestore.NewEmptyStore()
+}
+
 // email represents an email returned by the MailDev REST API.
 // See https://github.com/djfarrelly/MailDev/blob/master/docs/rest.md.
 type email struct {
@@ -162,7 +180,7 @@ func notifyEmailWithContext(ctx context.Context, t *testing.T, cfg *config.Email
 		return nil, false, err
 	}
 
-	email := New(cfg, tmpl, promslog.NewNopLogger())
+	email := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl), testEmptyStore())
 
 	retry, err := email.Notify(ctx, firingAlert)
 	if err != nil {
@@ -706,7 +724,7 @@ func TestEmailRejected(t *testing.T) {
 	tmpl, firingAlert, err := prepare(cfg)
 	require.NoError(t, err)
 
-	e := New(cfg, tmpl, promslog.NewNopLogger())
+	e := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl), testEmptyStore())
 
 	// Send the alert to mock SMTP server.
 	retry, err := e.Notify(context.Background(), firingAlert)
@@ -1030,6 +1048,103 @@ func TestEmailImplicitTLS(t *testing.T) {
 	}
 }
 
+func TestPrepareContent(t *testing.T) {
+	t.Run("custom template", func(t *testing.T) {
+		tmpl, err := template.FromGlobs([]string{})
+		require.NoError(t, err)
+		tmpl.ExternalURL, _ = url.Parse("http://am")
+
+		bodyTpl := "line $labels.instance"
+		a1 := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					model.LabelName("instance"): model.LabelValue("one"),
+				},
+				Annotations: model.LabelSet{
+					model.LabelName(ruletypes.AnnotationBodyTemplate): model.LabelValue(bodyTpl),
+				},
+			},
+		}
+		a2 := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					model.LabelName("instance"): model.LabelValue("two"),
+				},
+				Annotations: model.LabelSet{
+					model.LabelName(ruletypes.AnnotationBodyTemplate): model.LabelValue(bodyTpl),
+				},
+			},
+		}
+		alerts := []*types.Alert{a1, a2}
+
+		cfg := &config.EmailConfig{Headers: map[string]string{"Subject": "subj"}}
+		n := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl), testEmptyStore())
+
+		ctx := context.Background()
+		data := notify.GetTemplateData(ctx, tmpl, alerts, n.logger)
+		_, htmlBody, err := n.prepareContent(ctx, alerts, data)
+		require.NoError(t, err)
+		require.Equal(t, "<div><p>line one</p>\n</div><div><p>line two</p>\n</div>", htmlBody)
+	})
+
+	t.Run("default template with HTML and custom title template", func(t *testing.T) {
+		tmpl, err := template.FromGlobs([]string{})
+		require.NoError(t, err)
+		tmpl.ExternalURL, _ = url.Parse("http://am")
+
+		firingAlert := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{},
+				Annotations: model.LabelSet{
+					model.LabelName(ruletypes.AnnotationTitleTemplate): model.LabelValue("fixed from $alert.status"),
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		}
+		alerts := []*types.Alert{firingAlert}
+		cfg := &config.EmailConfig{
+			Headers: map[string]string{},
+			HTML:    "Status: {{ .Status }}",
+		}
+		n := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl), testEmptyStore())
+
+		ctx := context.Background()
+		data := notify.GetTemplateData(ctx, tmpl, alerts, n.logger)
+		subject, htmlBody, err := n.prepareContent(ctx, alerts, data)
+		require.NoError(t, err)
+		require.Equal(t, "Status: firing", htmlBody)
+		// custom title supplied → prepareContent returns a subject override.
+		require.Equal(t, "fixed from firing", subject)
+	})
+
+	t.Run("default template without HTML", func(t *testing.T) {
+		cfg := &config.EmailConfig{Headers: map[string]string{"Subject": "the email subject"}}
+		tmpl, err := template.FromGlobs([]string{})
+		require.NoError(t, err)
+		tmpl.ExternalURL, _ = url.Parse("http://am")
+		n := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl), testEmptyStore())
+
+		firingAlert := &types.Alert{
+			Alert: model.Alert{
+				Labels:   model.LabelSet{},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		}
+		alerts := []*types.Alert{firingAlert}
+		ctx := context.Background()
+		data := notify.GetTemplateData(ctx, tmpl, alerts, n.logger)
+		subject, htmlBody, err := n.prepareContent(ctx, alerts, data)
+		require.NoError(t, err)
+		require.Equal(t, "", htmlBody)
+		// No custom title annotation → empty subject signals "use the configured
+		// Subject header", which Notify then runs through the normal header
+		// template pipeline.
+		require.Equal(t, "", subject)
+	})
+}
+
 func ptrTo(b bool) *bool {
 	return &b
 }

pkg/alertmanager/alertmanagernotify/msteamsv2/msteamsv2.go

@@ -15,7 +15,9 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -44,6 +46,7 @@ type Notifier struct {
 	retrier      *notify.Retrier
 	webhookURL   *config.SecretURL
 	postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
+	templater    alertmanagertypes.Templater
 }
 
 // https://learn.microsoft.com/en-us/connectors/teams/?tabs=text1#adaptivecarditemschema
@@ -94,7 +97,7 @@ type teamsMessage struct {
 }
 
 // New returns a new notifier that uses the Microsoft Teams Power Platform connector.
-func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
 	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
@@ -109,6 +112,7 @@ func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *s
 		retrier:      &notify.Retrier{},
 		webhookURL:   c.WebhookURL,
 		postJSONFunc: notify.PostJSON,
+		templater:    templater,
 	}
 
 	return n, nil
@@ -128,25 +132,11 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		return false, err
 	}
 
-	title := tmpl(n.conf.Title)
-	if err != nil {
-		return false, err
-	}
-
 	titleLink := tmpl(n.titleLink)
 	if err != nil {
 		return false, err
 	}
 
-	alerts := types.Alerts(as...)
-	color := colorGrey
-	switch alerts.Status() {
-	case model.AlertFiring:
-		color = colorRed
-	case model.AlertResolved:
-		color = colorGreen
-	}
-
 	var url string
 	if n.conf.WebhookURL != nil {
 		url = n.conf.WebhookURL.String()
@@ -158,6 +148,12 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		url = strings.TrimSpace(string(content))
 	}
 
+	bodyBlocks, err := n.prepareContent(ctx, as)
+	if err != nil {
+		n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
+		return false, err
+	}
+
 	// A message as referenced in https://learn.microsoft.com/en-us/connectors/teams/?tabs=text1%2Cdotnet#request-body-schema
 	t := teamsMessage{
 		Type: "message",
@@ -169,17 +165,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 					Schema:  "http://adaptivecards.io/schemas/adaptive-card.json",
 					Type:    "AdaptiveCard",
 					Version: "1.2",
-					Body: []Body{
-						{
-							Type:   "TextBlock",
-							Text:   title,
-							Weight: "Bolder",
-							Size:   "Medium",
-							Wrap:   true,
-							Style:  "heading",
-							Color:  color,
-						},
-					},
+					Body:    bodyBlocks,
 					Actions: []Action{
 						{
 							Type:  "Action.OpenUrl",
@@ -195,20 +181,6 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		},
 	}
 
-	// add labels and annotations to the body of all alerts
-	for _, alert := range as {
-		t.Attachments[0].Content.Body = append(t.Attachments[0].Content.Body, Body{
-			Type:   "TextBlock",
-			Text:   "Alerts",
-			Weight: "Bolder",
-			Size:   "Medium",
-			Wrap:   true,
-			Color:  color,
-		})
-
-		t.Attachments[0].Content.Body = append(t.Attachments[0].Content.Body, n.createLabelsAndAnnotationsBody(alert)...)
-	}
-
 	var payload bytes.Buffer
 	if err = json.NewEncoder(&payload).Encode(t); err != nil {
 		return false, err
@@ -228,6 +200,77 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	return shouldRetry, err
 }
 
+// prepareContent builds the Adaptive Card body blocks for the notification.
+// The first block is always the title; the remainder depends on whether the
+// alerts carried a custom body template.
+func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert) ([]Body, error) {
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: n.conf.Title,
+		// Default body is not a template — it's built per-alert below from
+		// labels/annotations as Adaptive Card FactSets, so leave it empty.
+		DefaultBodyTemplate: "",
+	}, alerts)
+	if err != nil {
+		return nil, err
+	}
+
+	color := colorGrey
+	switch types.Alerts(alerts...).Status() {
+	case model.AlertFiring:
+		color = colorRed
+	case model.AlertResolved:
+		color = colorGreen
+	}
+
+	blocks := []Body{{
+		Type:   "TextBlock",
+		Text:   result.Title,
+		Weight: "Bolder",
+		Size:   "Medium",
+		Wrap:   true,
+		Style:  "heading",
+		Color:  color,
+	}}
+
+	if result.IsDefaultBody {
+		for _, alert := range alerts {
+			blocks = append(blocks, Body{
+				Type:   "TextBlock",
+				Text:   "Alerts",
+				Weight: "Bolder",
+				Size:   "Medium",
+				Wrap:   true,
+				Color:  color,
+			})
+			blocks = append(blocks, n.createLabelsAndAnnotationsBody(alert)...)
+		}
+		return blocks, nil
+	}
+
+	// Custom body path: result.Body is positionally aligned with alerts;
+	// entries for alerts whose template rendered empty are kept as "" so we
+	// can skip them here without shifting the per-alert color index.
+	for i, body := range result.Body {
+		if body == "" || i >= len(alerts) {
+			continue
+		}
+		perAlertColor := colorRed
+		if alerts[i].Resolved() {
+			perAlertColor = colorGreen
+		}
+		blocks = append(blocks, Body{
+			Type:  "TextBlock",
+			Text:  body,
+			Wrap:  true,
+			Color: perAlertColor,
+		})
+	}
+	return blocks, nil
+}
+
 func (*Notifier) createLabelsAndAnnotationsBody(alert *types.Alert) []Body {
 	bodies := []Body{}
 	bodies = append(bodies, Body{
@@ -258,7 +301,11 @@ func (*Notifier) createLabelsAndAnnotationsBody(alert *types.Alert) []Body {
 
 	annotationsFacts := []Fact{}
 	for k, v := range alert.Annotations {
-		if slices.Contains([]string{"summary", "related_logs", "related_traces"}, string(k)) {
+		// Skip private (`_`-prefixed) annotations — templating inputs,
+		// threshold metadata, related-link URLs — and "summary", which default
+		// channels surface as the attachment pretext rather than a fact row.
+		if slices.Contains([]string{"summary", "related_logs", "related_traces"}, string(k)) ||
+			alertmanagertypes.IsPrivateAnnotation(string(k)) {
 			continue
 		}
 		annotationsFacts = append(annotationsFacts, Fact{Title: string(k), Value: string(v)})

pkg/alertmanager/alertmanagernotify/msteamsv2/msteamsv2_test.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -15,6 +16,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
@@ -23,21 +27,28 @@ import (
 	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
 	"github.com/prometheus/alertmanager/types"
 )
 
+func newTestTemplater(tmpl *template.Template) alertmanagertypes.Templater {
+	return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
+}
+
 // This is a test URL that has been modified to not be valid.
 var testWebhookURL, _ = url.Parse("https://example.westeurope.logic.azure.com:443/workflows/xxx/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=xxx")
 
 func TestMSTeamsV2Retry(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.MSTeamsV2Config{
 			WebhookURL: &config.SecretURL{URL: testWebhookURL},
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		`{{ template "msteamsv2.default.titleLink" . }}`,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -64,14 +75,16 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			tmpl := test.CreateTmpl(t)
 			notifier, err := New(
 				&config.MSTeamsV2Config{
 					WebhookURL: &config.SecretURL{URL: testWebhookURL},
 					HTTPConfig: &commoncfg.HTTPClientConfig{},
 				},
-				test.CreateTmpl(t),
+				tmpl,
 				`{{ template "msteamsv2.default.titleLink" . }}`,
 				promslog.NewNopLogger(),
+				newTestTemplater(tmpl),
 			)
 			require.NoError(t, err)
 
@@ -153,7 +166,8 @@ func TestMSTeamsV2Templating(t *testing.T) {
 		t.Run(tc.title, func(t *testing.T) {
 			tc.cfg.WebhookURL = &config.SecretURL{URL: u}
 			tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
-			pd, err := New(tc.cfg, test.CreateTmpl(t), tc.titleLink, promslog.NewNopLogger())
+			tmpl := test.CreateTmpl(t)
+			pd, err := New(tc.cfg, tmpl, tc.titleLink, promslog.NewNopLogger(), newTestTemplater(tmpl))
 			require.NoError(t, err)
 
 			ctx := context.Background()
@@ -186,20 +200,124 @@ func TestMSTeamsV2RedactedURL(t *testing.T) {
 	defer fn()
 
 	secret := "secret"
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.MSTeamsV2Config{
 			WebhookURL: &config.SecretURL{URL: u},
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		`{{ template "msteamsv2.default.titleLink" . }}`,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
 	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, secret)
 }
 
+func TestPrepareContent(t *testing.T) {
+	t.Run("default template - firing alerts", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		notifier, err := New(
+			&config.MSTeamsV2Config{
+				WebhookURL: &config.SecretURL{URL: testWebhookURL},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+				Title:      "Alertname: {{ .CommonLabels.alertname }}",
+			},
+			tmpl,
+			`{{ template "msteamsv2.default.titleLink" . }}`,
+			promslog.NewNopLogger(),
+			newTestTemplater(tmpl),
+		)
+		require.NoError(t, err)
+
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{"alertname": "test"},
+					// Custom body template
+					Annotations: model.LabelSet{
+						ruletypes.AnnotationBodyTemplate: "Firing alert: $alertname",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+		}
+		blocks, err := notifier.prepareContent(ctx, alerts)
+		require.NoError(t, err)
+		require.NotEmpty(t, blocks)
+		// First block should be the title with color (firing = red)
+		require.Equal(t, "Bolder", blocks[0].Weight)
+		require.Equal(t, colorRed, blocks[0].Color)
+		// verify title text
+		require.Equal(t, "Alertname: test", blocks[0].Text)
+		// verify body text
+		require.Equal(t, "Firing alert: test", blocks[1].Text)
+	})
+
+	t.Run("custom template - per-alert color", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		notifier, err := New(
+			&config.MSTeamsV2Config{
+				WebhookURL: &config.SecretURL{URL: testWebhookURL},
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			tmpl,
+			`{{ template "msteamsv2.default.titleLink" . }}`,
+			promslog.NewNopLogger(),
+			newTestTemplater(tmpl),
+		)
+		require.NoError(t, err)
+
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{"alertname": "test1"},
+					Annotations: model.LabelSet{
+						"summary":                         "test",
+						ruletypes.AnnotationTitleTemplate: "Custom Title",
+						ruletypes.AnnotationBodyTemplate:  "custom body $alertname",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{"alertname": "test2"},
+					Annotations: model.LabelSet{
+						"summary":                         "test",
+						ruletypes.AnnotationTitleTemplate: "Custom Title",
+						ruletypes.AnnotationBodyTemplate:  "custom body $alertname",
+					},
+					StartsAt: time.Now().Add(-time.Hour),
+					EndsAt:   time.Now().Add(-time.Minute),
+				},
+			},
+		}
+		blocks, err := notifier.prepareContent(ctx, alerts)
+		require.NoError(t, err)
+		require.NotEmpty(t, blocks)
+		// total 3 blocks: title and 2 body blocks
+		require.True(t, len(blocks) == 3)
+		// First block: title color is overall color of the alerts
+		require.Equal(t, colorRed, blocks[0].Color)
+		// verify title text
+		require.Equal(t, "Custom Title", blocks[0].Text)
+		// Body blocks should have per-alert color
+		require.Equal(t, colorRed, blocks[1].Color)   // firing
+		require.Equal(t, colorGreen, blocks[2].Color) // resolved
+	})
+}
+
 func TestMSTeamsV2ReadingURLFromFile(t *testing.T) {
 	ctx, u, fn := test.GetContextWithCancelingURL()
 	defer fn()
@@ -209,14 +327,16 @@ func TestMSTeamsV2ReadingURLFromFile(t *testing.T) {
 	_, err = f.WriteString(u.String() + "\n")
 	require.NoError(t, err, "writing to temp file failed")
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.MSTeamsV2Config{
 			WebhookURLFile: f.Name(),
 			HTTPConfig:     &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		`{{ template "msteamsv2.default.titleLink" . }}`,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie.go

@@ -15,7 +15,10 @@ import (
 	"os"
 	"strings"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 
@@ -34,25 +37,27 @@ const maxMessageLenRunes = 130
 
 // Notifier implements a Notifier for OpsGenie notifications.
 type Notifier struct {
-	conf    *config.OpsGenieConfig
-	tmpl    *template.Template
-	logger  *slog.Logger
-	client  *http.Client
-	retrier *notify.Retrier
+	conf      *config.OpsGenieConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	client    *http.Client
+	retrier   *notify.Retrier
+	templater alertmanagertypes.Templater
 }
 
 // New returns a new OpsGenie notifier.
-func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
 	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
 	}
 	return &Notifier{
-		conf:    c,
-		tmpl:    t,
-		logger:  l,
-		client:  client,
-		retrier: &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
+		conf:      c,
+		tmpl:      t,
+		logger:    l,
+		client:    client,
+		retrier:   &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
+		templater: templater,
 	}, nil
 }
 
@@ -123,6 +128,55 @@ func safeSplit(s, sep string) []string {
 	return b
 }
 
+// prepareContent expands alert templates and returns the OpsGenie-ready title
+// (truncated to the 130-rune limit) and HTML description. Custom bodies are
+// rendered to HTML and stitched together with <hr> dividers; default bodies
+// are joined with newlines (OpsGenie's legacy plain-text description).
+func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert) (string, string, error) {
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: n.conf.Message,
+		DefaultBodyTemplate:  n.conf.Description,
+	}, alerts)
+	if err != nil {
+		return "", "", err
+	}
+
+	var description string
+	if result.IsDefaultBody {
+		description = strings.Join(result.Body, "\n")
+	} else {
+		var b strings.Builder
+		first := true
+		for _, part := range result.Body {
+			if part == "" {
+				continue
+			}
+			rendered, renderErr := markdownrenderer.RenderHTML(part)
+			if renderErr != nil {
+				return "", "", renderErr
+			}
+			if !first {
+				b.WriteString("<hr>")
+			}
+			b.WriteString("<div>")
+			b.WriteString(rendered)
+			b.WriteString("</div>")
+			first = false
+		}
+		description = b.String()
+	}
+
+	title, truncated := notify.TruncateInRunes(result.Title, maxMessageLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated message", slog.Int("max_runes", maxMessageLenRunes))
+	}
+
+	return title, description, nil
+}
+
 // Create requests for a list of alerts.
 func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) {
 	key, err := notify.ExtractGroupKey(ctx)
@@ -168,9 +222,10 @@ func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*h
 		}
 		requests = append(requests, req.WithContext(ctx))
 	default:
-		message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
-		if truncated {
-			logger.WarnContext(ctx, "Truncated message", slog.Any("alert", key), slog.Int("max_runes", maxMessageLenRunes))
+		message, description, err := n.prepareContent(ctx, as)
+		if err != nil {
+			n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
+			return nil, false, err
 		}
 
 		createEndpointURL := n.conf.APIURL.Copy()
@@ -209,7 +264,7 @@ func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*h
 		msg := &opsGenieCreateMessage{
 			Alias:       alias,
 			Message:     message,
-			Description: tmpl(n.conf.Description),
+			Description: description,
 			Details:     details,
 			Source:      tmpl(n.conf.Source),
 			Responders:  responders,

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie_test.go

@@ -8,12 +8,16 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"net/url"
 	"os"
 	"testing"
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
@@ -22,16 +26,23 @@ import (
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/notify/test"
+	"github.com/prometheus/alertmanager/template"
 	"github.com/prometheus/alertmanager/types"
 )
 
+func newTestTemplater(tmpl *template.Template) alertmanagertypes.Templater {
+	return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
+}
+
 func TestOpsGenieRetry(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.OpsGenieConfig{
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -47,14 +58,16 @@ func TestOpsGenieRedactedURL(t *testing.T) {
 	defer fn()
 
 	key := "key"
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.OpsGenieConfig{
 			APIURL:     &config.URL{URL: u},
 			APIKey:     config.Secret(key),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -72,14 +85,16 @@ func TestGettingOpsGegineApikeyFromFile(t *testing.T) {
 	_, err = f.WriteString(key)
 	require.NoError(t, err, "writing to temp file failed")
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.OpsGenieConfig{
 			APIURL:     &config.URL{URL: u},
 			APIKeyFile: f.Name(),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -202,7 +217,7 @@ func TestOpsGenie(t *testing.T) {
 		},
 	} {
 		t.Run(tc.title, func(t *testing.T) {
-			notifier, err := New(tc.cfg, tmpl, logger)
+			notifier, err := New(tc.cfg, tmpl, logger, newTestTemplater(tmpl))
 			require.NoError(t, err)
 
 			ctx := context.Background()
@@ -278,7 +293,7 @@ func TestOpsGenieWithUpdate(t *testing.T) {
 		APIURL:       &config.URL{URL: u},
 		HTTPConfig:   &commoncfg.HTTPClientConfig{},
 	}
-	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger(), newTestTemplater(tmpl))
 	alert := &types.Alert{
 		Alert: model.Alert{
 			StartsAt: time.Now(),
@@ -321,7 +336,7 @@ func TestOpsGenieApiKeyFile(t *testing.T) {
 		APIURL:     &config.URL{URL: u},
 		HTTPConfig: &commoncfg.HTTPClientConfig{},
 	}
-	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
+	notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger(), newTestTemplater(tmpl))
 
 	require.NoError(t, err)
 	requests, _, err := notifierWithUpdate.createRequests(ctx)
@@ -329,6 +344,99 @@ func TestOpsGenieApiKeyFile(t *testing.T) {
 	require.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
 }
 
+func TestPrepareContent(t *testing.T) {
+	t.Run("default template", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		logger := promslog.NewNopLogger()
+
+		notifier := &Notifier{
+			conf: &config.OpsGenieConfig{
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+			},
+			tmpl:      tmpl,
+			logger:    logger,
+			templater: newTestTemplater(tmpl),
+		}
+
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+
+		alert := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"Message":     "Firing alert: test",
+					"Description": "Check runbook for more details",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		}
+
+		alerts := []*types.Alert{alert}
+
+		title, desc, prepErr := notifier.prepareContent(ctx, alerts)
+		require.NoError(t, prepErr)
+		require.Equal(t, "Firing alert: test", title)
+		require.Equal(t, "Check runbook for more details", desc)
+	})
+
+	t.Run("custom template", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		logger := promslog.NewNopLogger()
+
+		notifier := &Notifier{
+			conf: &config.OpsGenieConfig{
+				Message:     `{{ .CommonLabels.Message }}`,
+				Description: `{{ .CommonLabels.Description }}`,
+			},
+			tmpl:      tmpl,
+			logger:    logger,
+			templater: newTestTemplater(tmpl),
+		}
+
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+
+		alert1 := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"service":   "payment",
+					"namespace": "potter-the-harry",
+				},
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: "High request throughput for $service",
+					ruletypes.AnnotationBodyTemplate:  "Alert firing in NS: $labels.namespace",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		}
+		alert2 := &types.Alert{
+			Alert: model.Alert{
+				Labels: model.LabelSet{
+					"service":   "payment",
+					"namespace": "smart-the-rat",
+				},
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: "High request throughput for $service",
+					ruletypes.AnnotationBodyTemplate:  "Alert firing in NS: $labels.namespace",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			},
+		}
+
+		alerts := []*types.Alert{alert1, alert2}
+
+		title, desc, err := notifier.prepareContent(ctx, alerts)
+		require.NoError(t, err)
+		require.Equal(t, "High request throughput for payment", title)
+		// Each alert body wrapped in <div>, separated by <hr>
+		require.Equal(t, "<div><p>Alert firing in NS: potter-the-harry</p>\n</div><hr><div><p>Alert firing in NS: smart-the-rat</p>\n</div>", desc)
+	})
+}
+
 func readBody(t *testing.T, r *http.Request) string {
 	t.Helper()
 	body, err := io.ReadAll(r.Body)

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty.go

@@ -15,7 +15,9 @@ import (
 	"os"
 	"strings"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/alecthomas/units"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
@@ -40,21 +42,22 @@ const (
 
 // Notifier implements a Notifier for PagerDuty notifications.
 type Notifier struct {
-	conf    *config.PagerdutyConfig
-	tmpl    *template.Template
-	logger  *slog.Logger
-	apiV1   string // for tests.
-	client  *http.Client
-	retrier *notify.Retrier
+	conf      *config.PagerdutyConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	apiV1     string // for tests.
+	client    *http.Client
+	retrier   *notify.Retrier
+	templater alertmanagertypes.Templater
 }
 
 // New returns a new PagerDuty notifier.
-func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
 	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
 	}
-	n := &Notifier{conf: c, tmpl: t, logger: l, client: client}
+	n := &Notifier{conf: c, tmpl: t, logger: l, client: client, templater: templater}
 	if c.ServiceKey != "" || c.ServiceKeyFile != "" {
 		n.apiV1 = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
 		// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
@@ -143,11 +146,12 @@ func (n *Notifier) notifyV1(
 	key notify.Key,
 	data *template.Data,
 	details map[string]any,
+	title string,
 ) (bool, error) {
 	var tmplErr error
 	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
 
-	description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes)
+	description, truncated := notify.TruncateInRunes(title, maxV1DescriptionLenRunes)
 	if truncated {
 		n.logger.WarnContext(ctx, "Truncated description", slog.Any("key", key), slog.Int("max_runes", maxV1DescriptionLenRunes))
 	}
@@ -203,6 +207,7 @@ func (n *Notifier) notifyV2(
 	key notify.Key,
 	data *template.Data,
 	details map[string]any,
+	title string,
 ) (bool, error) {
 	var tmplErr error
 	tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
@@ -211,7 +216,7 @@ func (n *Notifier) notifyV2(
 		n.conf.Severity = "error"
 	}
 
-	summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV2SummaryLenRunes)
+	summary, truncated := notify.TruncateInRunes(title, maxV2SummaryLenRunes)
 	if truncated {
 		n.logger.WarnContext(ctx, "Truncated summary", slog.Any("key", key), slog.Int("max_runes", maxV2SummaryLenRunes))
 	}
@@ -294,6 +299,55 @@ func (n *Notifier) notifyV2(
 	return retry, err
 }
 
+// prepareTitle expands the notification title. PagerDuty has no body surface
+// we care about — the description/summary field is what users see as the
+// incident headline, so we feed the configured Description as the default
+// title template and ignore any custom body_template entirely.
+func (n *Notifier) prepareTitle(ctx context.Context, alerts []*types.Alert) (string, error) {
+	customTitle, _ := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
+		TitleTemplate:        customTitle,
+		DefaultTitleTemplate: n.conf.Description,
+	}, alerts)
+	if err != nil {
+		return "", err
+	}
+	return result.Title, nil
+}
+
+// stripPrivateAnnotations returns a copy of alerts with every private (`_`-
+// prefixed) annotation removed. These keys — templating inputs, threshold
+// metadata, related-link URLs — are internal inputs to the notifier, not
+// content the external receiver should see. The original alert
+// objects are never mutated: they are shared with other receivers in the
+// fanout.
+func stripPrivateAnnotations(alerts []*types.Alert) []*types.Alert {
+	out := make([]*types.Alert, len(alerts))
+	for i, alert := range alerts {
+		hasPrivate := false
+		for k := range alert.Annotations {
+			if alertmanagertypes.IsPrivateAnnotation(string(k)) {
+				hasPrivate = true
+				break
+			}
+		}
+		if !hasPrivate {
+			out[i] = alert
+			continue
+		}
+		cloned := *alert
+		cloned.Annotations = make(model.LabelSet, len(alert.Annotations))
+		for k, v := range alert.Annotations {
+			if alertmanagertypes.IsPrivateAnnotation(string(k)) {
+				continue
+			}
+			cloned.Annotations[k] = v
+		}
+		out[i] = &cloned
+	}
+	return out
+}
+
 // Notify implements the Notifier interface.
 func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 	key, err := notify.ExtractGroupKey(ctx)
@@ -302,6 +356,16 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	}
 	logger := n.logger.With(slog.Any("group_key", key))
 
+	title, err := n.prepareTitle(ctx, as)
+	if err != nil {
+		n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
+		return false, err
+	}
+
+	// strip private annotations so they're not sent to the external receiver,
+	// pagerduty default details template contains labels and annotations.
+	as = stripPrivateAnnotations(as)
+
 	var (
 		alerts    = types.Alerts(as...)
 		data      = notify.GetTemplateData(ctx, n.tmpl, as, logger)
@@ -329,7 +393,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	if n.apiV1 != "" {
 		nf = n.notifyV1
 	}
-	retry, err := nf(ctx, eventType, key, data, details)
+	retry, err := nf(ctx, eventType, key, data, details, title)
 	if err != nil {
 		if ctx.Err() != nil {
 			err = errors.WrapInternalf(err, errors.CodeInternal, "failed to notify PagerDuty: %v", context.Cause(ctx))

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -17,7 +18,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
@@ -30,14 +34,20 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+func newTestTemplater(tmpl *template.Template) alertmanagertypes.Templater {
+	return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
+}
+
 func TestPagerDutyRetryV1(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.PagerdutyConfig{
 			ServiceKey: config.Secret("01234567890123456789012345678901"),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -49,13 +59,15 @@ func TestPagerDutyRetryV1(t *testing.T) {
 }
 
 func TestPagerDutyRetryV2(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.PagerdutyConfig{
 			RoutingKey: config.Secret("01234567890123456789012345678901"),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -71,13 +83,15 @@ func TestPagerDutyRedactedURLV1(t *testing.T) {
 	defer fn()
 
 	key := "01234567890123456789012345678901"
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.PagerdutyConfig{
 			ServiceKey: config.Secret(key),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 	notifier.apiV1 = u.String()
@@ -90,14 +104,16 @@ func TestPagerDutyRedactedURLV2(t *testing.T) {
 	defer fn()
 
 	key := "01234567890123456789012345678901"
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.PagerdutyConfig{
 			URL:        &config.URL{URL: u},
 			RoutingKey: config.Secret(key),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -114,13 +130,15 @@ func TestPagerDutyV1ServiceKeyFromFile(t *testing.T) {
 	ctx, u, fn := test.GetContextWithCancelingURL()
 	defer fn()
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.PagerdutyConfig{
 			ServiceKeyFile: f.Name(),
 			HTTPConfig:     &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 	notifier.apiV1 = u.String()
@@ -138,14 +156,16 @@ func TestPagerDutyV2RoutingKeyFromFile(t *testing.T) {
 	ctx, u, fn := test.GetContextWithCancelingURL()
 	defer fn()
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.PagerdutyConfig{
 			URL:            &config.URL{URL: u},
 			RoutingKeyFile: f.Name(),
 			HTTPConfig:     &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -302,7 +322,8 @@ func TestPagerDutyTemplating(t *testing.T) {
 		t.Run(tc.title, func(t *testing.T) {
 			tc.cfg.URL = &config.URL{URL: u}
 			tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
-			pd, err := New(tc.cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			tmpl := test.CreateTmpl(t)
+			pd, err := New(tc.cfg, tmpl, promslog.NewNopLogger(), newTestTemplater(tmpl))
 			require.NoError(t, err)
 			if pd.apiV1 != "" {
 				pd.apiV1 = u.String()
@@ -392,13 +413,15 @@ func TestEventSizeEnforcement(t *testing.T) {
 		Details:    bigDetailsV1,
 	}
 
+	tmpl := test.CreateTmpl(t)
 	notifierV1, err := New(
 		&config.PagerdutyConfig{
 			ServiceKey: config.Secret("01234567890123456789012345678901"),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -420,8 +443,9 @@ func TestEventSizeEnforcement(t *testing.T) {
 			RoutingKey: config.Secret("01234567890123456789012345678901"),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -536,7 +560,8 @@ func TestPagerDutyEmptySrcHref(t *testing.T) {
 		Links:      links,
 	}
 
-	pagerDuty, err := New(&pagerDutyConfig, test.CreateTmpl(t), promslog.NewNopLogger())
+	pdTmpl := test.CreateTmpl(t)
+	pagerDuty, err := New(&pagerDutyConfig, pdTmpl, promslog.NewNopLogger(), newTestTemplater(pdTmpl))
 	require.NoError(t, err)
 
 	ctx := context.Background()
@@ -603,7 +628,8 @@ func TestPagerDutyTimeout(t *testing.T) {
 				Timeout:    tt.timeout,
 			}
 
-			pd, err := New(&cfg, test.CreateTmpl(t), promslog.NewNopLogger())
+			tmpl := test.CreateTmpl(t)
+			pd, err := New(&cfg, tmpl, promslog.NewNopLogger(), newTestTemplater(tmpl))
 			require.NoError(t, err)
 
 			ctx := context.Background()
@@ -881,3 +907,132 @@ func TestRenderDetails(t *testing.T) {
 		})
 	}
 }
+
+func TestPrepareContent(t *testing.T) {
+	prepareContext := func() context.Context {
+		ctx := context.Background()
+		ctx = notify.WithGroupKey(ctx, "1")
+		ctx = notify.WithReceiverName(ctx, "test-receiver")
+		ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": "HighCPU for Payment service"})
+		return ctx
+	}
+	t.Run("default template uses go text template config for title", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		notifier, err := New(
+			&config.PagerdutyConfig{
+				RoutingKey:  config.Secret("01234567890123456789012345678901"),
+				HTTPConfig:  &commoncfg.HTTPClientConfig{},
+				Description: `{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})`,
+			},
+			tmpl,
+			promslog.NewNopLogger(),
+			newTestTemplater(tmpl),
+		)
+		require.NoError(t, err)
+
+		ctx := prepareContext()
+
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels:   model.LabelSet{"alertname": "HighCPU for Payment service"},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+		}
+
+		title, err := notifier.prepareTitle(ctx, alerts)
+		require.NoError(t, err)
+		require.Equal(t, "HighCPU for Payment service (FIRING)", title)
+	})
+
+	t.Run("custom template uses $variable annotation for title", func(t *testing.T) {
+		tmpl := test.CreateTmpl(t)
+		notifier, err := New(
+			&config.PagerdutyConfig{
+				RoutingKey: config.Secret("01234567890123456789012345678901"),
+				HTTPConfig: &commoncfg.HTTPClientConfig{},
+			},
+			tmpl,
+			promslog.NewNopLogger(),
+			newTestTemplater(tmpl),
+		)
+		require.NoError(t, err)
+
+		ctx := prepareContext()
+
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"alertname": "HighCPU",
+						"service":   "api-server",
+					},
+					Annotations: model.LabelSet{
+						ruletypes.AnnotationTitleTemplate: "$rule.name on $service is in $alert.status state",
+					},
+					StartsAt: time.Now().Add(-time.Hour),
+					EndsAt:   time.Now(),
+				},
+			},
+		}
+
+		title, err := notifier.prepareTitle(ctx, alerts)
+		require.NoError(t, err)
+		require.Equal(t, "HighCPU on api-server is in resolved state", title)
+	})
+}
+
+func TestStripPrivateAnnotations(t *testing.T) {
+	t.Run("template annotations are removed from the copy, originals untouched", func(t *testing.T) {
+		// The original alert objects are shared with other receivers in the
+		// fanout and must not be mutated. The rendered values should not be
+		// sent to the webhook receiver at all.
+		origTitle := model.LabelValue("Alert: $labels.alertname")
+		origBody := model.LabelValue("Severity is $labels.severity")
+		in := []*types.Alert{
+			{Alert: model.Alert{
+				Labels: model.LabelSet{"alertname": "TestAlert", "severity": "critical"},
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: origTitle,
+					ruletypes.AnnotationBodyTemplate:  origBody,
+					"summary":                         "keep this",
+				},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			}},
+		}
+
+		out := stripPrivateAnnotations(in)
+		require.Len(t, out, 1)
+
+		_, hasTitle := out[0].Annotations[ruletypes.AnnotationTitleTemplate]
+		_, hasBody := out[0].Annotations[ruletypes.AnnotationBodyTemplate]
+		require.False(t, hasTitle, "title_template should be stripped from outgoing payload")
+		require.False(t, hasBody, "body_template should be stripped from outgoing payload")
+		require.Equal(t, model.LabelValue("keep this"), out[0].Annotations["summary"])
+
+		// Original alert annotations remain so downstream receivers still see
+		// the raw templates.
+		require.Equal(t, origTitle, in[0].Annotations[ruletypes.AnnotationTitleTemplate])
+		require.Equal(t, origBody, in[0].Annotations[ruletypes.AnnotationBodyTemplate])
+	})
+
+	t.Run("alerts without template annotations are returned unchanged", func(t *testing.T) {
+		original := &types.Alert{Alert: model.Alert{
+			Labels: model.LabelSet{"alertname": "NoTemplateAlert"},
+			Annotations: model.LabelSet{
+				"summary": "keep this",
+			},
+			StartsAt: time.Now(),
+			EndsAt:   time.Now().Add(time.Hour),
+		}}
+
+		out := stripPrivateAnnotations([]*types.Alert{original})
+		require.Len(t, out, 1)
+		// Pass-through optimization: when there's nothing to strip, the
+		// original pointer is reused rather than deep-copying.
+		require.Same(t, original, out[0])
+	})
+}

pkg/alertmanager/alertmanagernotify/receiver.go

@@ -26,12 +26,15 @@ var customNotifierIntegrations = []string{
 	msteamsv2.Integration,
 }
 
-func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error) {
+func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger, deps alertmanagertypes.NotificationDeps) ([]notify.Integration, error) {
 	upstreamIntegrations, err := receiver.BuildReceiverIntegrations(nc, tmpl, logger)
 	if err != nil {
 		return nil, err
 	}
 
+	templater := deps.Templater
+	emailStore := deps.EmailTemplateStore
+
 	var (
 		errs         types.MultiError
 		integrations []notify.Integration
@@ -53,23 +56,25 @@ func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Templ
 	}
 
 	for i, c := range nc.WebhookConfigs {
-		add(webhook.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) })
+		add(webhook.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l, templater) })
 	}
 	for i, c := range nc.EmailConfigs {
-		add(email.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil })
+		add(email.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
+			return email.New(c, tmpl, l, templater, emailStore), nil
+		})
 	}
 	for i, c := range nc.PagerdutyConfigs {
-		add(pagerduty.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) })
+		add(pagerduty.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l, templater) })
 	}
 	for i, c := range nc.OpsGenieConfigs {
-		add(opsgenie.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) })
+		add(opsgenie.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l, templater) })
 	}
 	for i, c := range nc.SlackConfigs {
-		add(slack.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) })
+		add(slack.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l, templater) })
 	}
 	for i, c := range nc.MSTeamsV2Configs {
 		add(msteamsv2.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
-			return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l)
+			return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l, templater)
 		})
 	}
 

pkg/alertmanager/alertmanagernotify/slack/slack.go

@@ -14,7 +14,11 @@ import (
 	"os"
 	"strings"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	commoncfg "github.com/prometheus/common/config"
 
 	"github.com/prometheus/alertmanager/config"
@@ -25,6 +29,8 @@ import (
 
 const (
 	Integration = "slack"
+	colorRed    = "#FF0000"
+	colorGreen  = "#00FF00"
 )
 
 // https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
@@ -32,17 +38,18 @@ const maxTitleLenRunes = 1024
 
 // Notifier implements a Notifier for Slack notifications.
 type Notifier struct {
-	conf    *config.SlackConfig
-	tmpl    *template.Template
-	logger  *slog.Logger
-	client  *http.Client
-	retrier *notify.Retrier
+	conf      *config.SlackConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	client    *http.Client
+	retrier   *notify.Retrier
+	templater alertmanagertypes.Templater
 
 	postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
 }
 
 // New returns a new Slack notification handler.
-func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
 	client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
@@ -54,6 +61,7 @@ func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts .
 		logger:       l,
 		client:       client,
 		retrier:      &notify.Retrier{},
+		templater:    templater,
 		postJSONFunc: notify.PostJSON,
 	}, nil
 }
@@ -81,9 +89,10 @@ type attachment struct {
 	Actions    []config.SlackAction `json:"actions,omitempty"`
 	ImageURL   string               `json:"image_url,omitempty"`
 	ThumbURL   string               `json:"thumb_url,omitempty"`
-	Footer     string               `json:"footer"`
+	Footer     string               `json:"footer,omitempty"`
 	Color      string               `json:"color,omitempty"`
 	MrkdwnIn   []string             `json:"mrkdwn_in,omitempty"`
+	Blocks     []any                `json:"blocks,omitempty"`
 }
 
 // Notify implements the Notifier interface.
@@ -100,79 +109,15 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		data     = notify.GetTemplateData(ctx, n.tmpl, as, logger)
 		tmplText = notify.TmplText(n.tmpl, data, &err)
 	)
-	var markdownIn []string
 
-	if len(n.conf.MrkdwnIn) == 0 {
-		markdownIn = []string{"fallback", "pretext", "text"}
-	} else {
-		markdownIn = n.conf.MrkdwnIn
+	attachments, err := n.prepareContent(ctx, as, tmplText)
+	if err != nil {
+		n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
+		return false, err
 	}
 
-	title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), maxTitleLenRunes)
-	if truncated {
-		logger.WarnContext(ctx, "Truncated title", slog.Int("max_runes", maxTitleLenRunes))
-	}
-	att := &attachment{
-		Title:      title,
-		TitleLink:  tmplText(n.conf.TitleLink),
-		Pretext:    tmplText(n.conf.Pretext),
-		Text:       tmplText(n.conf.Text),
-		Fallback:   tmplText(n.conf.Fallback),
-		CallbackID: tmplText(n.conf.CallbackID),
-		ImageURL:   tmplText(n.conf.ImageURL),
-		ThumbURL:   tmplText(n.conf.ThumbURL),
-		Footer:     tmplText(n.conf.Footer),
-		Color:      tmplText(n.conf.Color),
-		MrkdwnIn:   markdownIn,
-	}
-
-	numFields := len(n.conf.Fields)
-	if numFields > 0 {
-		fields := make([]config.SlackField, numFields)
-		for index, field := range n.conf.Fields {
-			// Check if short was defined for the field otherwise fallback to the global setting
-			var short bool
-			if field.Short != nil {
-				short = *field.Short
-			} else {
-				short = n.conf.ShortFields
-			}
-
-			// Rebuild the field by executing any templates and setting the new value for short
-			fields[index] = config.SlackField{
-				Title: tmplText(field.Title),
-				Value: tmplText(field.Value),
-				Short: &short,
-			}
-		}
-		att.Fields = fields
-	}
-
-	numActions := len(n.conf.Actions)
-	if numActions > 0 {
-		actions := make([]config.SlackAction, numActions)
-		for index, action := range n.conf.Actions {
-			slackAction := config.SlackAction{
-				Type:  tmplText(action.Type),
-				Text:  tmplText(action.Text),
-				URL:   tmplText(action.URL),
-				Style: tmplText(action.Style),
-				Name:  tmplText(action.Name),
-				Value: tmplText(action.Value),
-			}
-
-			if action.ConfirmField != nil {
-				slackAction.ConfirmField = &config.SlackConfirmationField{
-					Title:       tmplText(action.ConfirmField.Title),
-					Text:        tmplText(action.ConfirmField.Text),
-					OkText:      tmplText(action.ConfirmField.OkText),
-					DismissText: tmplText(action.ConfirmField.DismissText),
-				}
-			}
-
-			actions[index] = slackAction
-		}
-		att.Actions = actions
+	if len(attachments) > 0 {
+		n.addFieldsAndActions(&attachments[0], tmplText)
 	}
 
 	req := &request{
@@ -182,7 +127,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 		IconURL:     tmplText(n.conf.IconURL),
 		LinkNames:   n.conf.LinkNames,
 		Text:        tmplText(n.conf.MessageText),
-		Attachments: []attachment{*att},
+		Attachments: attachments,
 	}
 	if err != nil {
 		return false, err
@@ -238,6 +183,153 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	return retry, nil
 }
 
+// prepareContent expands alert templates and returns the Slack attachment(s)
+// ready to send. When alerts carry a custom body template, one title-only
+// attachment plus one body attachment per alert is returned so that each alert
+// can get its own firing/resolved color and per-alert action buttons.
+func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert, tmplText func(string) string) ([]attachment, error) {
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
+		TitleTemplate:        customTitle,
+		BodyTemplate:         customBody,
+		DefaultTitleTemplate: n.conf.Title,
+		// The default path renders the attachment body using the configured
+		// Text template (and Slack's own mrkdwn flavour) below, so the default
+		// body template here is left empty.
+		DefaultBodyTemplate: "",
+	}, alerts)
+	if err != nil {
+		return nil, err
+	}
+
+	title, truncated := notify.TruncateInRunes(result.Title, maxTitleLenRunes)
+	if truncated {
+		n.logger.WarnContext(ctx, "Truncated title", slog.Int("max_runes", maxTitleLenRunes))
+	}
+
+	if result.IsDefaultBody {
+		var markdownIn []string
+		if len(n.conf.MrkdwnIn) == 0 {
+			markdownIn = []string{"fallback", "pretext", "text"}
+		} else {
+			markdownIn = n.conf.MrkdwnIn
+		}
+		return []attachment{
+			{
+				Title:      title,
+				TitleLink:  tmplText(n.conf.TitleLink),
+				Pretext:    tmplText(n.conf.Pretext),
+				Text:       tmplText(n.conf.Text),
+				Fallback:   tmplText(n.conf.Fallback),
+				CallbackID: tmplText(n.conf.CallbackID),
+				ImageURL:   tmplText(n.conf.ImageURL),
+				ThumbURL:   tmplText(n.conf.ThumbURL),
+				Footer:     tmplText(n.conf.Footer),
+				Color:      tmplText(n.conf.Color),
+				MrkdwnIn:   markdownIn,
+			},
+		}, nil
+	}
+
+	// Custom template path: one title attachment + one attachment per
+	// non-empty alert body. result.Body is positionally aligned with alerts,
+	// so we index alerts[i] directly and skip empty entries.
+	attachments := make([]attachment, 0, 1+len(result.Body))
+	attachments = append(attachments, attachment{
+		Title:     title,
+		TitleLink: tmplText(n.conf.TitleLink),
+	})
+
+	for i, body := range result.Body {
+		if body == "" || i >= len(alerts) {
+			continue
+		}
+
+		// Custom bodies are authored in markdown; render each non-empty body to
+		// Slack's mrkdwn flavour. Default bodies skip this because the Text
+		// template is already channel-ready.
+		rendered, renderErr := markdownrenderer.RenderSlackMrkdwn(body)
+		if renderErr != nil {
+			return nil, renderErr
+		}
+
+		color := colorRed
+		if alerts[i].Resolved() {
+			color = colorGreen
+		}
+		attachments = append(attachments, attachment{
+			Text:     rendered,
+			Color:    color,
+			MrkdwnIn: []string{"text"},
+			Actions:  buildRelatedLinkActions(alerts[i]),
+		})
+	}
+
+	return attachments, nil
+}
+
+// buildRelatedLinkActions returns the "View Related Logs/Traces" action
+// buttons for an alert, or nil when no related-link annotations are present.
+func buildRelatedLinkActions(alert *types.Alert) []config.SlackAction {
+	var actions []config.SlackAction
+	if link := alert.Annotations[ruletypes.AnnotationRelatedLogs]; link != "" {
+		actions = append(actions, config.SlackAction{Type: "button", Text: "View Related Logs", URL: string(link)})
+	}
+	if link := alert.Annotations[ruletypes.AnnotationRelatedTraces]; link != "" {
+		actions = append(actions, config.SlackAction{Type: "button", Text: "View Related Traces", URL: string(link)})
+	}
+	return actions
+}
+
+// addFieldsAndActions populates fields and actions on the attachment from the Slack config.
+func (n *Notifier) addFieldsAndActions(att *attachment, tmplText func(string) string) {
+	numFields := len(n.conf.Fields)
+	if numFields > 0 {
+		fields := make([]config.SlackField, numFields)
+		for index, field := range n.conf.Fields {
+			var short bool
+			if field.Short != nil {
+				short = *field.Short
+			} else {
+				short = n.conf.ShortFields
+			}
+			fields[index] = config.SlackField{
+				Title: tmplText(field.Title),
+				Value: tmplText(field.Value),
+				Short: &short,
+			}
+		}
+		att.Fields = fields
+	}
+
+	numActions := len(n.conf.Actions)
+	if numActions > 0 {
+		actions := make([]config.SlackAction, numActions)
+		for index, action := range n.conf.Actions {
+			slackAction := config.SlackAction{
+				Type:  tmplText(action.Type),
+				Text:  tmplText(action.Text),
+				URL:   tmplText(action.URL),
+				Style: tmplText(action.Style),
+				Name:  tmplText(action.Name),
+				Value: tmplText(action.Value),
+			}
+
+			if action.ConfirmField != nil {
+				slackAction.ConfirmField = &config.SlackConfirmationField{
+					Title:       tmplText(action.ConfirmField.Title),
+					Text:        tmplText(action.ConfirmField.Text),
+					OkText:      tmplText(action.ConfirmField.OkText),
+					DismissText: tmplText(action.ConfirmField.DismissText),
+				}
+			}
+
+			actions[index] = slackAction
+		}
+		att.Actions = actions
+	}
+}
+
 // checkResponseError parses out the error message from Slack API response.
 func checkResponseError(resp *http.Response) (bool, error) {
 	body, err := io.ReadAll(resp.Body)

pkg/alertmanager/alertmanagernotify/slack/slack_test.go

@@ -17,6 +17,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
@@ -29,13 +32,19 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+func newTestTemplater(tmpl *template.Template) alertmanagertypes.Templater {
+	return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
+}
+
 func TestSlackRetry(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.SlackConfig{
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -49,13 +58,15 @@ func TestSlackRedactedURL(t *testing.T) {
 	ctx, u, fn := test.GetContextWithCancelingURL()
 	defer fn()
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.SlackConfig{
 			APIURL:     &config.SecretURL{URL: u},
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -71,13 +82,15 @@ func TestGettingSlackURLFromFile(t *testing.T) {
 	_, err = f.WriteString(u.String())
 	require.NoError(t, err, "writing to temp file failed")
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.SlackConfig{
 			APIURLFile: f.Name(),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -93,13 +106,15 @@ func TestTrimmingSlackURLFromFile(t *testing.T) {
 	_, err = f.WriteString(u.String() + "\n\n")
 	require.NoError(t, err, "writing to temp file failed")
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.SlackConfig{
 			APIURLFile: f.Name(),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		newTestTemplater(tmpl),
 	)
 	require.NoError(t, err)
 
@@ -184,6 +199,7 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			apiurl, _ := url.Parse("https://slack.com/post.Message")
+			tmpl := test.CreateTmpl(t)
 			notifier, err := New(
 				&config.SlackConfig{
 					NotifierConfig: config.NotifierConfig{},
@@ -191,8 +207,9 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
 					APIURL:         &config.SecretURL{URL: apiurl},
 					Channel:        "channelname",
 				},
-				test.CreateTmpl(t),
+				tmpl,
 				promslog.NewNopLogger(),
+				newTestTemplater(tmpl),
 			)
 			require.NoError(t, err)
 
@@ -242,6 +259,7 @@ func TestSlackTimeout(t *testing.T) {
 	for name, tt := range tests {
 		t.Run(name, func(t *testing.T) {
 			u, _ := url.Parse("https://slack.com/post.Message")
+			tmpl := test.CreateTmpl(t)
 			notifier, err := New(
 				&config.SlackConfig{
 					NotifierConfig: config.NotifierConfig{},
@@ -250,8 +268,9 @@ func TestSlackTimeout(t *testing.T) {
 					Channel:        "channelname",
 					Timeout:        tt.timeout,
 				},
-				test.CreateTmpl(t),
+				tmpl,
 				promslog.NewNopLogger(),
+				newTestTemplater(tmpl),
 			)
 			require.NoError(t, err)
 			notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
@@ -282,6 +301,225 @@ func TestSlackTimeout(t *testing.T) {
 	}
 }
 
+// setupTestContext creates a context with group key, receiver name, and group labels
+// required by the notification processor.
+func setupTestContext() context.Context {
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "slack")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{
+		"alertname": "TestAlert",
+		"severity":  "critical",
+	})
+	return ctx
+}
+
+func TestPrepareContent(t *testing.T) {
+	t.Run("default template uses go text template config for title and body", func(t *testing.T) {
+		// When alerts have no custom annotation templates (title_template / body_template),
+		tmpl := test.CreateTmpl(t)
+		templater := newTestTemplater(tmpl)
+		notifier := &Notifier{
+			conf: &config.SlackConfig{
+				Title:     `{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})`,
+				Text:      `{{ range .Alerts }}Alert: {{ .Labels.alertname }} - severity {{ .Labels.severity }}{{ end }}`,
+				Color:     `{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}`,
+				TitleLink: "https://alertmanager.signoz.com",
+			},
+			tmpl:      tmpl,
+			logger:    slog.New(slog.DiscardHandler),
+			templater: templater,
+		}
+
+		ctx := setupTestContext()
+		alerts := []*types.Alert{
+			{Alert: model.Alert{
+				Labels:   model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			}},
+		}
+
+		// Build tmplText the same way Notify does
+		var err error
+		data := notify.GetTemplateData(ctx, tmpl, alerts, slog.New(slog.DiscardHandler))
+		tmplText := notify.TmplText(tmpl, data, &err)
+
+		atts, attErr := notifier.prepareContent(ctx, alerts, tmplText)
+		require.NoError(t, attErr)
+		require.NoError(t, err)
+		require.Len(t, atts, 1)
+
+		require.Equal(t, "HighCPU (FIRING)", atts[0].Title)
+		require.Equal(t, "Alert: HighCPU - severity critical", atts[0].Text)
+		// Color is templated — firing alert should be "danger"
+		require.Equal(t, "danger", atts[0].Color)
+		// No BlockKit blocks for default template
+		require.Nil(t, atts[0].Blocks)
+		// Default markdownIn when config has none
+		require.Equal(t, []string{"fallback", "pretext", "text"}, atts[0].MrkdwnIn)
+	})
+
+	t.Run("custom template produces 1+N attachments with per-alert color", func(t *testing.T) {
+		// When alerts carry custom $variable annotation templates (title_template / body_template)
+		tmpl := test.CreateTmpl(t)
+		templater := newTestTemplater(tmpl)
+		notifier := &Notifier{
+			conf: &config.SlackConfig{
+				Title:     "default title fallback",
+				Text:      "default text fallback",
+				TitleLink: "https://alertmanager.signoz.com",
+			},
+			tmpl:      tmpl,
+			logger:    slog.New(slog.DiscardHandler),
+			templater: templater,
+		}
+		tmplText := func(s string) string { return s }
+
+		bodyTemplate := `## $rule.name
+
+**Service:** *$labels.service*
+**Instance:** *$labels.instance*
+**Region:** *$labels.region*
+**Method:** *$labels.http_method*
+
+---
+
+| Metric | Value |
+|--------|-------|
+| **Current** | *$value* |
+| **Threshold** | *$threshold.value* |
+
+**Status:** $alert.status | **Severity:** $labels.severity`
+		titleTemplate := "[$alert.status] $rule.name — $labels.service"
+
+		ctx := setupTestContext()
+		firingAlert := &types.Alert{
+			Alert: model.Alert{
+				Labels:   model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical", "service": "api-server", "instance": "i-0abc123", "region": "us-east-1", "http_method": "GET"},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: model.LabelValue(titleTemplate),
+					ruletypes.AnnotationBodyTemplate:  model.LabelValue(bodyTemplate),
+					"value":                           "100",
+					"threshold.value":                 "200",
+				},
+			},
+		}
+		resolvedAlert := &types.Alert{
+			Alert: model.Alert{
+				Labels:   model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical", "service": "api-server", "instance": "i-0abc123", "region": "us-east-1", "http_method": "GET"},
+				StartsAt: time.Now().Add(-2 * time.Hour),
+				EndsAt:   time.Now().Add(-time.Hour),
+				Annotations: model.LabelSet{
+					ruletypes.AnnotationTitleTemplate: model.LabelValue(titleTemplate),
+					ruletypes.AnnotationBodyTemplate:  model.LabelValue(bodyTemplate),
+					"value":                           "50",
+					"threshold.value":                 "200",
+				},
+			},
+		}
+
+		atts, err := notifier.prepareContent(ctx, []*types.Alert{firingAlert, resolvedAlert}, tmplText)
+		require.NoError(t, err)
+
+		// 1 title attachment + 2 body attachments (one per alert)
+		require.Len(t, atts, 3)
+
+		// First attachment: title-only, no color, no blocks
+		require.Equal(t, "[firing] HighCPU — api-server", atts[0].Title)
+		require.Empty(t, atts[0].Color)
+		require.Nil(t, atts[0].Blocks)
+		require.Equal(t, "https://alertmanager.signoz.com", atts[0].TitleLink)
+
+		expectedFiringBody := "*HighCPU*\n\n" +
+			"*Service:* _api-server_\n*Instance:* _i-0abc123_\n*Region:* _us-east-1_\n*Method:* _GET_\n\n" +
+			"---\n\n" +
+			"```\nMetric    | Value\n----------|------\nCurrent   | 100  \nThreshold | 200  \n```\n\n" +
+			"*Status:* firing | *Severity:* critical\n\n"
+
+		expectedResolvedBody := "*HighCPU*\n\n" +
+			"*Service:* _api-server_\n*Instance:* _i-0abc123_\n*Region:* _us-east-1_\n*Method:* _GET_\n\n" +
+			"---\n\n" +
+			"```\nMetric    | Value\n----------|------\nCurrent   | 50   \nThreshold | 200  \n```\n\n" +
+			"*Status:* resolved | *Severity:* critical\n\n"
+
+		// Second attachment: firing alert body rendered as slack mrkdwn text, red color
+		require.Nil(t, atts[1].Blocks)
+		require.Equal(t, "#FF0000", atts[1].Color)
+		require.Equal(t, []string{"text"}, atts[1].MrkdwnIn)
+		require.Equal(t, expectedFiringBody, atts[1].Text)
+
+		// Third attachment: resolved alert body rendered as slack mrkdwn text, green color
+		require.Nil(t, atts[2].Blocks)
+		require.Equal(t, "#00FF00", atts[2].Color)
+		require.Equal(t, []string{"text"}, atts[2].MrkdwnIn)
+		require.Equal(t, expectedResolvedBody, atts[2].Text)
+	})
+
+	t.Run("default template with fields and actions", func(t *testing.T) {
+		// Verifies that addFieldsAndActions (called from Notify after prepareContent)
+		// correctly populates fields and actions on the attachment from config.
+		tmpl := test.CreateTmpl(t)
+		templater := newTestTemplater(tmpl)
+		short := true
+		notifier := &Notifier{
+			conf: &config.SlackConfig{
+				Title: `{{ .CommonLabels.alertname }}`,
+				Text:  "alert text",
+				Color: "warning",
+				Fields: []*config.SlackField{
+					{Title: "Severity", Value: "critical", Short: &short},
+					{Title: "Service", Value: "api-server", Short: &short},
+				},
+				Actions: []*config.SlackAction{
+					{Type: "button", Text: "View Alert", URL: "https://alertmanager.signoz.com"},
+				},
+				TitleLink: "https://alertmanager.signoz.com",
+			},
+			tmpl:      tmpl,
+			logger:    slog.New(slog.DiscardHandler),
+			templater: templater,
+		}
+		tmplText := func(s string) string { return s }
+
+		ctx := setupTestContext()
+		alerts := []*types.Alert{
+			{Alert: model.Alert{
+				Labels:   model.LabelSet{ruletypes.LabelAlertName: "TestAlert"},
+				StartsAt: time.Now(),
+				EndsAt:   time.Now().Add(time.Hour),
+			}},
+		}
+		atts, err := notifier.prepareContent(ctx, alerts, tmplText)
+		require.NoError(t, err)
+		require.Len(t, atts, 1)
+
+		// prepareContent does not populate fields/actions — that's done by
+		// addFieldsAndActions which is called from Notify.
+		require.Nil(t, atts[0].Fields)
+		require.Nil(t, atts[0].Actions)
+
+		// Simulate what Notify does after prepareContent
+		notifier.addFieldsAndActions(&atts[0], tmplText)
+
+		// Verify fields
+		require.Len(t, atts[0].Fields, 2)
+		require.Equal(t, "Severity", atts[0].Fields[0].Title)
+		require.Equal(t, "critical", atts[0].Fields[0].Value)
+		require.True(t, *atts[0].Fields[0].Short)
+		require.Equal(t, "Service", atts[0].Fields[1].Title)
+		require.Equal(t, "api-server", atts[0].Fields[1].Value)
+
+		// Verify actions
+		require.Len(t, atts[0].Actions, 1)
+		require.Equal(t, "button", atts[0].Actions[0].Type)
+		require.Equal(t, "View Alert", atts[0].Actions[0].Text)
+		require.Equal(t, "https://alertmanager.signoz.com", atts[0].Actions[0].URL)
+	})
+}
+
 func TestSlackMessageField(t *testing.T) {
 	// 1. Setup a fake Slack server
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -329,7 +567,7 @@ func TestSlackMessageField(t *testing.T) {
 	tmpl.ExternalURL = u
 
 	logger := slog.New(slog.DiscardHandler)
-	notifier, err := New(conf, tmpl, logger)
+	notifier, err := New(conf, tmpl, logger, newTestTemplater(tmpl))
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/alertmanager/alertmanagernotify/webhook/webhook.go

@@ -13,8 +13,12 @@ import (
 	"os"
 	"strings"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	commoncfg "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/notify"
@@ -23,20 +27,23 @@ import (
 )
 
 const (
-	Integration = "webhook"
+	Integration    = "webhook"
+	templatedTitle = "templated_title"
+	templatedBody  = "templated_body"
 )
 
 // Notifier implements a Notifier for generic webhooks.
 type Notifier struct {
-	conf    *config.WebhookConfig
-	tmpl    *template.Template
-	logger  *slog.Logger
-	client  *http.Client
-	retrier *notify.Retrier
+	conf      *config.WebhookConfig
+	tmpl      *template.Template
+	logger    *slog.Logger
+	client    *http.Client
+	retrier   *notify.Retrier
+	templater alertmanagertemplate.Templater
 }
 
 // New returns a new Webhook.
-func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
+func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, templater alertmanagertemplate.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
 	client, err := notify.NewClientWithTracing(*conf.HTTPConfig, Integration, httpOpts...)
 	if err != nil {
 		return nil, err
@@ -48,7 +55,8 @@ func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, httpO
 		client: client,
 		// Webhooks are assumed to respond with 2xx response codes on a successful
 		// request and 5xx response codes are assumed to be recoverable.
-		retrier: &notify.Retrier{},
+		retrier:   &notify.Retrier{},
+		templater: templater,
 	}, nil
 }
 
@@ -70,9 +78,48 @@ func truncateAlerts(maxAlerts uint64, alerts []*types.Alert) ([]*types.Alert, ui
 	return alerts, 0
 }
 
+// templateTitleBody extracts custom templates from alert annotations, expands them and
+// replaces the private annotations with the rendered title and body.
+func (n *Notifier) templateTitleBody(ctx context.Context, alerts []*types.Alert) error {
+	customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
+	result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
+		TitleTemplate: customTitle,
+		BodyTemplate:  customBody,
+	}, alerts)
+	if err != nil {
+		return err
+	}
+
+	for i, alert := range alerts {
+		if alert.Annotations == nil {
+			continue
+		}
+		// Update templated_title annotation with rendered title, only if key exists and result is non-blank
+		if _, ok := alert.Annotations[ruletypes.AnnotationTitleTemplate]; ok {
+			delete(alert.Annotations, ruletypes.AnnotationTitleTemplate)
+			if result.Title != "" {
+				alert.Annotations[templatedTitle] = model.LabelValue(result.Title)
+			}
+		}
+		// Update templated_body annotation with rendered body, only if key exists and result is non-blank
+		if _, ok := alert.Annotations[ruletypes.AnnotationBodyTemplate]; ok {
+			delete(alert.Annotations, ruletypes.AnnotationBodyTemplate)
+			if i < len(result.Body) && result.Body[i] != "" {
+				alert.Annotations[templatedBody] = model.LabelValue(result.Body[i])
+			}
+		}
+	}
+
+	return nil
+}
+
 // Notify implements the Notifier interface.
 func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
 	alerts, numTruncated := truncateAlerts(n.conf.MaxAlerts, alerts)
+	// expand custom templating annotations
+	if err := n.templateTitleBody(ctx, alerts); err != nil {
+		n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
+	}
 	data := notify.GetTemplateData(ctx, n.tmpl, alerts, n.logger)
 
 	groupKey, err := notify.ExtractGroupKey(ctx)

pkg/alertmanager/alertmanagernotify/webhook/webhook_test.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -20,6 +21,8 @@ import (
 	"github.com/prometheus/common/promslog"
 	"github.com/stretchr/testify/require"
 
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/notify/test"
@@ -27,13 +30,15 @@ import (
 )
 
 func TestWebhookRetry(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.WebhookConfig{
 			URL:        config.SecretTemplateURL("http://example.com"),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		alertmanagertemplate.New(tmpl, slog.Default()),
 	)
 	if err != nil {
 		require.NoError(t, err)
@@ -96,13 +101,16 @@ func TestWebhookRedactedURL(t *testing.T) {
 	defer fn()
 
 	secret := "secret"
+	tmpl := test.CreateTmpl(t)
+	templater := alertmanagertemplate.New(tmpl, slog.Default())
 	notifier, err := New(
 		&config.WebhookConfig{
 			URL:        config.SecretTemplateURL(u.String()),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		templater,
 	)
 	require.NoError(t, err)
 
@@ -118,13 +126,15 @@ func TestWebhookReadingURLFromFile(t *testing.T) {
 	_, err = f.WriteString(u.String() + "\n")
 	require.NoError(t, err, "writing to temp file failed")
 
+	tmpl := test.CreateTmpl(t)
 	notifier, err := New(
 		&config.WebhookConfig{
 			URLFile:    f.Name(),
 			HTTPConfig: &commoncfg.HTTPClientConfig{},
 		},
-		test.CreateTmpl(t),
+		tmpl,
 		promslog.NewNopLogger(),
+		alertmanagertemplate.New(tmpl, slog.Default()),
 	)
 	require.NoError(t, err)
 
@@ -178,13 +188,15 @@ func TestWebhookURLTemplating(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			calledURL = "" // Reset for each test
 
+			tmpl := test.CreateTmpl(t)
 			notifier, err := New(
 				&config.WebhookConfig{
 					URL:        config.SecretTemplateURL(tc.url),
 					HTTPConfig: &commoncfg.HTTPClientConfig{},
 				},
-				test.CreateTmpl(t),
+				tmpl,
 				promslog.NewNopLogger(),
+				alertmanagertemplate.New(tmpl, slog.Default()),
 			)
 			require.NoError(t, err)
 
@@ -216,3 +228,103 @@ func TestWebhookURLTemplating(t *testing.T) {
 		})
 	}
 }
+
+func TestTemplateTitleBody(t *testing.T) {
+	tmpl := test.CreateTmpl(t)
+	templater := alertmanagertemplate.New(tmpl, slog.Default())
+
+	notifier, err := New(
+		&config.WebhookConfig{
+			URL:        config.SecretTemplateURL("http://example.com"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		tmpl,
+		slog.Default(),
+		templater,
+	)
+	require.NoError(t, err)
+
+	t.Run("annotations are updated with custom title and body templates", func(t *testing.T) {
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"alertname": "TestAlert",
+						"severity":  "critical",
+					},
+					Annotations: model.LabelSet{
+						ruletypes.AnnotationTitleTemplate: "Alert: $labels.alertname",
+						ruletypes.AnnotationBodyTemplate:  "Severity is $labels.severity",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"alertname": "TestAlert",
+						"severity":  "warning",
+					},
+					Annotations: model.LabelSet{
+						ruletypes.AnnotationTitleTemplate: "Alert: $labels.alertname",
+						ruletypes.AnnotationBodyTemplate:  "Severity is $labels.severity",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+		}
+
+		ctx := context.Background()
+		err := notifier.templateTitleBody(ctx, alerts)
+		require.NoError(t, err)
+
+		for _, a := range alerts {
+			_, hasTitleTmpl := a.Annotations[ruletypes.AnnotationTitleTemplate]
+			_, hasBodyTmpl := a.Annotations[ruletypes.AnnotationBodyTemplate]
+			require.False(t, hasTitleTmpl, "private title template key should be stripped")
+			require.False(t, hasBodyTmpl, "private body template key should be stripped")
+		}
+
+		require.Equal(t, model.LabelValue("Alert: TestAlert"), alerts[0].Annotations[templatedTitle])
+		require.Equal(t, model.LabelValue("Alert: TestAlert"), alerts[1].Annotations[templatedTitle])
+		require.Equal(t, model.LabelValue("Severity is critical"), alerts[0].Annotations[templatedBody])
+		require.Equal(t, model.LabelValue("Severity is warning"), alerts[1].Annotations[templatedBody])
+	})
+
+	t.Run("annotations not updated when template keys are absent", func(t *testing.T) {
+		alerts := []*types.Alert{
+			{
+				Alert: model.Alert{
+					Labels: model.LabelSet{
+						"alertname": "NoTemplateAlert",
+					},
+					Annotations: model.LabelSet{
+						"summary":     "keep this",
+						"description": "keep this too",
+					},
+					StartsAt: time.Now(),
+					EndsAt:   time.Now().Add(time.Hour),
+				},
+			},
+		}
+
+		ctx := context.Background()
+		err := notifier.templateTitleBody(ctx, alerts)
+		require.NoError(t, err)
+
+		_, hasTitleTemplate := alerts[0].Annotations[ruletypes.AnnotationTitleTemplate]
+		_, hasBodyTemplate := alerts[0].Annotations[ruletypes.AnnotationBodyTemplate]
+		require.False(t, hasTitleTemplate, "title_template should not be added when absent")
+		require.False(t, hasBodyTemplate, "body_template should not be added when absent")
+
+		_, hasTemplatedTitle := alerts[0].Annotations[templatedTitle]
+		_, hasTemplatedBody := alerts[0].Annotations[templatedBody]
+		require.False(t, hasTemplatedTitle, "templated_title should not be added when no custom templates")
+		require.False(t, hasTemplatedBody, "templated_body should not be added when no custom templates")
+
+		require.Equal(t, model.LabelValue("keep this"), alerts[0].Annotations["summary"])
+		require.Equal(t, model.LabelValue("keep this too"), alerts[0].Annotations["description"])
+	})
+}

pkg/alertmanager/alertmanagerserver/config.go

@@ -28,6 +28,9 @@ type Config struct {
 
 	// Configuration for the notification log.
 	NFLog NFLogConfig `mapstructure:"nflog"`
+
+	// EmailTemplatesDirectory is the directory containing email layout templates (.gotmpl files).
+	EmailTemplatesDirectory string `mapstructure:"email_templates_directory"`
 }
 
 type AlertsConfig struct {
@@ -100,5 +103,6 @@ func NewConfig() Config {
 			MaintenanceInterval: 15 * time.Minute,
 			Retention:           120 * time.Hour,
 		},
+		EmailTemplatesDirectory: "/root/templates",
 	}
 }

pkg/alertmanager/alertmanagerserver/server.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prometheus/alertmanager/types"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/prometheus/alertmanager/dispatch"
 	"github.com/prometheus/alertmanager/featurecontrol"
 	"github.com/prometheus/alertmanager/inhibit"
@@ -23,9 +24,11 @@ import (
 	"github.com/prometheus/common/model"
 
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify"
+	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
 	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager"
-	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/emailing/templatestore/filetemplatestore"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
 )
 
 var (
@@ -66,6 +69,8 @@ type Server struct {
 	pipelineBuilder     *notify.PipelineBuilder
 	marker              *alertmanagertypes.MemMarker
 	tmpl                *template.Template
+	notificationDeps    alertmanagertypes.NotificationDeps
+	emailTemplateStore  emailtypes.TemplateStore
 	wg                  sync.WaitGroup
 	stopc               chan struct{}
 	notificationManager nfmanager.NotificationManager
@@ -198,6 +203,12 @@ func New(ctx context.Context, logger *slog.Logger, registry prometheus.Registere
 
 	server.pipelineBuilder = notify.NewPipelineBuilder(signozRegisterer, featurecontrol.NoopFlags{})
 	server.dispatcherMetrics = NewDispatcherMetrics(false, signozRegisterer)
+	emailTemplateStore, storeErr := filetemplatestore.NewStore(ctx, srvConfig.EmailTemplatesDirectory, emailtypes.Templates, server.logger)
+	if storeErr != nil {
+		server.logger.ErrorContext(ctx, "failed to create alert email template store, using empty store", errors.Attr(storeErr))
+		emailTemplateStore = filetemplatestore.NewEmptyStore()
+	}
+	server.emailTemplateStore = emailTemplateStore
 
 	return server, nil
 }
@@ -234,6 +245,11 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 
 	server.tmpl.ExternalURL = server.srvConfig.ExternalURL
 
+	server.notificationDeps = alertmanagertypes.NotificationDeps{
+		Templater:          alertmanagertemplate.New(server.tmpl, server.logger),
+		EmailTemplateStore: server.emailTemplateStore,
+	}
+
 	// Build the routing tree and record which receivers are used.
 	routes := dispatch.NewRoute(config.Route, nil)
 	activeReceivers := make(map[string]struct{})
@@ -250,7 +266,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 			server.logger.InfoContext(ctx, "skipping creation of receiver not referenced by any route", slog.String("receiver", rcv.Name))
 			continue
 		}
-		integrations, err := alertmanagernotify.NewReceiverIntegrations(rcv, server.tmpl, server.logger)
+		integrations, err := alertmanagernotify.NewReceiverIntegrations(rcv, server.tmpl, server.logger, server.notificationDeps)
 		if err != nil {
 			return err
 		}
@@ -326,7 +342,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
 
 func (server *Server) TestReceiver(ctx context.Context, receiver alertmanagertypes.Receiver) error {
 	testAlert := alertmanagertypes.NewTestAlert(receiver, time.Now(), time.Now())
-	return alertmanagertypes.TestReceiver(ctx, receiver, alertmanagernotify.NewReceiverIntegrations, server.alertmanagerConfig, server.tmpl, server.logger, testAlert.Labels, testAlert)
+	return alertmanagertypes.TestReceiver(ctx, receiver, alertmanagernotify.NewReceiverIntegrations, server.alertmanagerConfig, server.tmpl, server.logger, server.notificationDeps, testAlert.Labels, testAlert)
 }
 
 func (server *Server) TestAlert(ctx context.Context, receiversMap map[*alertmanagertypes.PostableAlert][]string, config *alertmanagertypes.NotificationConfig) error {
@@ -409,6 +425,7 @@ func (server *Server) TestAlert(ctx context.Context, receiversMap map[*alertmana
 					server.alertmanagerConfig,
 					server.tmpl,
 					server.logger,
+					server.notificationDeps,
 					group.groupLabels,
 					group.alerts...,
 				)

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate.go

@@ -137,6 +137,9 @@ func (at *templater) expandTitle(
 }
 
 // expandBody expands the body template for each individual alert. Returns nil if the template is empty.
+// Non-nil results are positionally aligned with ntd.Alerts: sb[i] corresponds to alerts[i], and
+// entries for alerts whose template expands to empty are kept as "" so callers can index per-alert
+// metadata (related links, firing/resolved color) by the same index.
 func (at *templater) expandBody(
 	bodyTemplate string,
 	ntd *alertmanagertypes.NotificationTemplateData,
@@ -144,7 +147,7 @@ func (at *templater) expandBody(
 	if bodyTemplate == "" {
 		return nil, nil, nil
 	}
-	var sb []string
+	sb := make([]string, len(ntd.Alerts))
 	missingVars := make(map[string]bool)
 	for i := range ntd.Alerts {
 		processRes, err := preProcessTemplateAndData(bodyTemplate, &ntd.Alerts[i])
@@ -155,13 +158,10 @@ func (at *templater) expandBody(
 		if err != nil {
 			return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to execute custom body template: %s", err.Error())
 		}
-		// add unknown variables and templated text to the result
 		for k := range processRes.UnknownVars {
 			missingVars[k] = true
 		}
-		if strings.TrimSpace(part) != "" {
-			sb = append(sb, strings.TrimSpace(part))
-		}
+		sb[i] = strings.TrimSpace(part)
 	}
 	return sb, missingVars, nil
 }
@@ -189,17 +189,20 @@ func (at *templater) buildNotificationTemplateData(
 		externalURL = at.tmpl.ExternalURL.String()
 	}
 
-	commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+	// Raw (including private `_*`) kv first so buildRuleInfo can read the
+	// private rule-metadata annotations (threshold, op, match_type). The
+	// filtered copies are what ends up on the template-visible surfaces.
+	rawCommonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
 	commonLabels := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
 
 	// aggregate labels and annotations from all alerts
 	labels := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
 	annotations := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
 
-	// Strip private annotations from surfaces visible to templates or
-	// notifications; the structured fields on AlertInfo/RuleInfo already hold
-	// anything a template needs from them.
-	commonAnnotations = alertmanagertypes.FilterPublicAnnotations(commonAnnotations)
+	// Strip private annotations from template-visible surfaces; the structured
+	// fields on AlertInfo/RuleInfo already hold anything a template needs from
+	// them.
+	commonAnnotations := alertmanagertypes.FilterPublicAnnotations(rawCommonAnnotations)
 	annotations = alertmanagertypes.FilterPublicAnnotations(annotations)
 
 	// build the alert data slice
@@ -233,7 +236,7 @@ func (at *templater) buildNotificationTemplateData(
 			TotalFiring:   firing,
 			TotalResolved: resolved,
 		},
-		Rule:              buildRuleInfo(commonLabels, commonAnnotations),
+		Rule:              buildRuleInfo(commonLabels, rawCommonAnnotations),
 		GroupLabels:       gl,
 		CommonLabels:      commonLabels,
 		CommonAnnotations: commonAnnotations,

pkg/authz/authz.go

@@ -26,7 +26,7 @@ type AuthZ interface {
 	Write(context.Context, []*openfgav1.TupleKey, []*openfgav1.TupleKey) error
 
 	// Lists the selectors for objects assigned to subject (s) with relation (r) on resource (s)
-	ListObjects(context.Context, string, authtypes.Relation, authtypes.Typeable) ([]*authtypes.Object, error)
+	ListObjects(context.Context, string, authtypes.Relation, authtypes.Type) ([]*authtypes.Object, error)
 
 	// Creates the role.
 	Create(context.Context, valuer.UUID, *authtypes.Role) error
@@ -78,8 +78,14 @@ type AuthZ interface {
 
 	// Bootstrap managed roles transactions and user assignments
 	CreateManagedUserRoleTransactions(context.Context, valuer.UUID, valuer.UUID) error
+
+	// ReadTuples reads tuples from the authorization server matching the given tuple key filter.
+	ReadTuples(context.Context, *openfgav1.ReadRequestTupleKey) ([]*openfgav1.TupleKey, error)
 }
 
+// OnBeforeRoleDelete is a callback invoked before a role is deleted.
+type OnBeforeRoleDelete func(context.Context, valuer.UUID, valuer.UUID) error
+
 type RegisterTypeable interface {
 	MustGetTypeables() []authtypes.Typeable
 

pkg/authz/authzstore/sqlauthzstore/store.go

@@ -18,7 +18,7 @@ func NewSqlAuthzStore(sqlstore sqlstore.SQLStore) authtypes.RoleStore {
 	return &store{sqlstore: sqlstore}
 }
 
-func (store *store) Create(ctx context.Context, role *authtypes.StorableRole) error {
+func (store *store) Create(ctx context.Context, role *authtypes.Role) error {
 	_, err := store.
 		sqlstore.
 		BunDBCtx(ctx).
@@ -32,8 +32,8 @@ func (store *store) Create(ctx context.Context, role *authtypes.StorableRole) er
 	return nil
 }
 
-func (store *store) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID) (*authtypes.StorableRole, error) {
-	role := new(authtypes.StorableRole)
+func (store *store) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID) (*authtypes.Role, error) {
+	role := new(authtypes.Role)
 	err := store.
 		sqlstore.
 		BunDBCtx(ctx).
@@ -49,8 +49,8 @@ func (store *store) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID)
 	return role, nil
 }
 
-func (store *store) GetByOrgIDAndName(ctx context.Context, orgID valuer.UUID, name string) (*authtypes.StorableRole, error) {
-	role := new(authtypes.StorableRole)
+func (store *store) GetByOrgIDAndName(ctx context.Context, orgID valuer.UUID, name string) (*authtypes.Role, error) {
+	role := new(authtypes.Role)
 	err := store.
 		sqlstore.
 		BunDBCtx(ctx).
@@ -66,8 +66,8 @@ func (store *store) GetByOrgIDAndName(ctx context.Context, orgID valuer.UUID, na
 	return role, nil
 }
 
-func (store *store) List(ctx context.Context, orgID valuer.UUID) ([]*authtypes.StorableRole, error) {
-	roles := make([]*authtypes.StorableRole, 0)
+func (store *store) List(ctx context.Context, orgID valuer.UUID) ([]*authtypes.Role, error) {
+	roles := make([]*authtypes.Role, 0)
 	err := store.
 		sqlstore.
 		BunDBCtx(ctx).
@@ -82,8 +82,8 @@ func (store *store) List(ctx context.Context, orgID valuer.UUID) ([]*authtypes.S
 	return roles, nil
 }
 
-func (store *store) ListByOrgIDAndNames(ctx context.Context, orgID valuer.UUID, names []string) ([]*authtypes.StorableRole, error) {
-	roles := make([]*authtypes.StorableRole, 0)
+func (store *store) ListByOrgIDAndNames(ctx context.Context, orgID valuer.UUID, names []string) ([]*authtypes.Role, error) {
+	roles := make([]*authtypes.Role, 0)
 	err := store.
 		sqlstore.
 		BunDBCtx(ctx).
@@ -103,8 +103,8 @@ func (store *store) ListByOrgIDAndNames(ctx context.Context, orgID valuer.UUID,
 	return roles, nil
 }
 
-func (store *store) ListByOrgIDAndIDs(ctx context.Context, orgID valuer.UUID, ids []valuer.UUID) ([]*authtypes.StorableRole, error) {
-	roles := make([]*authtypes.StorableRole, 0)
+func (store *store) ListByOrgIDAndIDs(ctx context.Context, orgID valuer.UUID, ids []valuer.UUID) ([]*authtypes.Role, error) {
+	roles := make([]*authtypes.Role, 0)
 	err := store.
 		sqlstore.
 		BunDBCtx(ctx).
@@ -124,7 +124,7 @@ func (store *store) ListByOrgIDAndIDs(ctx context.Context, orgID valuer.UUID, id
 	return roles, nil
 }
 
-func (store *store) Update(ctx context.Context, orgID valuer.UUID, role *authtypes.StorableRole) error {
+func (store *store) Update(ctx context.Context, orgID valuer.UUID, role *authtypes.Role) error {
 	_, err := store.
 		sqlstore.
 		BunDBCtx(ctx).
@@ -145,7 +145,7 @@ func (store *store) Delete(ctx context.Context, orgID valuer.UUID, id valuer.UUI
 		sqlstore.
 		BunDBCtx(ctx).
 		NewDelete().
-		Model(new(authtypes.StorableRole)).
+		Model(new(authtypes.Role)).
 		Where("org_id = ?", orgID).
 		Where("id = ?", id).
 		Exec(ctx)

pkg/authz/config.go

@@ -4,14 +4,30 @@ import (
 	"github.com/SigNoz/signoz/pkg/factory"
 )
 
-type Config struct{}
+type Config struct {
+	// Provider is the name of the authorization provider to use.
+	Provider string `mapstructure:"provider"`
+
+	// OpenFGA is the configuration specific to the OpenFGA authorization provider.
+	OpenFGA OpenFGAConfig `mapstructure:"openfga"`
+}
+
+type OpenFGAConfig struct {
+	// MaxTuplesPerWrite is the maximum number of tuples to include in a single write call.
+	MaxTuplesPerWrite int `mapstructure:"max_tuples_per_write"`
+}
 
 func NewConfigFactory() factory.ConfigFactory {
 	return factory.NewConfigFactory(factory.MustNewName("authz"), newConfig)
 }
 
 func newConfig() factory.Config {
-	return Config{}
+	return &Config{
+		Provider: "openfga",
+		OpenFGA: OpenFGAConfig{
+			MaxTuplesPerWrite: 100,
+		},
+	}
 }
 
 func (c Config) Validate() error {

pkg/authz/openfgaauthz/provider.go

@@ -68,68 +68,32 @@ func (provider *provider) Write(ctx context.Context, additions []*openfgav1.Tupl
 	return provider.server.Write(ctx, additions, deletions)
 }
 
-func (provider *provider) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, typeable authtypes.Typeable) ([]*authtypes.Object, error) {
-	return provider.server.ListObjects(ctx, subject, relation, typeable)
+func (provider *provider) ReadTuples(ctx context.Context, tupleKey *openfgav1.ReadRequestTupleKey) ([]*openfgav1.TupleKey, error) {
+	return provider.server.ReadTuples(ctx, tupleKey)
+}
+
+func (provider *provider) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType authtypes.Type) ([]*authtypes.Object, error) {
+	return provider.server.ListObjects(ctx, subject, relation, objectType)
 }
 
 func (provider *provider) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID) (*authtypes.Role, error) {
-	storableRole, err := provider.store.Get(ctx, orgID, id)
-	if err != nil {
-		return nil, err
-	}
-
-	return authtypes.NewRoleFromStorableRole(storableRole), nil
+	return provider.store.Get(ctx, orgID, id)
 }
 
 func (provider *provider) GetByOrgIDAndName(ctx context.Context, orgID valuer.UUID, name string) (*authtypes.Role, error) {
-	storableRole, err := provider.store.GetByOrgIDAndName(ctx, orgID, name)
-	if err != nil {
-		return nil, err
-	}
-
-	return authtypes.NewRoleFromStorableRole(storableRole), nil
+	return provider.store.GetByOrgIDAndName(ctx, orgID, name)
 }
 
 func (provider *provider) List(ctx context.Context, orgID valuer.UUID) ([]*authtypes.Role, error) {
-	storableRoles, err := provider.store.List(ctx, orgID)
-	if err != nil {
-		return nil, err
-	}
-
-	roles := make([]*authtypes.Role, len(storableRoles))
-	for idx, storableRole := range storableRoles {
-		roles[idx] = authtypes.NewRoleFromStorableRole(storableRole)
-	}
-
-	return roles, nil
+	return provider.store.List(ctx, orgID)
 }
 
 func (provider *provider) ListByOrgIDAndNames(ctx context.Context, orgID valuer.UUID, names []string) ([]*authtypes.Role, error) {
-	storableRoles, err := provider.store.ListByOrgIDAndNames(ctx, orgID, names)
-	if err != nil {
-		return nil, err
-	}
-
-	roles := make([]*authtypes.Role, len(storableRoles))
-	for idx, storable := range storableRoles {
-		roles[idx] = authtypes.NewRoleFromStorableRole(storable)
-	}
-
-	return roles, nil
+	return provider.store.ListByOrgIDAndNames(ctx, orgID, names)
 }
 
 func (provider *provider) ListByOrgIDAndIDs(ctx context.Context, orgID valuer.UUID, ids []valuer.UUID) ([]*authtypes.Role, error) {
-	storableRoles, err := provider.store.ListByOrgIDAndIDs(ctx, orgID, ids)
-	if err != nil {
-		return nil, err
-	}
-
-	roles := make([]*authtypes.Role, len(storableRoles))
-	for idx, storable := range storableRoles {
-		roles[idx] = authtypes.NewRoleFromStorableRole(storable)
-	}
-
-	return roles, nil
+	return provider.store.ListByOrgIDAndIDs(ctx, orgID, ids)
 }
 
 func (provider *provider) Grant(ctx context.Context, orgID valuer.UUID, names []string, subject string) error {
@@ -197,7 +161,7 @@ func (provider *provider) Revoke(ctx context.Context, orgID valuer.UUID, names [
 func (provider *provider) CreateManagedRoles(ctx context.Context, _ valuer.UUID, managedRoles []*authtypes.Role) error {
 	err := provider.store.RunInTx(ctx, func(ctx context.Context) error {
 		for _, role := range managedRoles {
-			err := provider.store.Create(ctx, authtypes.NewStorableRoleFromRole(role))
+			err := provider.store.Create(ctx, role)
 			if err != nil {
 				return err
 			}

pkg/authz/openfgaserver/server.go

@@ -265,17 +265,45 @@ func (server *Server) Write(ctx context.Context, additions []*openfgav1.TupleKey
 	return nil
 }
 
-func (server *Server) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, typeable authtypes.Typeable) ([]*authtypes.Object, error) {
+func (server *Server) ReadTuples(ctx context.Context, tupleKey *openfgav1.ReadRequestTupleKey) ([]*openfgav1.TupleKey, error) {
+	storeID, _ := server.getStoreIDandModelID()
+	var tuples []*openfgav1.TupleKey
+	continuationToken := ""
+
+	for {
+		response, err := server.openfgaServer.Read(ctx, &openfgav1.ReadRequest{
+			StoreId:           storeID,
+			TupleKey:          tupleKey,
+			ContinuationToken: continuationToken,
+		})
+		if err != nil {
+			return nil, errors.Wrapf(err, errors.TypeInternal, authtypes.ErrCodeAuthZUnavailable, "failed to read tuples from authorization server")
+		}
+
+		for _, tuple := range response.Tuples {
+			tuples = append(tuples, tuple.Key)
+		}
+
+		if response.ContinuationToken == "" {
+			break
+		}
+		continuationToken = response.ContinuationToken
+	}
+
+	return tuples, nil
+}
+
+func (server *Server) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType authtypes.Type) ([]*authtypes.Object, error) {
 	storeID, modelID := server.getStoreIDandModelID()
 	response, err := server.openfgaServer.ListObjects(ctx, &openfgav1.ListObjectsRequest{
 		StoreId:              storeID,
 		AuthorizationModelId: modelID,
 		User:                 subject,
 		Relation:             relation.StringValue(),
-		Type:                 typeable.Type().StringValue(),
+		Type:                 objectType.StringValue(),
 	})
 	if err != nil {
-		return nil, errors.Wrapf(err, errors.TypeInternal, authtypes.ErrCodeAuthZUnavailable, "cannot list objects for subject %s with relation %s for type %s", subject, relation.StringValue(), typeable.Type().StringValue())
+		return nil, errors.Wrapf(err, errors.TypeInternal, authtypes.ErrCodeAuthZUnavailable, "cannot list objects for subject %s with relation %s for type %s", subject, relation.StringValue(), objectType.StringValue())
 	}
 
 	return authtypes.MustNewObjectsFromStringSlice(response.Objects), nil

pkg/modules/cloudintegration/implcloudintegration/handler.go

@@ -373,7 +373,13 @@ func (handler *handler) UpdateService(rw http.ResponseWriter, r *http.Request) {
 
 	// update or create service
 	if svc.CloudIntegrationService == nil {
-		cloudIntegrationService := cloudintegrationtypes.NewCloudIntegrationService(serviceID, cloudIntegrationID, req.Config)
+		var cloudIntegrationService *cloudintegrationtypes.CloudIntegrationService
+		cloudIntegrationService, err = cloudintegrationtypes.NewCloudIntegrationService(serviceID, cloudIntegrationID, provider, req.Config)
+		if err != nil {
+			render.Error(rw, err)
+			return
+		}
+
 		err = handler.module.CreateService(ctx, orgID, cloudIntegrationService, provider)
 	} else {
 		err = svc.CloudIntegrationService.Update(provider, serviceID, req.Config)

pkg/modules/serviceaccount/implserviceaccount/getter.go

@@ -0,0 +1,30 @@
+package implserviceaccount
+
+import (
+	"context"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
+	"github.com/SigNoz/signoz/pkg/types/authtypes"
+	"github.com/SigNoz/signoz/pkg/types/serviceaccounttypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+type getter struct {
+	store serviceaccounttypes.Store
+}
+
+func NewGetter(store serviceaccounttypes.Store) serviceaccount.Getter {
+	return &getter{store: store}
+}
+
+func (getter *getter) OnBeforeRoleDelete(ctx context.Context, orgID valuer.UUID, roleID valuer.UUID) error {
+	serviceAccounts, err := getter.store.GetServiceAccountsByOrgIDAndRoleID(ctx, orgID, roleID)
+	if err != nil {
+		return err
+	}
+	if len(serviceAccounts) > 0 {
+		return errors.New(errors.TypeInvalidInput, authtypes.ErrCodeRoleHasServiceAccountAssignees, "role has active service account assignments, remove them before deleting")
+	}
+	return nil
+}

pkg/modules/serviceaccount/implserviceaccount/store.go

@@ -123,6 +123,25 @@ func (store *store) GetByIDAndStatus(ctx context.Context, id valuer.UUID, status
 	return storable, nil
 }
 
+func (store *store) GetServiceAccountsByOrgIDAndRoleID(ctx context.Context, orgID valuer.UUID, roleID valuer.UUID) ([]*serviceaccounttypes.ServiceAccount, error) {
+	serviceAccounts := make([]*serviceaccounttypes.ServiceAccount, 0)
+
+	err := store.
+		sqlstore.
+		BunDBCtx(ctx).
+		NewSelect().
+		Model(&serviceAccounts).
+		Join(`JOIN service_account_role ON service_account_role.service_account_id = service_account.id`).
+		Where(`service_account.org_id = ?`, orgID).
+		Where("service_account_role.role_id = ?", roleID).
+		Scan(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return serviceAccounts, nil
+}
+
 func (store *store) CountByOrgID(ctx context.Context, orgID valuer.UUID) (int64, error) {
 	storable := new(serviceaccounttypes.ServiceAccount)
 

pkg/modules/serviceaccount/serviceaccount.go

@@ -11,6 +11,11 @@ import (
 	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
+type Getter interface {
+	// OnBeforeRoleDelete checks if any service accounts are assigned to the role and rejects deletion if so.
+	OnBeforeRoleDelete(ctx context.Context, orgID valuer.UUID, roleID valuer.UUID) error
+}
+
 type Module interface {
 	// Creates a new service account for an organization.
 	Create(context.Context, valuer.UUID, *serviceaccounttypes.ServiceAccount) error

pkg/modules/user/impluser/getter.go

@@ -225,3 +225,14 @@ func (module *getter) GetResetPasswordTokenByOrgIDAndUserID(ctx context.Context,
 func (module *getter) GetUsersByOrgIDAndRoleID(ctx context.Context, orgID valuer.UUID, roleID valuer.UUID) ([]*types.User, error) {
 	return module.store.GetUsersByOrgIDAndRoleID(ctx, orgID, roleID)
 }
+
+func (module *getter) OnBeforeRoleDelete(ctx context.Context, orgID valuer.UUID, roleID valuer.UUID) error {
+	users, err := module.GetUsersByOrgIDAndRoleID(ctx, orgID, roleID)
+	if err != nil {
+		return err
+	}
+	if len(users) > 0 {
+		return errors.New(errors.TypeInvalidInput, authtypes.ErrCodeRoleHasUserAssignees, "role has active user assignments, remove them before deleting")
+	}
+	return nil
+}

pkg/modules/user/user.go

@@ -91,6 +91,9 @@ type Getter interface {
 
 	// Gets all the user with role using role id in an org id
 	GetUsersByOrgIDAndRoleID(ctx context.Context, orgID valuer.UUID, roleID valuer.UUID) ([]*types.User, error)
+
+	// OnBeforeRoleDelete checks if any users are assigned to the role and rejects deletion if so.
+	OnBeforeRoleDelete(ctx context.Context, orgID valuer.UUID, roleID valuer.UUID) error
 }
 
 type Handler interface {

pkg/query-service/rules/prom_rule.go

@@ -211,8 +211,23 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 
 		annotations := make(ruletypes.Labels, 0, len(r.annotations.Map()))
 		for name, value := range r.annotations.Map() {
+			// no need to expand custom templating annotations — they get expanded in the notifier layer
+			if ruletypes.IsCustomTemplatingAnnotation(name) {
+				annotations = append(annotations, ruletypes.Label{Name: name, Value: value})
+				continue
+			}
 			annotations = append(annotations, ruletypes.Label{Name: name, Value: expand(value)})
 		}
+		// Add values to be used in notifier layer for notification templates
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationValue, Value: valueFormatter.Format(result.V, r.Unit())})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationThresholdValue, Value: threshold})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationCompareOp, Value: result.CompareOperator.Literal()})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationMatchType, Value: result.MatchType.Literal()})
+
+		if result.IsRecovering {
+			lb.Set(ruletypes.LabelIsRecovering, "true")
+		}
+
 		if result.IsMissing {
 			lb.Set(ruletypes.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(ruletypes.NoDataLabel, "true")

pkg/query-service/rules/threshold_rule.go

@@ -337,8 +337,23 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 
 		annotations := make(ruletypes.Labels, 0, len(r.annotations.Map()))
 		for name, value := range r.annotations.Map() {
+			// no need to expand custom templating annotations — they get expanded in the notifier layer
+			if ruletypes.IsCustomTemplatingAnnotation(name) {
+				annotations = append(annotations, ruletypes.Label{Name: name, Value: value})
+				continue
+			}
 			annotations = append(annotations, ruletypes.Label{Name: name, Value: expand(value)})
 		}
+		// Add values to be used in notifier layer for notification templates
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationValue, Value: value})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationThresholdValue, Value: threshold})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationCompareOp, Value: smpl.CompareOperator.Literal()})
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationMatchType, Value: smpl.MatchType.Literal()})
+
+		if smpl.IsRecovering {
+			lb.Set(ruletypes.LabelIsRecovering, "true")
+		}
+
 		if smpl.IsMissing {
 			lb.Set(ruletypes.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(ruletypes.NoDataLabel, "true")
@@ -352,13 +367,13 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 			link := r.prepareLinksToTraces(ctx, ts, smpl.Metric)
 			if link != "" && r.hostFromSource() != "" {
 				r.logger.InfoContext(ctx, "adding traces link to annotations", slog.String("annotation.link", fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)))
-				annotations = append(annotations, ruletypes.Label{Name: "related_traces", Value: fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)})
+				annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationRelatedTraces, Value: fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)})
 			}
 		case ruletypes.AlertTypeLogs:
 			link := r.prepareLinksToLogs(ctx, ts, smpl.Metric)
 			if link != "" && r.hostFromSource() != "" {
 				r.logger.InfoContext(ctx, "adding logs link to annotations", slog.String("annotation.link", fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)))
-				annotations = append(annotations, ruletypes.Label{Name: "related_logs", Value: fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)})
+				annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationRelatedLogs, Value: fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)})
 			}
 		}
 

pkg/query-service/rules/threshold_rule_test.go

@@ -869,7 +869,7 @@ func TestThresholdRuleTracesLink(t *testing.T) {
 			assert.Equal(t, c.expectAlerts, alertsFound, "case %d", idx)
 			for _, item := range rule.Active {
 				for name, value := range item.Annotations.Map() {
-					if name == "related_traces" {
+					if name == ruletypes.AnnotationRelatedTraces {
 						assert.NotEmpty(t, value, "case %d", idx)
 						assert.Contains(t, value, "GET")
 					}
@@ -986,7 +986,7 @@ func TestThresholdRuleLogsLink(t *testing.T) {
 			assert.Equal(t, c.expectAlerts, alertsFound, "case %d", idx)
 			for _, item := range rule.Active {
 				for name, value := range item.Annotations.Map() {
-					if name == "related_logs" {
+					if name == ruletypes.AnnotationRelatedLogs {
 						assert.NotEmpty(t, value, "case %d", idx)
 						assert.Contains(t, value, "testcontainer")
 					}

pkg/signoz/config.go

@@ -12,6 +12,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/analytics"
 	"github.com/SigNoz/signoz/pkg/apiserver"
 	"github.com/SigNoz/signoz/pkg/auditor"
+	"github.com/SigNoz/signoz/pkg/authz"
 	"github.com/SigNoz/signoz/pkg/cache"
 	"github.com/SigNoz/signoz/pkg/config"
 	"github.com/SigNoz/signoz/pkg/emailing"
@@ -135,6 +136,9 @@ type Config struct {
 
 	// CloudIntegration config
 	CloudIntegration cloudintegration.Config `mapstructure:"cloudintegration"`
+
+	// Authz config
+	Authz authz.Config `mapstructure:"authz"`
 }
 
 func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.ResolverConfig) (Config, error) {
@@ -168,6 +172,7 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
 		serviceaccount.NewConfigFactory(),
 		auditor.NewConfigFactory(),
 		cloudintegration.NewConfigFactory(),
+		authz.NewConfigFactory(),
 	}
 
 	conf, err := config.New(ctx, resolverConfig, configFactories)

pkg/signoz/signoz.go

@@ -100,7 +100,7 @@ func New(
 	sqlstoreProviderFactories factory.NamedMap[factory.ProviderFactory[sqlstore.SQLStore, sqlstore.Config]],
 	telemetrystoreProviderFactories factory.NamedMap[factory.ProviderFactory[telemetrystore.TelemetryStore, telemetrystore.Config]],
 	authNsCallback func(ctx context.Context, providerSettings factory.ProviderSettings, store authtypes.AuthNStore, licensing licensing.Licensing) (map[authtypes.AuthNProvider]authn.AuthN, error),
-	authzCallback func(context.Context, sqlstore.SQLStore, licensing.Licensing, dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error),
+	authzCallback func(context.Context, sqlstore.SQLStore, licensing.Licensing, []authz.OnBeforeRoleDelete, dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error),
 	dashboardModuleCallback func(sqlstore.SQLStore, factory.ProviderSettings, analytics.Analytics, organization.Getter, queryparser.QueryParser, querier.Querier, licensing.Licensing) dashboard.Module,
 	gatewayProviderFactory func(licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config],
 	auditorProviderFactories func(licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]],
@@ -328,19 +328,28 @@ func New(
 	// Initialize dashboard module (needed for authz registry)
 	dashboard := dashboardModuleCallback(sqlstore, providerSettings, analytics, orgGetter, queryParser, querier, licensing)
 
-	// Initialize authz
-	authzProviderFactory, err := authzCallback(ctx, sqlstore, licensing, dashboard)
-	if err != nil {
-		return nil, err
-	}
-	authz, err := authzProviderFactory.New(ctx, providerSettings, authz.Config{})
-	if err != nil {
-		return nil, err
-	}
-
 	// Initialize user getter
 	userGetter := impluser.NewGetter(userStore, userRoleStore, flagger)
 
+	// Initialize service account getter
+	serviceAccountGetter := implserviceaccount.NewGetter(implserviceaccount.NewStore(sqlstore))
+
+	// Build pre-delete callbacks from modules
+	onBeforeRoleDelete := []authz.OnBeforeRoleDelete{
+		userGetter.OnBeforeRoleDelete,
+		serviceAccountGetter.OnBeforeRoleDelete,
+	}
+
+	// Initialize authz
+	authzProviderFactory, err := authzCallback(ctx, sqlstore, licensing, onBeforeRoleDelete, dashboard)
+	if err != nil {
+		return nil, err
+	}
+	authz, err := authzProviderFactory.New(ctx, providerSettings, config.Authz)
+	if err != nil {
+		return nil, err
+	}
+
 	// Initialize notification manager from the available notification manager provider factories
 	nfManager, err := factory.NewProviderFromNamedMap(
 		ctx,

pkg/sqlmigration/059_add_managed_roles.go

@@ -54,7 +54,7 @@ func (migration *addManagedRoles) Up(ctx context.Context, db *bun.DB) error {
 		return err
 	}
 
-	managedRoles := []*authtypes.StorableRole{}
+	managedRoles := []*authtypes.Role{}
 	for _, orgIDStr := range orgIDs {
 		orgID, err := valuer.NewUUID(orgIDStr)
 		if err != nil {
@@ -63,19 +63,19 @@ func (migration *addManagedRoles) Up(ctx context.Context, db *bun.DB) error {
 
 		// signoz admin
 		signozAdminRole := authtypes.NewRole(authtypes.SigNozAdminRoleName, authtypes.SigNozAdminRoleDescription, authtypes.RoleTypeManaged, orgID)
-		managedRoles = append(managedRoles, authtypes.NewStorableRoleFromRole(signozAdminRole))
+		managedRoles = append(managedRoles, signozAdminRole)
 
 		// signoz editor
 		signozEditorRole := authtypes.NewRole(authtypes.SigNozEditorRoleName, authtypes.SigNozEditorRoleDescription, authtypes.RoleTypeManaged, orgID)
-		managedRoles = append(managedRoles, authtypes.NewStorableRoleFromRole(signozEditorRole))
+		managedRoles = append(managedRoles, signozEditorRole)
 
 		// signoz viewer
 		signozViewerRole := authtypes.NewRole(authtypes.SigNozViewerRoleName, authtypes.SigNozViewerRoleDescription, authtypes.RoleTypeManaged, orgID)
-		managedRoles = append(managedRoles, authtypes.NewStorableRoleFromRole(signozViewerRole))
+		managedRoles = append(managedRoles, signozViewerRole)
 
 		// signoz anonymous
 		signozAnonymousRole := authtypes.NewRole(authtypes.SigNozAnonymousRoleName, authtypes.SigNozAnonymousRoleDescription, authtypes.RoleTypeManaged, orgID)
-		managedRoles = append(managedRoles, authtypes.NewStorableRoleFromRole(signozAnonymousRole))
+		managedRoles = append(managedRoles, signozAnonymousRole)
 	}
 
 	if len(managedRoles) > 0 {

pkg/types/alertmanagertypes/notification.go

@@ -0,0 +1,29 @@
+package alertmanagertypes
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/SigNoz/signoz/pkg/types/emailtypes"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// Templater expands user-authored title and body templates against a group
+// of alerts. Implemented by pkg/alertmanager/alertmanagertemplate.
+type Templater interface {
+	Expand(ctx context.Context, req ExpandRequest, alerts []*types.Alert) (*ExpandResult, error)
+}
+
+// NotificationDeps carries the shared helpers every notifier needs to turn
+// custom templates into channel-ready content. EmailTemplateStore is only
+// consumed by the email notifier; other channels ignore it.
+type NotificationDeps struct {
+	Templater          Templater
+	EmailTemplateStore emailtypes.TemplateStore
+}
+
+// ReceiverIntegrationsFunc constructs the notify.Integration list for a
+// configured receiver.
+type ReceiverIntegrationsFunc = func(nc Receiver, tmpl *template.Template, logger *slog.Logger, deps NotificationDeps) ([]notify.Integration, error)

pkg/types/alertmanagertypes/receiver.go

@@ -4,10 +4,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/prometheus/common/model"
 	"log/slog"
 	"time"
 
+	"github.com/prometheus/common/model"
+
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/template"
@@ -18,8 +19,7 @@ import (
 
 type (
 	// Receiver is the type for the receiver configuration.
-	Receiver                 = config.Receiver
-	ReceiverIntegrationsFunc = func(nc Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error)
+	Receiver = config.Receiver
 )
 
 // Creates a new receiver from a string. The input is initialized with the default values from the upstream alertmanager.
@@ -50,7 +50,7 @@ func NewReceiver(input string) (Receiver, error) {
 	return receiverWithDefaults, nil
 }
 
-func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFunc ReceiverIntegrationsFunc, config *Config, tmpl *template.Template, logger *slog.Logger, lSet model.LabelSet, alert ...*Alert) error {
+func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFunc ReceiverIntegrationsFunc, config *Config, tmpl *template.Template, logger *slog.Logger, deps NotificationDeps, lSet model.LabelSet, alert ...*Alert) error {
 	ctx = notify.WithGroupKey(ctx, fmt.Sprintf("%s-%s-%d", receiver.Name, lSet.Fingerprint(), time.Now().Unix()))
 	ctx = notify.WithGroupLabels(ctx, lSet)
 	ctx = notify.WithReceiverName(ctx, receiver.Name)
@@ -72,7 +72,7 @@ func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFu
 		return err
 	}
 
-	integrations, err := receiverIntegrationsFunc(receiver, tmpl, logger)
+	integrations, err := receiverIntegrationsFunc(receiver, tmpl, logger, deps)
 	if err != nil {
 		return err
 	}

pkg/types/authtypes/role.go

@@ -20,6 +20,8 @@ var (
 	ErrCodeRoleNotFound                     = errors.MustNewCode("role_not_found")
 	ErrCodeRoleFailedTransactionsFromString = errors.MustNewCode("role_failed_transactions_from_string")
 	ErrCodeRoleUnsupported                  = errors.MustNewCode("role_unsupported")
+	ErrCodeRoleHasUserAssignees             = errors.MustNewCode("role_has_user_assignees")
+	ErrCodeRoleHasServiceAccountAssignees   = errors.MustNewCode("role_has_service_account_assignees")
 )
 
 var (
@@ -60,17 +62,6 @@ var (
 	TypeableResourcesRoles = MustNewTypeableMetaResources(MustNewName("roles"))
 )
 
-type StorableRole struct {
-	bun.BaseModel `bun:"table:role"`
-
-	types.Identifiable
-	types.TimeAuditable
-	Name        string `bun:"name,type:string" json:"name"`
-	Description string `bun:"description,type:string" json:"description"`
-	Type        string `bun:"type,type:string" json:"type"`
-	OrgID       string `bun:"org_id,type:string" json:"orgId"`
-}
-
 type Role struct {
 	bun.BaseModel `bun:"table:role"`
 
@@ -91,28 +82,6 @@ type PatchableRole struct {
 	Description string `json:"description" required:"true"`
 }
 
-func NewStorableRoleFromRole(role *Role) *StorableRole {
-	return &StorableRole{
-		Identifiable:  role.Identifiable,
-		TimeAuditable: role.TimeAuditable,
-		Name:          role.Name,
-		Description:   role.Description,
-		Type:          role.Type.String(),
-		OrgID:         role.OrgID.StringValue(),
-	}
-}
-
-func NewRoleFromStorableRole(storableRole *StorableRole) *Role {
-	return &Role{
-		Identifiable:  storableRole.Identifiable,
-		TimeAuditable: storableRole.TimeAuditable,
-		Name:          storableRole.Name,
-		Description:   storableRole.Description,
-		Type:          valuer.NewString(storableRole.Type),
-		OrgID:         valuer.MustNewUUID(storableRole.OrgID),
-	}
-}
-
 func NewRole(name, description string, roleType valuer.String, orgID valuer.UUID) *Role {
 	return &Role{
 		Identifiable: types.Identifiable{
@@ -264,13 +233,13 @@ func MustGetSigNozManagedRoleFromExistingRole(role types.Role) string {
 }
 
 type RoleStore interface {
-	Create(context.Context, *StorableRole) error
-	Get(context.Context, valuer.UUID, valuer.UUID) (*StorableRole, error)
-	GetByOrgIDAndName(context.Context, valuer.UUID, string) (*StorableRole, error)
-	List(context.Context, valuer.UUID) ([]*StorableRole, error)
-	ListByOrgIDAndNames(context.Context, valuer.UUID, []string) ([]*StorableRole, error)
-	ListByOrgIDAndIDs(context.Context, valuer.UUID, []valuer.UUID) ([]*StorableRole, error)
-	Update(context.Context, valuer.UUID, *StorableRole) error
+	Create(context.Context, *Role) error
+	Get(context.Context, valuer.UUID, valuer.UUID) (*Role, error)
+	GetByOrgIDAndName(context.Context, valuer.UUID, string) (*Role, error)
+	List(context.Context, valuer.UUID) ([]*Role, error)
+	ListByOrgIDAndNames(context.Context, valuer.UUID, []string) ([]*Role, error)
+	ListByOrgIDAndIDs(context.Context, valuer.UUID, []valuer.UUID) ([]*Role, error)
+	Update(context.Context, valuer.UUID, *Role) error
 	Delete(context.Context, valuer.UUID, valuer.UUID) error
 	RunInTx(context.Context, func(ctx context.Context) error) error
 }

pkg/types/cloudintegrationtypes/account.go

@@ -29,8 +29,13 @@ type AgentReport struct {
 }
 
 type AccountConfig struct {
-	// required till new providers are added
-	AWS *AWSAccountConfig `json:"aws" required:"true" nullable:"false"`
+	AWS   *AWSAccountConfig   `json:"aws,omitempty" required:"false" nullable:"false"`
+	Azure *AzureAccountConfig `json:"azure,omitempty" required:"false" nullable:"false"`
+}
+
+type UpdatableAccountConfig struct {
+	AWS   *UpdatableAWSAccountConfig   `json:"aws,omitempty" required:"false" nullable:"false"`
+	Azure *UpdatableAzureAccountConfig `json:"azure,omitempty" required:"false" nullable:"false"`
 }
 
 type PostableAccount struct {
@@ -41,7 +46,8 @@ type PostableAccount struct {
 type PostableAccountConfig struct {
 	// as agent version is common for all providers, we can keep it at top level of this struct
 	AgentVersion string
-	AWS          *AWSPostableAccountConfig `json:"aws" required:"true" nullable:"false"`
+	AWS          *AWSPostableAccountConfig   `json:"aws,omitempty" required:"false" nullable:"false"`
+	Azure        *AzurePostableAccountConfig `json:"azure,omitempty" required:"false" nullable:"false"`
 }
 
 type Credentials struct {
@@ -58,7 +64,8 @@ type GettableAccountWithConnectionArtifact struct {
 
 type ConnectionArtifact struct {
 	// required till new providers are added
-	AWS *AWSConnectionArtifact `json:"aws" required:"true" nullable:"false"`
+	AWS   *AWSConnectionArtifact   `json:"aws,omitempty" required:"false" nullable:"false"`
+	Azure *AzureConnectionArtifact `json:"azure,omitempty" required:"false" nullable:"false"`
 }
 
 type GetConnectionArtifactRequest = PostableAccount
@@ -68,7 +75,7 @@ type GettableAccounts struct {
 }
 
 type UpdatableAccount struct {
-	Config *AccountConfig `json:"config" required:"true" nullable:"false"`
+	Config *UpdatableAccountConfig `json:"config" required:"true" nullable:"false"`
 }
 
 func NewAccount(orgID valuer.UUID, provider CloudProviderType, config *AccountConfig) *Account {
@@ -119,6 +126,13 @@ func NewAccountFromStorable(storableAccount *StorableCloudIntegration) (*Account
 			return nil, err
 		}
 		account.Config.AWS = awsConfig
+	case CloudProviderTypeAzure:
+		azureConfig := new(AzureAccountConfig)
+		err := json.Unmarshal([]byte(storableAccount.Config), azureConfig)
+		if err != nil {
+			return nil, err
+		}
+		account.Config.Azure = azureConfig
 	}
 
 	if storableAccount.LastAgentReport != nil {
@@ -179,6 +193,24 @@ func NewAccountConfigFromPostable(provider CloudProviderType, config *PostableAc
 		}
 
 		return &AccountConfig{AWS: &AWSAccountConfig{Regions: config.AWS.Regions}}, nil
+	case CloudProviderTypeAzure:
+		if config.Azure == nil {
+			return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "Azure config can not be nil for Azure provider")
+		}
+
+		if config.Azure.DeploymentRegion == "" {
+			return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "deployment region is required for Azure provider")
+		}
+
+		if err := validateAzureRegion(config.Azure.DeploymentRegion); err != nil {
+			return nil, err
+		}
+
+		if len(config.Azure.ResourceGroups) == 0 {
+			return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "at least one resource group is required for Azure provider")
+		}
+
+		return &AccountConfig{Azure: &AzureAccountConfig{DeploymentRegion: config.Azure.DeploymentRegion, ResourceGroups: config.Azure.ResourceGroups}}, nil
 	default:
 		return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
 	}
@@ -202,6 +234,16 @@ func NewAccountConfigFromUpdatable(provider CloudProviderType, config *Updatable
 		}
 
 		return &AccountConfig{AWS: &AWSAccountConfig{Regions: config.Config.AWS.Regions}}, nil
+	case CloudProviderTypeAzure:
+		if config.Config.Azure == nil {
+			return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "Azure config can not be nil for Azure provider")
+		}
+
+		if len(config.Config.Azure.ResourceGroups) == 0 {
+			return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "at least one resource group is required for Azure provider")
+		}
+
+		return &AccountConfig{Azure: &AzureAccountConfig{ResourceGroups: config.Config.Azure.ResourceGroups}}, nil
 	default:
 		return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
 	}
@@ -223,8 +265,14 @@ func GetSigNozAPIURLFromDeployment(deployment *zeustypes.GettableDeployment) (st
 }
 
 func (account *Account) Update(provider CloudProviderType, config *AccountConfig) error {
+	// deployment region can not be updated once set for Azure
+	if provider == CloudProviderTypeAzure {
+		config.Azure.DeploymentRegion = account.Config.Azure.DeploymentRegion
+	}
+
 	account.Config = config
 	account.UpdatedAt = time.Now()
+
 	return nil
 }
 
@@ -288,6 +336,10 @@ func (config *AccountConfig) ToJSON() ([]byte, error) {
 		return json.Marshal(config.AWS)
 	}
 
+	if config.Azure != nil {
+		return json.Marshal(config.Azure)
+	}
+
 	return nil, errors.NewInternalf(errors.CodeInternal, "no provider account config found")
 }
 

pkg/types/cloudintegrationtypes/checkin.go

@@ -48,7 +48,8 @@ type IntegrationConfig struct {
 }
 
 type ProviderIntegrationConfig struct {
-	AWS *AWSIntegrationConfig `json:"aws" required:"true" nullable:"false"`
+	AWS   *AWSIntegrationConfig   `json:"aws,omitempty" required:"false" nullable:"false"`
+	Azure *AzureIntegrationConfig `json:"azure,omitempty" required:"false" nullable:"false"`
 }
 
 // NewGettableAgentCheckIn constructs a backward-compatible response from an AgentCheckInResponse.

pkg/types/cloudintegrationtypes/cloudintegration.go

@@ -61,7 +61,8 @@ type StorableCloudIntegrationService struct {
 // Following Service config types are only internally used to store service config in DB and use JSON snake case keys for backward compatibility.
 
 type StorableServiceConfig struct {
-	AWS *StorableAWSServiceConfig
+	AWS   *StorableAWSServiceConfig
+	Azure *StorableAzureServiceConfig
 }
 
 type StorableAWSServiceConfig struct {
@@ -78,6 +79,19 @@ type StorableAWSMetricsServiceConfig struct {
 	Enabled bool `json:"enabled"`
 }
 
+type StorableAzureServiceConfig struct {
+	Logs    *StorableAzureLogsServiceConfig    `json:"logs,omitempty"`
+	Metrics *StorableAzureMetricsServiceConfig `json:"metrics,omitempty"`
+}
+
+type StorableAzureLogsServiceConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
+type StorableAzureMetricsServiceConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
 // Scan scans value from DB.
 func (r *StorableAgentReport) Scan(src any) error {
 	var data []byte
@@ -187,6 +201,30 @@ func newStorableServiceConfig(provider CloudProviderType, serviceID ServiceID, s
 		}
 
 		return &StorableServiceConfig{AWS: storableAWSServiceConfig}, nil
+	case CloudProviderTypeAzure:
+		storableAzureServiceConfig := new(StorableAzureServiceConfig)
+
+		if supportedSignals.Logs {
+			if serviceConfig.Azure.Logs == nil {
+				return nil, errors.NewInvalidInputf(ErrCodeCloudIntegrationInvalidConfig, "logs config is required for Azure service: %s", serviceID.StringValue())
+			}
+
+			storableAzureServiceConfig.Logs = &StorableAzureLogsServiceConfig{
+				Enabled: serviceConfig.Azure.Logs.Enabled,
+			}
+		}
+
+		if supportedSignals.Metrics {
+			if serviceConfig.Azure.Metrics == nil {
+				return nil, errors.NewInvalidInputf(ErrCodeCloudIntegrationInvalidConfig, "metrics config is required for Azure service: %s", serviceID.StringValue())
+			}
+
+			storableAzureServiceConfig.Metrics = &StorableAzureMetricsServiceConfig{
+				Enabled: serviceConfig.Azure.Metrics.Enabled,
+			}
+		}
+
+		return &StorableServiceConfig{Azure: storableAzureServiceConfig}, nil
 	default:
 		return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
 	}
@@ -201,6 +239,13 @@ func newStorableServiceConfigFromJSON(provider CloudProviderType, jsonStr string
 			return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't parse AWS service config JSON")
 		}
 		return &StorableServiceConfig{AWS: awsConfig}, nil
+	case CloudProviderTypeAzure:
+		azureConfig := new(StorableAzureServiceConfig)
+		err := json.Unmarshal([]byte(jsonStr), azureConfig)
+		if err != nil {
+			return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't parse Azure service config JSON")
+		}
+		return &StorableServiceConfig{Azure: azureConfig}, nil
 	default:
 		return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
 	}
@@ -214,6 +259,13 @@ func (config *StorableServiceConfig) toJSON(provider CloudProviderType) ([]byte,
 			return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't serialize AWS service config to JSON")
 		}
 
+		return jsonBytes, nil
+	case CloudProviderTypeAzure:
+		jsonBytes, err := json.Marshal(config.Azure)
+		if err != nil {
+			return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't serialize Azure service config to JSON")
+		}
+
 		return jsonBytes, nil
 	default:
 		return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())

pkg/types/cloudintegrationtypes/cloudprovider_aws.go

@@ -23,6 +23,8 @@ type AWSAccountConfig struct {
 	Regions []string `json:"regions" required:"true" nullable:"false"`
 }
 
+type UpdatableAWSAccountConfig = AWSAccountConfig
+
 // OldAWSCollectionStrategy is the backward-compatible snake_case form of AWSCollectionStrategy,
 // used in the legacy integration_config response field for older agents.
 type OldAWSCollectionStrategy struct {

pkg/types/cloudintegrationtypes/cloudprovider_azure.go

@@ -0,0 +1,64 @@
+package cloudintegrationtypes
+
+type AzureAccountConfig struct {
+	DeploymentRegion string   `json:"deploymentRegion" required:"true"`
+	ResourceGroups   []string `json:"resourceGroups" required:"true" nullable:"false"`
+}
+
+type UpdatableAzureAccountConfig struct {
+	ResourceGroups []string `json:"resourceGroups" required:"true" nullable:"false"`
+}
+
+type AzurePostableAccountConfig = AzureAccountConfig
+
+type AzureConnectionArtifact struct {
+	CLICommand             string `json:"cliCommand" required:"true"`
+	CloudPowerShellCommand string `json:"cloudPowerShellCommand" required:"true"`
+}
+
+type AzureServiceConfig struct {
+	Logs    *AzureServiceLogsConfig    `json:"logs" required:"true"`
+	Metrics *AzureServiceMetricsConfig `json:"metrics" required:"true"`
+}
+
+type AzureServiceLogsConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
+type AzureServiceMetricsConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
+type AzureTelemetryCollectionStrategy struct {
+	//https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-providers-and-types
+	ResourceProvider string                          `json:"resourceProvider" required:"true"`
+	ResourceType     string                          `json:"resourceType" required:"true"`
+	Metrics          *AzureMetricsCollectionStrategy `json:"metrics,omitempty" required:"false" nullable:"false"`
+	Logs             *AzureLogsCollectionStrategy    `json:"logs,omitempty" required:"false" nullable:"false"`
+}
+
+// AzureMetricsCollectionStrategy no additional config required for metrics, will be added in future as required.
+type AzureMetricsCollectionStrategy struct{}
+
+type AzureLogsCollectionStrategy struct {
+	// List of categories to enable for diagnostic settings, to start with it will have 'allLogs' and no filtering.
+	CategoryGroups []string `json:"categoryGroups" required:"true" nullable:"false"`
+}
+
+type AzureIntegrationConfig struct {
+	DeploymentRegion            string                              `json:"deploymentRegion" required:"true"`
+	ResourceGroups              []string                            `json:"resourceGroups" required:"true" nullable:"false"`
+	TelemetryCollectionStrategy []*AzureTelemetryCollectionStrategy `json:"telemetryCollectionStrategy" required:"true" nullable:"false"`
+}
+
+func NewAzureIntegrationConfig(
+	deploymentRegion string,
+	resourceGroups []string,
+	strategies []*AzureTelemetryCollectionStrategy,
+) *AzureIntegrationConfig {
+	return &AzureIntegrationConfig{
+		DeploymentRegion:            deploymentRegion,
+		ResourceGroups:              resourceGroups,
+		TelemetryCollectionStrategy: strategies,
+	}
+}

pkg/types/cloudintegrationtypes/regions.go

@@ -165,3 +165,13 @@ func validateAWSRegion(region string) error {
 
 	return errors.NewInvalidInputf(ErrCodeInvalidCloudRegion, "invalid AWS region: %s", region)
 }
+
+func validateAzureRegion(region string) error {
+	for _, r := range SupportedRegions[CloudProviderTypeAzure] {
+		if r.StringValue() == region {
+			return nil
+		}
+	}
+
+	return errors.NewInvalidInputf(ErrCodeInvalidCloudRegion, "invalid Azure region: %s", region)
+}

pkg/types/cloudintegrationtypes/service.go

@@ -21,8 +21,8 @@ type CloudIntegrationService struct {
 }
 
 type ServiceConfig struct {
-	// required till new providers are added
-	AWS *AWSServiceConfig `json:"aws" required:"true" nullable:"false"`
+	AWS   *AWSServiceConfig   `json:"aws,omitempty" required:"false" nullable:"false"`
+	Azure *AzureServiceConfig `json:"azure,omitempty" required:"false" nullable:"false"`
 }
 
 // ServiceMetadata helps to quickly list available services and whether it is enabled or not.
@@ -88,7 +88,8 @@ type DataCollected struct {
 // TelemetryCollectionStrategy is cloud provider specific configuration for signal collection,
 // this is used by agent to understand the nitty-gritty for collecting telemetry for the cloud provider.
 type TelemetryCollectionStrategy struct {
-	AWS *AWSTelemetryCollectionStrategy `json:"aws" required:"true" nullable:"false"`
+	AWS   *AWSTelemetryCollectionStrategy   `json:"aws,omitempty" required:"false" nullable:"false"`
+	Azure *AzureTelemetryCollectionStrategy `json:"azure,omitempty" required:"false" nullable:"false"`
 }
 
 // Assets represents the collection of dashboards.
@@ -122,7 +123,18 @@ type Dashboard struct {
 	Definition  dashboardtypes.StorableDashboardData `json:"definition,omitempty"`
 }
 
-func NewCloudIntegrationService(serviceID ServiceID, cloudIntegrationID valuer.UUID, config *ServiceConfig) *CloudIntegrationService {
+func NewCloudIntegrationService(serviceID ServiceID, cloudIntegrationID valuer.UUID, provider CloudProviderType, config *ServiceConfig) (*CloudIntegrationService, error) {
+	switch provider {
+	case CloudProviderTypeAWS:
+		if config.AWS == nil {
+			return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "AWS config is required for AWS service")
+		}
+	case CloudProviderTypeAzure:
+		if config.Azure == nil {
+			return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "Azure config is required for Azure service")
+		}
+	}
+
 	return &CloudIntegrationService{
 		Identifiable: types.Identifiable{
 			ID: valuer.GenerateUUID(),
@@ -134,7 +146,7 @@ func NewCloudIntegrationService(serviceID ServiceID, cloudIntegrationID valuer.U
 		Type:               serviceID,
 		Config:             config,
 		CloudIntegrationID: cloudIntegrationID,
-	}
+	}, nil
 }
 
 func NewCloudIntegrationServiceFromStorable(stored *StorableCloudIntegrationService, config *ServiceConfig) *CloudIntegrationService {
@@ -191,6 +203,22 @@ func NewServiceConfigFromJSON(provider CloudProviderType, jsonString string) (*S
 		}
 
 		return &ServiceConfig{AWS: awsServiceConfig}, nil
+	case CloudProviderTypeAzure:
+		azureServiceConfig := new(AzureServiceConfig)
+
+		if storableServiceConfig.Azure.Logs != nil {
+			azureServiceConfig.Logs = &AzureServiceLogsConfig{
+				Enabled: storableServiceConfig.Azure.Logs.Enabled,
+			}
+		}
+
+		if storableServiceConfig.Azure.Metrics != nil {
+			azureServiceConfig.Metrics = &AzureServiceMetricsConfig{
+				Enabled: storableServiceConfig.Azure.Metrics.Enabled,
+			}
+		}
+
+		return &ServiceConfig{Azure: azureServiceConfig}, nil
 	default:
 		return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
 	}
@@ -228,6 +256,10 @@ func (config *ServiceConfig) IsServiceEnabled(provider CloudProviderType) bool {
 		logsEnabled := config.AWS.Logs != nil && config.AWS.Logs.Enabled
 		metricsEnabled := config.AWS.Metrics != nil && config.AWS.Metrics.Enabled
 		return logsEnabled || metricsEnabled
+	case CloudProviderTypeAzure:
+		logsEnabled := config.Azure.Logs != nil && config.Azure.Logs.Enabled
+		metricsEnabled := config.Azure.Metrics != nil && config.Azure.Metrics.Enabled
+		return logsEnabled || metricsEnabled
 	default:
 		return false
 	}
@@ -239,6 +271,8 @@ func (config *ServiceConfig) IsMetricsEnabled(provider CloudProviderType) bool {
 	switch provider {
 	case CloudProviderTypeAWS:
 		return config.AWS.Metrics != nil && config.AWS.Metrics.Enabled
+	case CloudProviderTypeAzure:
+		return config.Azure.Metrics != nil && config.Azure.Metrics.Enabled
 	default:
 		return false
 	}
@@ -249,6 +283,8 @@ func (config *ServiceConfig) IsLogsEnabled(provider CloudProviderType) bool {
 	switch provider {
 	case CloudProviderTypeAWS:
 		return config.AWS.Logs != nil && config.AWS.Logs.Enabled
+	case CloudProviderTypeAzure:
+		return config.Azure.Logs != nil && config.Azure.Logs.Enabled
 	default:
 		return false
 	}
@@ -331,4 +367,3 @@ func GetDashboardsFromAssets(
 
 	return dashboards
 }
-

pkg/types/cloudintegrationtypes/serviceid.go

@@ -23,6 +23,10 @@ var (
 	AWSServiceS3Sync      = ServiceID{valuer.NewString("s3sync")}
 	AWSServiceSNS         = ServiceID{valuer.NewString("sns")}
 	AWSServiceSQS         = ServiceID{valuer.NewString("sqs")}
+
+	// Azure services.
+	AzureServiceStorageAccountsBlob = ServiceID{valuer.NewString("storageaccountsblob")}
+	AzureServiceCDNProfile          = ServiceID{valuer.NewString("cdnprofile")}
 )
 
 func (ServiceID) Enum() []any {
@@ -40,6 +44,8 @@ func (ServiceID) Enum() []any {
 		AWSServiceS3Sync,
 		AWSServiceSNS,
 		AWSServiceSQS,
+		AzureServiceStorageAccountsBlob,
+		AzureServiceCDNProfile,
 	}
 }
 
@@ -60,6 +66,10 @@ var SupportedServices = map[CloudProviderType][]ServiceID{
 		AWSServiceSNS,
 		AWSServiceSQS,
 	},
+	CloudProviderTypeAzure: {
+		AzureServiceStorageAccountsBlob,
+		AzureServiceCDNProfile,
+	},
 }
 
 func NewServiceID(provider CloudProviderType, service string) (ServiceID, error) {

pkg/types/emailtypes/template.go

@@ -12,13 +12,14 @@ import (
 var (
 	// Templates is a list of all the templates that are supported by the emailing service.
 	// This list should be updated whenever a new template is added.
-	Templates = []TemplateName{TemplateNameInvitationEmail, TemplateNameResetPassword}
+	Templates = []TemplateName{TemplateNameInvitationEmail, TemplateNameResetPassword, TemplateNameAlertEmailNotification}
 )
 
 var (
-	TemplateNameInvitationEmail = TemplateName{valuer.NewString("invitation")}
-	TemplateNameResetPassword   = TemplateName{valuer.NewString("reset_password")}
-	TemplateNameAPIKeyEvent     = TemplateName{valuer.NewString("api_key_event")}
+	TemplateNameInvitationEmail        = TemplateName{valuer.NewString("invitation")}
+	TemplateNameResetPassword          = TemplateName{valuer.NewString("reset_password")}
+	TemplateNameAPIKeyEvent            = TemplateName{valuer.NewString("api_key_event")}
+	TemplateNameAlertEmailNotification = TemplateName{valuer.NewString("alert_email_notification")}
 )
 
 type TemplateName struct{ valuer.String }
@@ -31,6 +32,8 @@ func NewTemplateName(name string) (TemplateName, error) {
 		return TemplateNameResetPassword, nil
 	case TemplateNameAPIKeyEvent.StringValue():
 		return TemplateNameAPIKeyEvent, nil
+	case TemplateNameAlertEmailNotification.StringValue():
+		return TemplateNameAlertEmailNotification, nil
 	default:
 		return TemplateName{}, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid template name: %s", name)
 	}

pkg/types/ruletypes/compare.go

@@ -77,6 +77,28 @@ func (c CompareOperator) Normalize() CompareOperator {
 	}
 }
 
+// Literal returns the canonical literal (string) form of the operator.
+func (c CompareOperator) Literal() string {
+	switch c.Normalize() {
+	case ValueIsAbove:
+		return ValueIsAboveLiteral.StringValue()
+	case ValueIsBelow:
+		return ValueIsBelowLiteral.StringValue()
+	case ValueIsEq:
+		return ValueIsEqLiteral.StringValue()
+	case ValueIsNotEq:
+		return ValueIsNotEqLiteral.StringValue()
+	case ValueAboveOrEq:
+		return ValueAboveOrEqLiteral.StringValue()
+	case ValueBelowOrEq:
+		return ValueBelowOrEqLiteral.StringValue()
+	case ValueOutsideBounds:
+		return ValueOutsideBoundsLiteral.StringValue()
+	default:
+		return c.StringValue()
+	}
+}
+
 func (c CompareOperator) Validate() error {
 	switch c {
 	case ValueIsAbove,

pkg/types/ruletypes/match.go

@@ -56,6 +56,24 @@ func (m MatchType) Normalize() MatchType {
 	}
 }
 
+// Literal returns the canonical literal (string) form of the match type.
+func (m MatchType) Literal() string {
+	switch m.Normalize() {
+	case AtleastOnce:
+		return AtleastOnceLiteral.StringValue()
+	case AllTheTimes:
+		return AllTheTimesLiteral.StringValue()
+	case OnAverage:
+		return OnAverageLiteral.StringValue()
+	case InTotal:
+		return InTotalLiteral.StringValue()
+	case Last:
+		return LastLiteral.StringValue()
+	default:
+		return m.StringValue()
+	}
+}
+
 func (m MatchType) Validate() error {
 	switch m {
 	case

pkg/types/ruletypes/result_types.go

@@ -24,6 +24,10 @@ type Sample struct {
 	RecoveryTarget *float64
 
 	TargetUnit string
+
+	// CompareOperator and MatchType carry the threshold evaluation context
+	CompareOperator CompareOperator
+	MatchType       MatchType
 }
 
 func (s Sample) String() string {

pkg/types/ruletypes/templating.go

@@ -0,0 +1,17 @@
+package ruletypes
+
+var CustomTemplatingAnnotations = []string{
+	AnnotationTitleTemplate,
+	AnnotationBodyTemplate,
+}
+
+// IsCustomTemplatingAnnotation checks if the given annotation is a custom templating annotation
+// in order to avoid expanding them in the rule manager layer.
+func IsCustomTemplatingAnnotation(name string) bool {
+	for _, annotation := range CustomTemplatingAnnotations {
+		if annotation == name {
+			return true
+		}
+	}
+	return false
+}

pkg/types/ruletypes/threshold.go

@@ -167,6 +167,8 @@ func (r BasicRuleThresholds) Eval(s *qbtypes.TimeSeries, unit string, evalData E
 				smpl.RecoveryTarget = threshold.RecoveryTarget
 			}
 			smpl.TargetUnit = threshold.TargetUnit
+			smpl.CompareOperator = threshold.CompareOperator
+			smpl.MatchType = threshold.MatchType
 			resultVector = append(resultVector, smpl)
 			continue
 		} else if evalData.SendUnmatched {
@@ -176,10 +178,12 @@ func (r BasicRuleThresholds) Eval(s *qbtypes.TimeSeries, unit string, evalData E
 			}
 			// prepare the sample with the first point of the series
 			smpl := Sample{
-				Point:      Point{T: series.Values[0].Timestamp, V: series.Values[0].Value},
-				Metric:     PrepareSampleLabelsForRule(series.Labels, threshold.Name),
-				Target:     *threshold.TargetValue,
-				TargetUnit: threshold.TargetUnit,
+				Point:           Point{T: series.Values[0].Timestamp, V: series.Values[0].Value},
+				Metric:          PrepareSampleLabelsForRule(series.Labels, threshold.Name),
+				Target:          *threshold.TargetValue,
+				TargetUnit:      threshold.TargetUnit,
+				CompareOperator: threshold.CompareOperator,
+				MatchType:       threshold.MatchType,
 			}
 			if threshold.RecoveryTarget != nil {
 				smpl.RecoveryTarget = threshold.RecoveryTarget
@@ -201,6 +205,8 @@ func (r BasicRuleThresholds) Eval(s *qbtypes.TimeSeries, unit string, evalData E
 				smpl.Target = *threshold.TargetValue
 				smpl.RecoveryTarget = threshold.RecoveryTarget
 				smpl.TargetUnit = threshold.TargetUnit
+				smpl.CompareOperator = threshold.CompareOperator
+				smpl.MatchType = threshold.MatchType
 				// IsRecovering to notify that metrics is in recovery stage
 				smpl.IsRecovering = true
 				resultVector = append(resultVector, smpl)

pkg/types/serviceaccounttypes/service_acccount.go

@@ -238,6 +238,7 @@ type Store interface {
 	GetActiveByOrgIDAndName(context.Context, valuer.UUID, string) (*ServiceAccount, error)
 	GetByID(context.Context, valuer.UUID) (*ServiceAccount, error)
 	GetByIDAndStatus(context.Context, valuer.UUID, ServiceAccountStatus) (*ServiceAccount, error)
+	GetServiceAccountsByOrgIDAndRoleID(context.Context, valuer.UUID, valuer.UUID) ([]*ServiceAccount, error)
 	CountByOrgID(context.Context, valuer.UUID) (int64, error)
 	List(context.Context, valuer.UUID) ([]*ServiceAccount, error)
 	Update(context.Context, valuer.UUID, *ServiceAccount) error

templates/email/alert_email_notification.gotmpl

@@ -0,0 +1,120 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <title>{{.Title}}</title>
+  <style>
+    code {
+      background: #f0f0f0;
+      padding: 2px 6px;
+      border-radius: 3px;
+      font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
+      font-size: 13px;
+    }
+    pre {
+      background: #f0f0f0;
+      padding: 12px 16px;
+      border-radius: 6px;
+      font-size: 13px;
+      overflow-x: auto;
+      white-space: pre;
+    }
+    pre code {
+      background: none;
+      padding: 0;
+      border-radius: 0;
+      font-size: inherit;
+    }
+    table:not([role="presentation"]) {
+      width: 100%;
+      border-collapse: collapse;
+      font-size: 14px;
+    }
+    table:not([role="presentation"]) th {
+      font-weight: 600;
+      text-align: left;
+      padding: 8px 12px;
+      border-bottom: 2px solid #d0d0d0;
+    }
+    table:not([role="presentation"]) td {
+      padding: 8px 12px;
+      border-bottom: 1px solid #e8e8e8;
+    }
+    table:not([role="presentation"]) tr:last-child td {
+      border-bottom: none;
+    }
+  </style>
+</head>
+
+<body style="margin:0;padding:0;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,'Helvetica Neue',Arial,sans-serif;line-height:1.6;color:#333;background:#fff">
+  <table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0" style="background:#fff">
+    <tr>
+      <td align="center" style="padding:0">
+        <table role="presentation" width="600" cellspacing="0" cellpadding="0" border="0" style="max-width:600px;width:100%;border:1px solid #e2e2e2;border-radius:12px;overflow:hidden">
+
+          <tr>
+            <td align="center" style="padding:20px 20px 12px">
+              <h2 style="margin:0 0 8px;font-size:20px;color:#333">{{.Title}}</h2>
+              <p style="margin:0;font-size:14px;color:#666">
+                Status: <strong>{{.Alert.Status}}</strong>
+                {{if .Alert.TotalFiring}} | Firing: <strong style="color:#e53e3e">{{.Alert.TotalFiring}}</strong>{{end}}
+                {{if .Alert.TotalResolved}} | Resolved: <strong style="color:#38a169">{{.Alert.TotalResolved}}</strong>{{end}}
+              </p>
+            </td>
+          </tr>
+
+          <tr>
+            <td style="padding:0 20px">
+              <table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
+                <tr><td style="border-top:1px solid #e2e2e2;font-size:0;line-height:0" height="1">&nbsp;</td></tr>
+              </table>
+            </td>
+          </tr>
+
+          {{range .Bodies}}
+          <tr>
+            <td style="padding:8px 20px">
+              <table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
+                <tr>
+                  <td style="padding:16px;background:#fafafa;border:1px solid #e8e8e8;border-radius:6px">
+                    {{.}}
+                  </td>
+                </tr>
+              </table>
+            </td>
+          </tr>
+          {{end}}
+
+          {{if .NotificationTemplateData.ExternalURL}}
+          <tr>
+            <td style="padding:16px 20px">
+              <table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
+                <tr>
+                  <td align="center">
+                    <a href="{{.NotificationTemplateData.ExternalURL}}" target="_blank" style="display:inline-block;padding:12px 32px;font-size:14px;font-weight:600;color:#fff;background:#4E74F8;text-decoration:none;border-radius:4px">
+                      View in SigNoz
+                    </a>
+                  </td>
+                </tr>
+              </table>
+            </td>
+          </tr>
+          {{end}}
+
+          <tr>
+            <td align="center" style="padding:8px 16px 16px">
+              <p style="margin:0;font-size:12px;color:#999;line-height:1.5">
+                Sent by SigNoz AlertManager
+              </p>
+            </td>
+          </tr>
+        </table>
+      </td>
+    </tr>
+  </table>
+</body>
+
+</html>

tests/conftest.py

@@ -23,6 +23,7 @@ pytest_plugins = [
     "fixtures.notification_channel",
     "fixtures.alerts",
     "fixtures.cloudintegrations",
+    "fixtures.maildev",
     "fixtures.seeder",
 ]
 

tests/fixtures/alerts.py

@@ -4,6 +4,7 @@ import time
 from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
+from urllib.parse import urlparse
 
 import pytest
 import requests
@@ -13,6 +14,7 @@ from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.fs import get_testdata_file_path
 from fixtures.logger import setup_logger
 from fixtures.logs import Logs
+from fixtures.maildev import verify_email_received
 from fixtures.metrics import Metrics
 from fixtures.traces import Traces
 
@@ -218,3 +220,118 @@ def update_rule_channel_name(rule_data: dict, channel_name: str):
         # loop over all the sepcs and update the channels
         for spec in thresholds["spec"]:
             spec["channels"] = [channel_name]
+
+
+def _is_json_subset(subset, superset) -> bool:
+    """Check if subset is contained within superset recursively.
+    - For dicts: all keys in subset must exist in superset with matching values
+    - For lists: all items in subset must be present in superset
+    - For scalars: exact equality
+    """
+    if isinstance(subset, dict):
+        if not isinstance(superset, dict):
+            return False
+        return all(key in superset and _is_json_subset(value, superset[key]) for key, value in subset.items())
+    if isinstance(subset, list):
+        if not isinstance(superset, list):
+            return False
+        return all(any(_is_json_subset(sub_item, sup_item) for sup_item in superset) for sub_item in subset)
+    return subset == superset
+
+
+def verify_webhook_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    validation_data: dict,
+) -> bool:
+    """Check if wiremock received a request at the given path
+    whose JSON body is a superset of the expected json_body."""
+    path = validation_data["path"]
+    json_body = validation_data["json_body"]
+
+    url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+    res = requests.post(url, json={"method": "POST", "url": path}, timeout=5)
+    assert res.status_code == HTTPStatus.OK, f"Failed to find requests for path {path}, status code: {res.status_code}, response: {res.text}"
+
+    for req in res.json()["requests"]:
+        body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+        # logger.info("Webhook request body: %s", json.dumps(body, indent=2))
+        if _is_json_subset(json_body, body):
+            return True
+    return False
+
+
+def _check_notification_validation(
+    validation: types.NotificationValidation,
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+) -> bool:
+    """Dispatch a single validation check to the appropriate verifier."""
+    if validation.destination_type == "webhook":
+        return verify_webhook_notification_expectation(notification_channel, validation.validation_data)
+    if validation.destination_type == "email":
+        return verify_email_received(maildev, validation.validation_data)
+    raise ValueError(f"Invalid destination type: {validation.destination_type}")
+
+
+def verify_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    expected_notification: types.AMNotificationExpectation,
+) -> bool:
+    """Poll for expected notifications across webhook and email channels."""
+    time_to_wait = datetime.now() + timedelta(seconds=expected_notification.wait_time_seconds)
+
+    while datetime.now() < time_to_wait:
+        all_found = all(_check_notification_validation(v, notification_channel, maildev) for v in expected_notification.notification_validations)
+
+        if expected_notification.should_notify and all_found:
+            logger.info("All expected notifications found")
+            return True
+
+        time.sleep(1)
+
+    # Timeout reached
+    if not expected_notification.should_notify:
+        # Verify no notifications were received
+        for validation in expected_notification.notification_validations:
+            found = _check_notification_validation(validation, notification_channel, maildev)
+            assert not found, f"Expected no notification but found one for {validation.destination_type} with data {validation.validation_data}"
+        logger.info("No notifications found, as expected")
+        return True
+
+    # Expected notifications but didn't get them all — report missing
+    missing = [v for v in expected_notification.notification_validations if not _check_notification_validation(v, notification_channel, maildev)]
+    assert len(missing) == 0, f"Expected all notifications to be found but missing: {missing}"
+    return True
+
+
+def update_raw_channel_config(
+    channel_config: dict,
+    channel_name: str,
+    notification_channel: types.TestContainerDocker,
+) -> dict:
+    """
+    Updates the channel config to point to the given wiremock
+    notification_channel container to receive notifications.
+    """
+    config = channel_config.copy()
+
+    config["name"] = channel_name
+
+    url_field_map = {
+        "slack_configs": "api_url",
+        "msteamsv2_configs": "webhook_url",
+        "webhook_configs": "url",
+        "pagerduty_configs": "url",
+        "opsgenie_configs": "api_url",
+    }
+
+    for config_key, url_field in url_field_map.items():
+        if config_key in config:
+            for entry in config[config_key]:
+                if url_field in entry:
+                    original_url = entry[url_field]
+                    path = urlparse(original_url).path
+                    entry[url_field] = notification_channel.container_configs["8080"].get(path)
+
+    return config

tests/fixtures/http.py

@@ -124,14 +124,19 @@ def gateway(
 
 
 @pytest.fixture(name="make_http_mocks", scope="function")
-def make_http_mocks() -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
+def make_http_mocks(
+    request: pytest.FixtureRequest,
+) -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
     def _make_http_mocks(container: types.TestContainerDocker, mappings: list[Mapping]) -> None:
         Config.base_url = container.host_configs["8080"].get("/__admin")
 
         for mapping in mappings:
             Mappings.create_mapping(mapping=mapping)
 
-    yield _make_http_mocks
+        def cleanup():
+            Mappings.delete_all_mappings()
+            Requests.reset_request_journal()
 
-    Mappings.delete_all_mappings()
-    Requests.reset_request_journal()
+        request.addfinalizer(cleanup)
+
+    return _make_http_mocks

tests/fixtures/maildev.py

@@ -0,0 +1,122 @@
+import json
+from http import HTTPStatus
+
+import docker
+import docker.errors
+import pytest
+import requests
+from testcontainers.core.container import DockerContainer, Network
+
+from fixtures import reuse, types
+from fixtures.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+@pytest.fixture(name="maildev", scope="package")
+def maildev(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    """
+    Package-scoped fixture for MailDev container.
+    Provides SMTP (port 1025) and HTTP API (port 1080) for email testing.
+    """
+
+    def create() -> types.TestContainerDocker:
+        container = DockerContainer(image="maildev/maildev:2.2.1")
+        container.with_exposed_ports(1025, 1080)
+        container.with_network(network=network)
+        container.start()
+
+        return types.TestContainerDocker(
+            id=container.get_wrapped_container().id,
+            host_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1025),
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1080),
+                ),
+            },
+            container_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_wrapped_container().name,
+                    port=1025,
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_wrapped_container().name,
+                    port=1080,
+                ),
+            },
+        )
+
+    def delete(container: types.TestContainerDocker):
+        client = docker.from_env()
+        try:
+            client.containers.get(container_id=container.id).stop()
+            client.containers.get(container_id=container.id).remove(v=True)
+        except docker.errors.NotFound:
+            logger.info(
+                "Skipping removal of MailDev, MailDev(%s) not found. Maybe it was manually removed?",
+                {"id": container.id},
+            )
+
+    def restore(cache: dict) -> types.TestContainerDocker:
+        return types.TestContainerDocker.from_cache(cache)
+
+    return reuse.wrap(
+        request,
+        pytestconfig,
+        "maildev",
+        lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
+        create,
+        delete,
+        restore,
+    )
+
+
+def get_all_mails(_maildev: types.TestContainerDocker) -> list[dict]:
+    """
+    Fetches all emails from the MailDev HTTP API.
+    Returns list of dicts with keys: subject, html, text.
+    """
+    url = _maildev.host_configs["1080"].get("/email")
+    response = requests.get(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to fetch emails from MailDev, status code: {response.status_code}, response: {response.text}"
+    emails = response.json()
+    # logger.info("Emails: %s", json.dumps(emails, indent=2))
+    return [
+        {
+            "subject": email.get("subject", ""),
+            "html": email.get("html", ""),
+            "text": email.get("text", ""),
+        }
+        for email in emails
+    ]
+
+
+def verify_email_received(_maildev: types.TestContainerDocker, filters: dict) -> bool:
+    """
+    Checks if any email in MailDev matches all the given filters.
+    Filters are matched with exact equality against the email fields (subject, html, text).
+    Returns True if at least one matching email is found.
+    """
+    emails = get_all_mails(_maildev)
+    for email in emails:
+        logger.info("Email: %s", json.dumps(email, indent=2))
+        if all(key in email and filter_value == email[key] for key, filter_value in filters.items()):
+            return True
+    return False
+
+
+def delete_all_mails(_maildev: types.TestContainerDocker) -> None:
+    """
+    Deletes all emails from the MailDev inbox.
+    """
+    url = _maildev.host_configs["1080"].get("/email/all")
+    response = requests.delete(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to delete emails from MailDev, status code: {response.status_code}, response: {response.text}"

tests/fixtures/notification_channel.py

@@ -1,3 +1,4 @@
+# pylint: disable=line-too-long
 from collections.abc import Callable
 from http import HTTPStatus
 
@@ -15,6 +16,87 @@ from fixtures.logger import setup_logger
 logger = setup_logger(__name__)
 
 
+"""
+Default notification channel configs shared across alertmanager tests.
+"""
+slack_default_config = {
+    # channel name configured on runtime
+    "slack_configs": [
+        {
+            "api_url": "services/TEAM_ID/BOT_ID/TOKEN_ID",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+# MSTeams default config
+msteams_default_config = {
+    "msteamsv2_configs": [
+        {
+            "webhook_url": "msteams/webhook_url",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+# pagerduty default config
+pagerduty_default_config = {
+    "pagerduty_configs": [
+        {
+            "routing_key": "PagerDutyRoutingKey",
+            "url": "v2/enqueue",  # base_url configured on runtime
+            "client": "SigNoz Alert Manager",
+            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+            "description": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n\t{{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n\t {{" "}}(\n\t {{- with .CommonLabels.Remove .GroupLabels.Names }}\n\t\t{{- range $index, $label := .SortedPairs -}}\n\t\t {{ if $index }}, {{ end }}\n\t\t {{- $label.Name }}="{{ $label.Value -}}"\n\t\t{{- end }}\n\t {{- end -}}\n\t )\n\t{{- end }}',
+            "details": {
+                "firing": '{{ template "pagerduty.default.instances" .Alerts.Firing }}',
+                "num_firing": "{{ .Alerts.Firing | len }}",
+                "num_resolved": "{{ .Alerts.Resolved | len }}",
+                "resolved": '{{ template "pagerduty.default.instances" .Alerts.Resolved }}',
+            },
+            "source": "SigNoz Alert Manager",
+            "severity": "{{ (index .Alerts 0).Labels.severity }}",
+        }
+    ],
+}
+# opsgenie default config
+opsgenie_default_config = {
+    "opsgenie_configs": [
+        {
+            "api_key": "OpsGenieAPIKey",
+            "api_url": "/",  # base_url configured on runtime
+            "description": '{{ if gt (len .Alerts.Firing) 0 -}}\r\n\tAlerts Firing:\r\n\t{{ range .Alerts.Firing }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n\tAlerts Resolved:\r\n\t{{ range .Alerts.Resolved }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}',
+            "priority": '{{ if eq (index .Alerts 0).Labels.severity "critical" }}P1{{ else if eq (index .Alerts 0).Labels.severity "warning" }}P2{{ else if eq (index .Alerts 0).Labels.severity "info" }}P3{{ else }}P4{{ end }}',
+            "message": "{{ .CommonLabels.alertname }}",
+            "details": {},
+        }
+    ]
+}
+
+# webhook default config
+webhook_default_config = {
+    "webhook_configs": [
+        {
+            "url": "webhook/webhook_url",  # base_url configured on runtime
+        }
+    ],
+}
+# email default config
+email_default_config = {
+    "email_configs": [
+        {
+            "to": "test@example.com",
+            "html": '<html><body>{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}</body></html>',
+            "headers": {
+                "Subject": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}'
+            },
+        }
+    ],
+}
+
+
 @pytest.fixture(name="notification_channel", scope="package")
 def notification_channel(
     network: Network,
@@ -67,6 +149,27 @@ def notification_channel(
     )
 
 
+@pytest.fixture(name="create_notification_channel", scope="function")
+def create_notification_channel(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> Callable[[dict], str]:
+    admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    def _create_notification_channel(channel_config: dict) -> str:
+        response = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels"),
+            json=channel_config,
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        assert response.status_code == HTTPStatus.CREATED, f"Failed to create channel, Response: {response.text} Response status: {response.status_code}"
+        return response.json()["data"]["id"]
+
+    return _create_notification_channel
+
+
 @pytest.fixture(name="create_webhook_notification_channel", scope="function")
 def create_webhook_notification_channel(
     signoz: types.SigNoz,

tests/fixtures/types.py

@@ -192,3 +192,40 @@ class AlertTestCase:
     alert_data: list[AlertData]
     # list of alert expectations for the test case
     alert_expectation: AlertExpectation
+
+
+@dataclass(frozen=True)
+class NotificationValidation:
+    # destination type of the notification, either webhook or email
+    # slack, msteams, pagerduty, opsgenie, webhook channels send notifications through webhook
+    # email channels send notifications through email
+    destination_type: Literal["webhook", "email"]
+    # validation data for validating the received notification payload
+    validation_data: dict[str, any]
+
+
+@dataclass(frozen=True)
+class AMNotificationExpectation:
+    # whether we expect any notifications to be fired or not, false when testing downtime scenarios
+    # or don't expect any notifications to be fired in given time period
+    should_notify: bool
+    # seconds to wait for the notifications to be fired, if no
+    # notifications are fired in the expected time, the test will fail
+    wait_time_seconds: int
+    # list of notifications to expect, as a single rule can trigger multiple notifications
+    # spanning across different notifiers
+    notification_validations: list[NotificationValidation]
+
+
+@dataclass(frozen=True)
+class AlertManagerNotificationTestCase:
+    # name of the test case
+    name: str
+    # path to the rule file in testdata directory
+    rule_path: str
+    # list of alert data that will be inserted into the database for the rule to be triggered
+    alert_data: list[AlertData]
+    # configuration for the notification channel
+    channel_config: dict[str, any]
+    # notification expectations for the test case
+    notification_expectation: AMNotificationExpectation

tests/integration/bootstrap/setup.py

@@ -39,5 +39,7 @@ def test_teardown(
     idp: types.TestContainerIDP,  # pylint: disable=unused-argument
     create_user_admin: types.Operation,  # pylint: disable=unused-argument
     migrator: types.Operation,  # pylint: disable=unused-argument
+    maildev: types.TestContainerDocker,  # pylint: disable=unused-argument
+    notification_channel: types.TestContainerDocker,  # pylint: disable=unused-argument
 ) -> None:
     pass

tests/integration/src/alertmanager/01_notifers.py

@@ -0,0 +1,411 @@
+import json
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    email_default_config,
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+# tests to verify the notifiers sending out the notifications with expected content
+NOTIFIERS_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="slack_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            # extra wait for alertmanager server setup
+            wait_time_seconds=60,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "title": '[FIRING:1] threshold_above_at_least_once for  (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
+                                    "text": "*Alert:* threshold_above_at_least_once - critical\r\n\r\n *Summary:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n *Description:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n *RelatedLogs:* \r\n *RelatedTraces:* \r\n\r\n *Details:*\r\n • *alertname:* threshold_above_at_least_once\r\n   • *severity:* critical\r\n   • *threshold.name:* critical\r\n   ",
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="msteams_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=60,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                                        "type": "AdaptiveCard",
+                                        "version": "1.2",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": '[FIRING:1] threshold_above_at_least_once for  (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                                "wrap": True,
+                                                "style": "heading",
+                                                "color": "Attention",
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Alerts",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                                "wrap": True,
+                                                "color": "Attention",
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Labels",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "threshold.name",
+                                                        "value": "critical",
+                                                    }
+                                                ],
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Annotations",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "threshold.value",
+                                                        "value": "10",
+                                                    },
+                                                    {
+                                                        "title": "compare_op",
+                                                        "value": "above",
+                                                    },
+                                                    {
+                                                        "title": "match_type",
+                                                        "value": "at_least_once",
+                                                    },
+                                                    {"title": "value", "value": "15"},
+                                                    {
+                                                        "title": "description",
+                                                        "value": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                                    },
+                                                ],
+                                            },
+                                        ],
+                                        "msteams": {"width": "full"},
+                                        "actions": [
+                                            {
+                                                "type": "Action.OpenUrl",
+                                                "title": "View Alert",
+                                            }
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=60,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "event_action": "trigger",
+                            "payload": {
+                                "summary": '[FIRING:1] threshold_above_at_least_once for  (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
+                                "source": "SigNoz Alert Manager",
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Annotations": [
+                                            "compare_op = above",
+                                            {"description = This alert is fired when the defined metric (current value": "15) crosses the threshold (10)"},
+                                            "match_type = at_least_once",
+                                            "threshold.value = 10",
+                                            "value = 15",
+                                        ],
+                                        "Labels": [
+                                            "alertname = threshold_above_at_least_once",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=60,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "threshold_above_at_least_once",
+                            "description": "Alerts Firing:\r\n\t\r\n\t - Message: This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\tLabels:\r\n\t   - alertname = threshold_above_at_least_once\r\n\t   - severity = critical\r\n\t   - threshold.name = critical\r\n\t   Annotations:\r\n\t   - compare_op = above\r\n\t   - description = This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\t   - match_type = at_least_once\r\n\t   - summary = This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\t   - threshold.value = 10\r\n\t   - value = 15\r\n\t   Source: \r\n\t\r\n",
+                            "details": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=60,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "threshold_above_at_least_once",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "compare_op": "above",
+                                        "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                        "match_type": "at_least_once",
+                                        "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                        "threshold.value": "10",
+                                        "value": "15",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "commonAnnotations": {
+                                "compare_op": "above",
+                                "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                "match_type": "at_least_once",
+                                "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                "threshold.value": "10",
+                                "value": "15",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="email_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=email_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=60,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="email",
+                    validation_data={
+                        "subject": '[FIRING:1] threshold_above_at_least_once for  (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
+                        "html": "<html><body>*Alert:* threshold_above_at_least_once - critical\n\n *Summary:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\n *Description:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\n *RelatedLogs:* \n *RelatedTraces:* \n\n *Details:*\n \u2022 *alertname:* threshold_above_at_least_once\n   \u2022 *severity:* critical\n   \u2022 *threshold.name:* critical\n   </body></html>",
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+logger = setup_logger(__name__)
+
+
+@pytest.mark.parametrize(
+    "notifier_test_case",
+    NOTIFIERS_TEST,
+    ids=lambda notifier_test_case: notifier_test_case.name,
+)
+def test_notifier_templating(
+    # wiremock container for webhook notifications
+    notification_channel: types.TestContainerDocker,
+    # function to create wiremock mocks
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    # function to create alert rule
+    create_alert_rule: Callable[[dict], str],
+    # Alert data insertion related fixture
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    # Mail dev container for email verification
+    maildev: types.TestContainerDocker,
+    # test case from parametrize
+    notifier_test_case: types.AlertManagerNotificationTestCase,
+):
+    # generate unique channel name
+    channel_name = str(uuid.uuid4())
+
+    # update channel config: set name and rewrite URLs to wiremock
+    channel_config = update_raw_channel_config(notifier_test_case.channel_config, channel_name, notification_channel)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    # setup wiremock mocks for webhook-based notification validations
+    webhook_validations = [v for v in notifier_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    # clear mails if any destination is email
+    if any(v.destination_type == "email" for v in notifier_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    # create notification channel
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    # insert alert data
+    insert_alert_data(
+        notifier_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=5),
+    )
+
+    # create alert rule
+    rule_path = get_testdata_file_path(notifier_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    # verify notification expectations
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        notifier_test_case.notification_expectation,
+    )

tests/integration/src/alertmanager/conftest.py

@@ -0,0 +1,42 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev: types.TestContainerDocker,
+    notification_channel: types.TestContainerDocker,
+) -> types.SigNoz:
+    """
+    Package-scoped fixture for setting up SigNoz.
+    Overrides SMTP, PagerDuty, and OpsGenie URLs to point to test containers.
+    """
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        env_overrides={
+            # SMTP config for email notifications via maildev
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__SMARTHOST": f"{maildev.container_configs['1025'].address}:{maildev.container_configs['1025'].port}",
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__REQUIRE__TLS": "false",
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__FROM": "alertmanager@signoz.io",
+            # PagerDuty API URL -> wiremock (default: https://events.pagerduty.com/v2/enqueue)
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_PAGERDUTY__URL": notification_channel.container_configs["8080"].get("/v2/enqueue"),
+            # OpsGenie API URL -> wiremock (default: https://api.opsgenie.com/)
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_OPSGENIE__API__URL": notification_channel.container_configs["8080"].get("/"),
+        },
+    )

tests/integration/testdata/alertmanager/content_templating/logs_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "User login successful", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Database connection established", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "API request received", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Response sent to client", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }

tests/integration/testdata/alertmanager/content_templating/logs_rule.json

@@ -0,0 +1,71 @@
+{
+  "alert": "content_templating_logs",
+  "ruleType": "threshold_rule",
+  "alertType": "LOGS_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 0,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "logs",
+            "filter": {
+              "expression": "body CONTAINS 'payment failure'"
+            },
+            "aggregations": [
+              {
+                "expression": "count()"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Payment failure spike detected on $service_name",
+    "summary": "Payment failures elevated on $service_name",
+    "_title_template": "[$alert.status] Payment failure spike in $labels.service.name",
+    "_body_template": "**Severity:** $rule.severity\n**Status:** $alert.status\n\n**Service:** $labels.service.name\n\n**Condition ($labels.threshold.name):**\n- **Current:** $alert.value\n- **Threshold:** $rule.threshold.op $rule.threshold.value\n\n**Description:** Payment failures observed at $alert.value over the evaluation window, crossing the $labels.threshold.name threshold of $rule.threshold.op $rule.threshold.value on $labels.service.name. Investigate downstream payment processor health.\n\n**Runbook:** https://signoz.io/docs/runbooks/payment-failure-spike"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/metrics_data.jsonl

@@ -0,0 +1,12 @@
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:01:00+00:00","value":80,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:02:00+00:00","value":95,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:03:00+00:00","value":110,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:04:00+00:00","value":120,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:05:00+00:00","value":125,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:06:00+00:00","value":130,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:07:00+00:00","value":135,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:08:00+00:00","value":140,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:09:00+00:00","value":145,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:10:00+00:00","value":150,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:11:00+00:00","value":155,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:12:00+00:00","value":160,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}

tests/integration/testdata/alertmanager/content_templating/metrics_rule.json

@@ -0,0 +1,74 @@
+{
+  "alert": "content_templating_metrics",
+  "ruleType": "threshold_rule",
+  "alertType": "METRIC_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 100,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "metrics",
+            "aggregations": [
+              {
+                "metricName": "container_memory_bytes_content_templating",
+                "timeAggregation": "avg",
+                "spaceAggregation": "max"
+              }
+            ],
+            "groupBy": [
+              {"name": "namespace", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "pod", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "container", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "node", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "severity", "fieldContext": "attribute", "fieldDataType": "string"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Container $container in pod $pod ($namespace) exceeded memory threshold",
+    "summary": "High container memory in $namespace/$pod",
+    "_title_template": "[$alert.status] High container memory in $labels.namespace/$labels.pod",
+    "_body_template": "**Severity:** $rule.severity\n**Status:** $alert.status\n\n**Pod Details:**\n- **Namespace:** $labels.namespace\n- **Pod:** $labels.pod\n- **Container:** $labels.container\n- **Node:** $labels.node\n\n**Condition ($labels.threshold.name):**\n- **Current:** $alert.value\n- **Threshold:** $rule.threshold.op $rule.threshold.value\n\n**Description:** Container $labels.container in pod $labels.pod ($labels.namespace) has memory usage at $alert.value, which crossed the $labels.threshold.name threshold of $rule.threshold.op $rule.threshold.value. Immediate investigation is recommended to prevent OOMKill.\n\n**Runbook:** https://signoz.io/docs/runbooks/container-memory-near-limit"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/traces_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "duration": "PT1.2S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a1", "span_id": "c1b2c3d4e5f6a7b8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "duration": "PT1.4S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a2", "span_id": "c2b3c4d5e6f7a8b9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "duration": "PT1.6S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a3", "span_id": "c3b4c5d6e7f8a9b0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "duration": "PT1.8S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a4", "span_id": "c4b5c6d7e8f9a0b1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "duration": "PT2.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a5", "span_id": "c5b6c7d8e9f0a1b2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "duration": "PT2.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a6", "span_id": "c6b7c8d9e0f1a2b3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "duration": "PT2.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a7", "span_id": "c7b8c9d0e1f2a3b4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "duration": "PT2.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a8", "span_id": "c8b9c0d1e2f3a4b5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "duration": "PT2.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a9", "span_id": "c9b0c1d2e3f4a5b6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "duration": "PT3.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b1", "span_id": "d1c2d3e4f5a6b7c8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "duration": "PT3.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b2", "span_id": "d2c3d4e5f6a7b8c9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "duration": "PT3.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b3", "span_id": "d3c4d5e6f7a8b9c0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "duration": "PT3.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b4", "span_id": "d4c5d6e7f8a9b0c1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "duration": "PT3.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b5", "span_id": "d5c6d7e8f9a0b1c2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "duration": "PT4.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b6", "span_id": "d6c7d8e9f0a1b2c3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "duration": "PT4.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b7", "span_id": "d7c8d9e0f1a2b3c4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "duration": "PT4.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b8", "span_id": "d8c9d0e1f2a3b4c5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "duration": "PT4.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b9", "span_id": "d9c0d1e2f3a4b5c6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "duration": "PT4.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c1", "span_id": "e1d2e3f4a5b6c7d8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "duration": "PT5.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c2", "span_id": "e2d3e4f5a6b7c8d9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }

tests/integration/testdata/alertmanager/content_templating/traces_rule.json

@@ -0,0 +1,73 @@
+{
+  "alert": "content_templating_traces",
+  "ruleType": "threshold_rule",
+  "alertType": "TRACES_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 1,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ],
+          "targetUnit": "s"
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "unit": "ns",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "traces",
+            "filter": {
+              "expression": "http.request.path = '/checkout'"
+            },
+            "aggregations": [
+              {
+                "expression": "p90(duration_nano)"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "p90 latency high on $service_name",
+    "summary": "p90 latency exceeded threshold on $service_name",
+    "_title_template": "[$alert.status] p90 latency high on $labels.service_name",
+    "_body_template": "**Severity:** $rule.severity\n**Status:** $alert.status\n\n**Service:** $labels.service_name\n\n**Condition ($labels.threshold.name):**\n- **Current:** $alert.value\n- **Threshold:** $rule.threshold.op $rule.threshold.value\n\n**Description:** p90 request latency on $labels.service_name reached $alert.value, crossing the $labels.threshold.name threshold of $rule.threshold.op $rule.threshold.value. Investigate downstream dependencies and recent deploys.\n\n**Runbook:** https://signoz.io/docs/runbooks/high-latency"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/pyproject.toml

@@ -75,3 +75,7 @@ ignore = [
 
 [tool.ruff.format]
 # Defaults align with black (double quotes, 4-space indent).
+
+[tool.ruff.lint.per-file-ignores]
+"integration/src/alertmanager/*" = ["E501"]
+"fixtures/notification_channel.py" = ["E501"]