fix(alertmanager): integration tests fixes (#11710)

* fix: integration tests

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

* test(alertmanager): add logs and traces e2e tests for custom templating

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

* fix: missing annotations

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

* fix: annotations and templating fixes

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

* chore: removed annotations

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

* fix: tests removed annotations

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

* fix: integration tests

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

* chore: py-lint run

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

---------

Signed-off-by: Niladri Adhikary <niladrix719@gmail.com>

.github/workflows/integrationci.yaml

@@ -39,6 +39,7 @@ jobs:
       matrix:
         suite:
           - alerts
+          - alertmanager
           - basepath
           - callbackauthn
           - cloudintegrations

docs/api/openapi.yml

@@ -409,10 +409,6 @@ components:
       properties:
         duration:
           type: string
-        endTime:
-          format: date-time
-          nullable: true
-          type: string
         repeatOn:
           items:
             $ref: '#/components/schemas/AlertmanagertypesRepeatOn'
@@ -420,11 +416,7 @@ components:
           type: array
         repeatType:
           $ref: '#/components/schemas/AlertmanagertypesRepeatType'
-        startTime:
-          format: date-time
-          type: string
       required:
-      - startTime
       - duration
       - repeatType
       type: object
@@ -458,6 +450,7 @@ components:
           type: string
       required:
       - timezone
+      - startTime
       type: object
     AuthtypesAttributeMapping:
       properties:

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -413,21 +413,11 @@ export interface AlertmanagertypesRecurrenceDTO {
 	 * @type string
 	 */
 	duration: string;
-	/**
-	 * @type string,null
-	 * @format date-time
-	 */
-	endTime?: string | null;
 	/**
 	 * @type array,null
 	 */
 	repeatOn?: AlertmanagertypesRepeatOnDTO[] | null;
 	repeatType: AlertmanagertypesRepeatTypeDTO;
-	/**
-	 * @type string
-	 * @format date-time
-	 */
-	startTime: string;
 }
 
 export interface AlertmanagertypesScheduleDTO {
@@ -441,7 +431,7 @@ export interface AlertmanagertypesScheduleDTO {
 	 * @type string
 	 * @format date-time
 	 */
-	startTime?: string;
+	startTime: string;
 	/**
 	 * @type string
 	 */

frontend/src/container/BillingContainer/BillingContainer.module.scss

@@ -1,8 +1,7 @@
 .billingContainer {
-	margin-bottom: var(--spacing-20);
 	padding-top: 36px;
 	width: 90%;
-	margin: 0 auto;
+	margin: 0 auto var(--spacing-20);
 
 	.pageHeader {
 		margin-bottom: var(--spacing-8);

frontend/src/container/GeneralSettings/LicenseKeyRow/LicenseRowDismissibleCallout/LicenseRowDismissible.styles.scss

@@ -1,6 +1,6 @@
 .license-key-callout {
 	margin: var(--spacing-4) var(--spacing-6);
-	width: auto;
+	width: auto !important;
 
 	.license-key-callout__description {
 		display: flex;

frontend/src/container/GeneralSettings/__tests__/index.test.tsx

@@ -0,0 +1,41 @@
+import { useQueries } from 'react-query';
+import { render, screen } from 'tests/test-utils';
+
+import GeneralSettings from '../index';
+
+jest.mock('react-query', () => ({
+	...jest.requireActual('react-query'),
+	useQueries: jest.fn(),
+}));
+
+const baseQueryResult = {
+	isError: false,
+	isLoading: false,
+	isFetching: false,
+	isSuccess: true,
+	data: undefined,
+	error: null,
+	refetch: jest.fn(),
+};
+
+describe('GeneralSettings index', () => {
+	it('renders fallback message when logs query fails with a non-APIError', () => {
+		(useQueries as jest.Mock).mockReturnValue([
+			{ ...baseQueryResult },
+			{ ...baseQueryResult },
+			{
+				...baseQueryResult,
+				isError: true,
+				isSuccess: false,
+				error: new TypeError(
+					"Cannot read properties of undefined (reading 'code')",
+				),
+			},
+			{ ...baseQueryResult },
+		]);
+
+		render(<GeneralSettings />);
+
+		expect(screen.getByText('something_went_wrong')).toBeInTheDocument();
+	});
+});

frontend/src/container/GeneralSettings/index.tsx

@@ -76,7 +76,9 @@ function GeneralSettings(): JSX.Element {
 	if (getRetentionPeriodLogsApiResponse.isError || getDisksResponse.isError) {
 		return (
 			<Typography>
-				{(getRetentionPeriodLogsApiResponse.error as APIError).getErrorMessage() ||
+				{(getRetentionPeriodLogsApiResponse.error instanceof APIError
+					? getRetentionPeriodLogsApiResponse.error.getErrorMessage()
+					: undefined) ||
 					getDisksResponse.data?.error ||
 					t('something_went_wrong')}
 			</Typography>

frontend/src/container/InfraMonitoringK8s/Volumes/table.config.tsx

@@ -86,9 +86,9 @@ export const k8sVolumesColumnsConfig: TableColumnDef<K8sVolumesData>[] = [
 	},
 	{
 		id: 'capacity',
-		header: 'Volume Capacity',
+		header: 'Capacity',
 		accessorFn: (row): number => row.volumeCapacity,
-		width: { min: 220 },
+		width: { min: 140 },
 		enableSort: true,
 		cell: ({ value }): React.ReactNode => {
 			const capacity = value as number;
@@ -105,9 +105,9 @@ export const k8sVolumesColumnsConfig: TableColumnDef<K8sVolumesData>[] = [
 	},
 	{
 		id: 'usage',
-		header: 'Volume Utilization',
+		header: 'Used',
 		accessorFn: (row): number => row.volumeUsage,
-		width: { min: 220 },
+		width: { min: 140 },
 		enableSort: true,
 		cell: ({ value }): React.ReactNode => {
 			const usage = value as number;
@@ -124,9 +124,9 @@ export const k8sVolumesColumnsConfig: TableColumnDef<K8sVolumesData>[] = [
 	},
 	{
 		id: 'available',
-		header: 'Volume Available',
+		header: 'Available',
 		accessorFn: (row): number => row.volumeAvailable,
-		width: { min: 220 },
+		width: { min: 140 },
 		enableSort: true,
 		cell: ({ value }): React.ReactNode => {
 			const available = value as number;
@@ -141,4 +141,61 @@ export const k8sVolumesColumnsConfig: TableColumnDef<K8sVolumesData>[] = [
 			);
 		},
 	},
+	{
+		id: 'inodes',
+		header: 'Inodes',
+		accessorFn: (row): number => row.volumeInodes,
+		width: { min: 140 },
+		enableSort: true,
+		cell: ({ value }): React.ReactNode => {
+			const inodes = value as number;
+			return (
+				<ValidateColumnValueWrapper
+					value={inodes}
+					entity={InfraMonitoringEntity.VOLUMES}
+					attribute="inodes metric"
+				>
+					<TanStackTable.Text>{inodes}</TanStackTable.Text>
+				</ValidateColumnValueWrapper>
+			);
+		},
+	},
+	{
+		id: 'inodesUsed',
+		header: 'Inodes Used',
+		accessorFn: (row): number => row.volumeInodesUsed,
+		width: { min: 160 },
+		enableSort: true,
+		cell: ({ value }): React.ReactNode => {
+			const inodesUsed = value as number;
+			return (
+				<ValidateColumnValueWrapper
+					value={inodesUsed}
+					entity={InfraMonitoringEntity.VOLUMES}
+					attribute="inodes used metric"
+				>
+					<TanStackTable.Text>{inodesUsed}</TanStackTable.Text>
+				</ValidateColumnValueWrapper>
+			);
+		},
+	},
+	{
+		id: 'inodesFree',
+		header: 'Inodes Free',
+		accessorFn: (row): number => row.volumeInodesFree,
+		width: { min: 160 },
+		enableSort: true,
+		cell: ({ value }): React.ReactNode => {
+			const inodesFree = value as number;
+			return (
+				<ValidateColumnValueWrapper
+					value={inodesFree}
+					entity={InfraMonitoringEntity.VOLUMES}
+					attribute="inodes free metric"
+				>
+					<TanStackTable.Text>{inodesFree}</TanStackTable.Text>
+				</ValidateColumnValueWrapper>
+			);
+		},
+	},
 ];

frontend/src/container/PlannedDowntime/PlannedDowntimeForm.tsx

@@ -151,6 +151,11 @@ export function PlannedDowntimeForm(
 
 	const saveHandler = useCallback(
 		async (values: PlannedDowntimeFormData) => {
+			const { startTime, timezone } = values;
+			if (!startTime || !timezone) {
+				// unreachable: required fields should always be present on submitting.
+				return;
+			}
 			const data: AlertmanagertypesPostablePlannedMaintenanceDTO = {
 				alertIds:
 					values.alertRuleScope === 'all'
@@ -161,9 +166,9 @@ export function PlannedDowntimeForm(
 				name: values.name,
 				scope: values.scope,
 				schedule: {
-					startTime: values.startTime?.format(),
+					startTime: startTime.format(),
 					endTime: values.endTime?.format(),
-					timezone: values.timezone!,
+					timezone,
 					recurrence: values.recurrence,
 				},
 			};
@@ -200,25 +205,17 @@ export function PlannedDowntimeForm(
 		],
 	);
 	const onFinish = async (values: PlannedDowntimeFormData): Promise<void> => {
-		const { recurrence } = values;
-		const recurrenceData =
-			!recurrence ||
-			recurrence.repeatType === recurrenceOptions.doesNotRepeat.value
-				? undefined
-				: {
-						duration: recurrence.duration
-							? `${recurrence.duration}${durationUnit}`
-							: '',
-						startTime: values.startTime!.format(),
-						endTime: values.endTime?.format(),
-						repeatOn: recurrence.repeatOn,
-						repeatType: recurrence.repeatType,
-					};
+		const rec = values.recurrence;
+		const recurrence =
+			rec && rec.repeatType !== recurrenceOptions.doesNotRepeat.value
+				? {
+						duration: `${rec.duration}${durationUnit}`,
+						repeatOn: rec.repeatOn,
+						repeatType: rec.repeatType,
+					}
+				: undefined;
 
-		await saveHandler({
-			...values,
-			recurrence: recurrenceData,
-		});
+		await saveHandler({ ...values, recurrence });
 	};
 
 	const handleFormData = (data: Partial<PlannedDowntimeFormData>): void => {
@@ -275,9 +272,6 @@ export function PlannedDowntimeForm(
 
 	const formattedInitialValues = useMemo((): PlannedDowntimeFormData => {
 		const { schedule } = initialValues;
-		const startTime = schedule?.recurrence?.startTime || schedule?.startTime;
-		const endTime = schedule?.recurrence?.endTime || schedule?.endTime;
-
 		const initialAlertIds = initialValues.alertIds || [];
 
 		return {
@@ -285,8 +279,12 @@ export function PlannedDowntimeForm(
 			alertRuleScope:
 				isEditMode && initialAlertIds.length === 0 ? 'all' : 'specific',
 			alertRules: getAlertOptionsFromIds(initialAlertIds, alertOptions),
-			startTime: startTime ? dayjs(startTime).tz(schedule.timezone) : null,
-			endTime: endTime ? dayjs(endTime).tz(schedule.timezone) : null,
+			startTime: schedule?.startTime
+				? dayjs(schedule.startTime).tz(schedule.timezone)
+				: null,
+			endTime: schedule?.endTime
+				? dayjs(schedule.endTime).tz(schedule.timezone)
+				: null,
 			recurrence: {
 				...schedule?.recurrence,
 				repeatType: !isScheduleRecurring(schedule)
@@ -297,7 +295,7 @@ export function PlannedDowntimeForm(
 			timezone: schedule?.timezone as string,
 			scope: initialValues.scope || '',
 		};
-	}, [initialValues, alertOptions]);
+	}, [initialValues, isEditMode, alertOptions]);
 
 	useEffect(() => {
 		setSelectedTags(formattedInitialValues.alertRules);
@@ -341,7 +339,7 @@ export function PlannedDowntimeForm(
 		const formattedEndTime = endTime.format(TIME_FORMAT);
 		const formattedEndDate = endTime.format(DATE_FORMAT);
 		return `Scheduled to end maintenance on ${formattedEndDate} at ${formattedEndTime}.`;
-	}, [formData, recurrenceType]);
+	}, [formData]);
 
 	return (
 		<Modal

frontend/src/container/PlannedDowntime/PlannedDowntimeList.tsx

@@ -142,7 +142,6 @@ export function CollapseListContent({
 	updated_by_name?: string;
 	alertOptions?: DefaultOptionType[];
 }): JSX.Element {
-	const repeats = schedule?.recurrence;
 	const renderItems = (title: string, value: ReactNode): JSX.Element => (
 		<div className="render-item-collapse-list">
 			<Typography>{title}</Typography>
@@ -193,10 +192,7 @@ export function CollapseListContent({
 				'Timezone',
 				<Typography>{schedule?.timezone || '-'}</Typography>,
 			)}
-			{renderItems(
-				'Repeats',
-				<Typography>{recurrenceInfo(repeats, schedule?.timezone)}</Typography>,
-			)}
+			{renderItems('Repeats', <Typography>{recurrenceInfo(schedule)}</Typography>)}
 			{renderItems(
 				'Alerts silenced',
 				alertOptions?.length ? (

frontend/src/container/PlannedDowntime/PlannedDowntimeutils.ts

@@ -6,7 +6,7 @@ import type {
 	DeleteDowntimeScheduleByIDPathParameters,
 	RenderErrorResponseDTO,
 	AlertmanagertypesPlannedMaintenanceDTO,
-	AlertmanagertypesRecurrenceDTO,
+	AlertmanagertypesScheduleDTO,
 } from 'api/generated/services/sigNoz.schemas';
 import type { ErrorType } from 'api/generatedAPIInstance';
 import { AxiosError } from 'axios';
@@ -66,14 +66,17 @@ export const getAlertOptionsFromIds = (
 	);
 
 export const recurrenceInfo = (
-	recurrence?: AlertmanagertypesRecurrenceDTO | null,
-	timezone?: string,
+	schedule?: AlertmanagertypesScheduleDTO | null,
 ): string => {
+	if (!schedule) {
+		return 'No';
+	}
+	const { startTime, endTime, timezone, recurrence } = schedule;
 	if (!recurrence) {
 		return 'No';
 	}
 
-	const { startTime, duration, repeatOn, repeatType, endTime } = recurrence;
+	const { duration, repeatOn, repeatType } = recurrence;
 
 	const formattedStartTime = startTime
 		? formatDateTime(startTime, timezone)
@@ -95,7 +98,7 @@ export const defaultInitialValues: Partial<AlertmanagertypesPlannedMaintenanceDT
 			timezone: '',
 			endTime: undefined,
 			recurrence: undefined,
-			startTime: undefined,
+			startTime: '',
 		},
 		alertIds: [],
 		createdAt: undefined,

frontend/src/container/PlannedDowntime/__test__/testUtils.ts

@@ -11,7 +11,7 @@ export const buildSchedule = (
 	schedule: Partial<AlertmanagertypesScheduleDTO>,
 ): AlertmanagertypesScheduleDTO => ({
 	timezone: schedule?.timezone ?? '',
-	startTime: schedule?.startTime,
+	startTime: schedule?.startTime ?? '',
 	endTime: schedule?.endTime,
 	recurrence: schedule?.recurrence,
 });

frontend/src/container/SideNav/SideNav.styles.scss

@@ -1135,17 +1135,9 @@
 
 .settings-dropdown,
 .help-support-dropdown {
-	.ant-dropdown-menu-item {
-		min-height: 32px;
-
-		.ant-dropdown-menu-title-content {
-			color: var(--l1-foreground) !important;
-		}
-
-		.user-settings-dropdown-logout-section {
-			color: var(--danger-background);
-			pointer-events: auto;
-		}
+	.user-settings-dropdown-logout-section {
+		color: var(--danger-background);
+		pointer-events: auto;
 	}
 }
 

frontend/src/pages/Settings/Settings.styles.scss

@@ -1,8 +1,12 @@
 .settings-page {
-	max-height: 100vh;
+	flex: 1;
+	min-height: 0;
+	display: flex;
+	flex-direction: column;
 	overflow: hidden;
 
 	.settings-page-header {
+		flex-shrink: 0;
 		border-bottom: 1px solid var(--l1-border);
 		background: var(--l1-background);
 		backdrop-filter: blur(20px);
@@ -24,13 +28,14 @@
 	}
 
 	.settings-page-content-container {
+		flex: 1;
+		min-height: 0;
 		display: flex;
 		flex-direction: row;
-		align-items: flex-start;
+		align-items: stretch;
 
 		.settings-page-sidenav {
 			width: 240px;
-			height: calc(100vh - 48px);
 			border-right: 1px solid var(--l1-border);
 			background: var(--l1-background);
 			padding-top: var(--padding-1);
@@ -74,7 +79,6 @@
 
 		.settings-page-content {
 			flex: 1;
-			height: calc(100vh - 48px);
 			background: var(--l1-background);
 			padding: 10px 8px;
 			overflow-y: auto;

pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore/maintenance.go

@@ -2,6 +2,7 @@ package sqlalertmanagerstore
 
 import (
 	"context"
+	"encoding/json"
 	"log/slog"
 	"time"
 
@@ -39,16 +40,20 @@ func (r *maintenance) ListPlannedMaintenance(ctx context.Context, orgID string)
 		return nil, err
 	}
 
-	gettablePlannedMaintenance := make([]*alertmanagertypes.PlannedMaintenance, 0)
+	plannedMaintenances := make([]*alertmanagertypes.PlannedMaintenance, 0, len(gettableMaintenancesRules))
 	for _, gettableMaintenancesRule := range gettableMaintenancesRules {
-		m := gettableMaintenancesRule.ToPlannedMaintenance()
-		gettablePlannedMaintenance = append(gettablePlannedMaintenance, m)
-		if m.HasScheduleRecurrenceBoundsMismatch() {
-			r.logger.WarnContext(ctx, "planned_downtime_recurrence_schedule_mismatch", slog.String("maintenance_id", m.ID.StringValue()))
+		pm, err := gettableMaintenancesRule.ToPlannedMaintenance()
+		if err != nil {
+			// Don't return an error because we want to process all the valid records.
+			// Log and skip instead.
+			r.logger.WarnContext(ctx, "skipping planned maintenance", slog.String("maintenance_id", gettableMaintenancesRule.ID.StringValue()), errors.Attr(err))
+			continue
 		}
+
+		plannedMaintenances = append(plannedMaintenances, pm)
 	}
 
-	return gettablePlannedMaintenance, nil
+	return plannedMaintenances, nil
 }
 
 func (r *maintenance) GetPlannedMaintenanceByID(ctx context.Context, id valuer.UUID) (*alertmanagertypes.PlannedMaintenance, error) {
@@ -64,7 +69,7 @@ func (r *maintenance) GetPlannedMaintenanceByID(ctx context.Context, id valuer.U
 		return nil, r.sqlstore.WrapNotFoundErrf(err, errors.CodeNotFound, "planned maintenance with ID: %s does not exist", id.StringValue())
 	}
 
-	return storableMaintenanceRule.ToPlannedMaintenance(), nil
+	return storableMaintenanceRule.ToPlannedMaintenance()
 }
 
 func (r *maintenance) CreatePlannedMaintenance(ctx context.Context, maintenance *alertmanagertypes.PostablePlannedMaintenance) (*alertmanagertypes.PlannedMaintenance, error) {
@@ -73,6 +78,11 @@ func (r *maintenance) CreatePlannedMaintenance(ctx context.Context, maintenance
 		return nil, err
 	}
 
+	schedule, err := json.Marshal(maintenance.Schedule)
+	if err != nil {
+		return nil, err
+	}
+
 	storablePlannedMaintenance := alertmanagertypes.StorablePlannedMaintenance{
 		Identifiable: types.Identifiable{
 			ID: valuer.GenerateUUID(),
@@ -87,7 +97,7 @@ func (r *maintenance) CreatePlannedMaintenance(ctx context.Context, maintenance
 		},
 		Name:        maintenance.Name,
 		Description: maintenance.Description,
-		Schedule:    maintenance.Schedule,
+		Schedule:    string(schedule),
 		OrgID:       claims.OrgID,
 		Scope:       maintenance.Scope,
 	}
@@ -135,18 +145,21 @@ func (r *maintenance) CreatePlannedMaintenance(ctx context.Context, maintenance
 		return nil, err
 	}
 
-	return &alertmanagertypes.PlannedMaintenance{
+	pm := &alertmanagertypes.PlannedMaintenance{
 		ID:          storablePlannedMaintenance.ID,
 		Name:        storablePlannedMaintenance.Name,
 		Description: storablePlannedMaintenance.Description,
-		Schedule:    storablePlannedMaintenance.Schedule,
 		RuleIDs:     maintenance.AlertIds,
 		Scope:       maintenance.Scope,
 		CreatedAt:   storablePlannedMaintenance.CreatedAt,
 		CreatedBy:   storablePlannedMaintenance.CreatedBy,
 		UpdatedAt:   storablePlannedMaintenance.UpdatedAt,
 		UpdatedBy:   storablePlannedMaintenance.UpdatedBy,
-	}, nil
+	}
+	if err = json.Unmarshal([]byte(storablePlannedMaintenance.Schedule), &pm.Schedule); err != nil {
+		return nil, err
+	}
+	return pm, nil
 }
 
 func (r *maintenance) DeletePlannedMaintenance(ctx context.Context, id valuer.UUID) error {
@@ -174,6 +187,11 @@ func (r *maintenance) UpdatePlannedMaintenance(ctx context.Context, maintenance
 		return err
 	}
 
+	schedule, err := json.Marshal(maintenance.Schedule)
+	if err != nil {
+		return err
+	}
+
 	storablePlannedMaintenance := alertmanagertypes.StorablePlannedMaintenance{
 		Identifiable: types.Identifiable{
 			ID: id,
@@ -188,7 +206,7 @@ func (r *maintenance) UpdatePlannedMaintenance(ctx context.Context, maintenance
 		},
 		Name:        maintenance.Name,
 		Description: maintenance.Description,
-		Schedule:    maintenance.Schedule,
+		Schedule:    string(schedule),
 		OrgID:       claims.OrgID,
 		Scope:       maintenance.Scope,
 	}

pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore/maintenance_test.go

@@ -0,0 +1,94 @@
+package sqlalertmanagerstore
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/SigNoz/signoz/pkg/factory/factorytest"
+	"github.com/SigNoz/signoz/pkg/sqlstore"
+	"github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore"
+	"github.com/SigNoz/signoz/pkg/types"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+func newTestStore(t *testing.T) sqlstore.SQLStore {
+	t.Helper()
+	store, err := sqlitesqlstore.New(t.Context(), factorytest.NewSettings(), sqlstore.Config{
+		Provider: "sqlite",
+		Connection: sqlstore.ConnectionConfig{
+			MaxOpenConns:    1,
+			MaxConnLifetime: 0,
+		},
+		Sqlite: sqlstore.SqliteConfig{
+			Path:            filepath.Join(t.TempDir(), "test.db"),
+			Mode:            "wal",
+			BusyTimeout:     5 * time.Second,
+			TransactionMode: "deferred",
+		},
+	})
+	require.NoError(t, err)
+
+	_, err = store.BunDB().NewCreateTable().
+		Model((*alertmanagertypes.StorablePlannedMaintenance)(nil)).
+		IfNotExists().
+		Exec(t.Context())
+	require.NoError(t, err)
+
+	_, err = store.BunDB().NewCreateTable().
+		Model((*alertmanagertypes.StorablePlannedMaintenanceRule)(nil)).
+		IfNotExists().
+		Exec(t.Context())
+	require.NoError(t, err)
+
+	return store
+}
+
+// TestListPlannedMaintenanceSkipsInvalid asserts that a single corrupt record
+// (here, an unloadable timezone) is skipped rather than failing the whole list.
+func TestListPlannedMaintenanceSkipsInvalid(t *testing.T) {
+	store := newTestStore(t)
+	orgID := valuer.GenerateUUID().StringValue()
+	now := time.Now().UTC()
+
+	valid := &alertmanagertypes.StorablePlannedMaintenance{
+		Identifiable:  types.Identifiable{ID: valuer.GenerateUUID()},
+		TimeAuditable: types.TimeAuditable{CreatedAt: now, UpdatedAt: now},
+		Name:          "valid",
+		Schedule:      `{"timezone":"UTC","startTime":"2024-01-01T12:00:00Z","recurrence":{"duration":"2h","repeatType":"daily"}}`,
+		OrgID:         orgID,
+	}
+	result, err := store.BunDB().NewInsert().Model(valid).Exec(t.Context())
+	require.NoError(t, err)
+	rowsAffected, err := result.RowsAffected()
+	require.NoError(t, err)
+	require.Equal(t, int64(1), rowsAffected)
+
+	// A schedule with "zero" startTime
+	invalid := &alertmanagertypes.StorablePlannedMaintenance{
+		Identifiable: types.Identifiable{ID: valuer.GenerateUUID()},
+		TimeAuditable: types.TimeAuditable{
+			CreatedAt: now,
+			UpdatedAt: now,
+		},
+		Name:     "invalid",
+		Schedule: `{"timezone":"UTC","recurrence":{"duration":"2h","repeatType":"daily"}}`,
+		OrgID:    orgID,
+	}
+	result, err = store.BunDB().NewInsert().Model(invalid).Exec(t.Context())
+	require.NoError(t, err)
+	rowsAffected, err = result.RowsAffected()
+	require.NoError(t, err)
+	require.Equal(t, int64(1), rowsAffected)
+
+	maintenanceStore := NewMaintenanceStore(store, factorytest.NewSettings())
+
+	list, err := maintenanceStore.ListPlannedMaintenance(t.Context(), orgID)
+	require.NoError(t, err)
+	require.Len(t, list, 1)
+	assert.Equal(t, valid.ID, list[0].ID)
+}

pkg/alertmanager/config_test.go

@@ -3,6 +3,8 @@ package alertmanager
 import (
 	"context"
 	"net/url"
+	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -14,7 +16,23 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const prefix = "SIGNOZ_"
+
+// clearSignozEnv unsets all existing SIGNOZ_* env vars for the duration of the test.
+func clearSignozEnv(t *testing.T) {
+	t.Helper()
+	for _, kv := range os.Environ() {
+		if strings.HasPrefix(kv, prefix) {
+			key := strings.SplitN(kv, "=", 2)[0]
+			orig, _ := os.LookupEnv(key)
+			_ = os.Unsetenv(key)
+			t.Cleanup(func() { _ = os.Setenv(key, orig) })
+		}
+	}
+}
+
 func TestNewWithEnvProvider(t *testing.T) {
+	clearSignozEnv(t)
 	t.Setenv("SIGNOZ_ALERTMANAGER_PROVIDER", "signoz")
 	t.Setenv("SIGNOZ_ALERTMANAGER_LEGACY_API__URL", "http://localhost:9093/api")
 	t.Setenv("SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_REPEAT__INTERVAL", "5m")

pkg/config/envprovider/provider_test.go

@@ -18,8 +18,8 @@ func clearSignozEnv(t *testing.T) {
 		if strings.HasPrefix(kv, prefix) {
 			key := strings.SplitN(kv, "=", 2)[0]
 			orig, _ := os.LookupEnv(key)
-			os.Unsetenv(key)
-			t.Cleanup(func() { os.Setenv(key, orig) })
+			_ = os.Unsetenv(key)
+			t.Cleanup(func() { _ = os.Setenv(key, orig) })
 		}
 	}
 }

pkg/signoz/provider.go

@@ -213,6 +213,7 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewCloudIntegrationRemoveCascadeDeleteFactory(sqlschema),
 		sqlmigration.NewAddUserDashboardPreferenceFactory(sqlstore, sqlschema),
 		sqlmigration.NewRecreateUserDashboardPreferenceFactory(sqlstore, sqlschema),
+		sqlmigration.NewMigrateRecurrenceBoundsFactory(sqlstore),
 	)
 }
 

pkg/sqlmigration/027_update_rules.go

@@ -11,7 +11,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/sqlstore"
 	"github.com/SigNoz/signoz/pkg/types"
-	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/uptrace/bun"
 	"github.com/uptrace/bun/migrate"
@@ -57,15 +56,15 @@ type newRule struct {
 
 type existingMaintenance struct {
 	bun.BaseModel `bun:"table:planned_maintenance"`
-	ID            int                 `bun:"id,pk,autoincrement"`
-	Name          string              `bun:"name,type:text,notnull"`
-	Description   string              `bun:"description,type:text"`
-	AlertIDs      *AlertIds           `bun:"alert_ids,type:text"`
-	Schedule      *alertmanagertypes.Schedule `bun:"schedule,type:text,notnull"`
-	CreatedAt     time.Time           `bun:"created_at,type:datetime,notnull"`
-	CreatedBy     string              `bun:"created_by,type:text,notnull"`
-	UpdatedAt     time.Time           `bun:"updated_at,type:datetime,notnull"`
-	UpdatedBy     string              `bun:"updated_by,type:text,notnull"`
+	ID            int       `bun:"id,pk,autoincrement"`
+	Name          string    `bun:"name,type:text,notnull"`
+	Description   string    `bun:"description,type:text"`
+	AlertIDs      *AlertIds `bun:"alert_ids,type:text"`
+	Schedule      *schedule `bun:"schedule,type:text,notnull"`
+	CreatedAt     time.Time `bun:"created_at,type:datetime,notnull"`
+	CreatedBy     string    `bun:"created_by,type:text,notnull"`
+	UpdatedAt     time.Time `bun:"updated_at,type:datetime,notnull"`
+	UpdatedBy     string    `bun:"updated_by,type:text,notnull"`
 }
 
 type newMaintenance struct {
@@ -73,10 +72,10 @@ type newMaintenance struct {
 	types.Identifiable
 	types.TimeAuditable
 	types.UserAuditable
-	Name        string              `bun:"name,type:text,notnull"`
-	Description string              `bun:"description,type:text"`
-	Schedule    *alertmanagertypes.Schedule `bun:"schedule,type:text,notnull"`
-	OrgID       string              `bun:"org_id,type:text"`
+	Name        string    `bun:"name,type:text,notnull"`
+	Description string    `bun:"description,type:text"`
+	Schedule    *schedule `bun:"schedule,type:text,notnull"`
+	OrgID       string    `bun:"org_id,type:text"`
 }
 
 type storablePlannedMaintenanceRule struct {
@@ -92,6 +91,21 @@ type ruleHistory struct {
 	RuleUUID      valuer.UUID `bun:"rule_uuid"`
 }
 
+type schedule struct {
+	Timezone   string      `json:"timezone"`
+	StartTime  time.Time   `json:"startTime"`
+	EndTime    time.Time   `json:"endTime,omitzero"`
+	Recurrence *recurrence `json:"recurrence"`
+}
+
+type recurrence struct {
+	StartTime  time.Time           `json:"startTime"`
+	EndTime    time.Time           `json:"endTime,omitzero"`
+	Duration   valuer.TextDuration `json:"duration"`
+	RepeatType string              `json:"repeatType"`
+	RepeatOn   []string            `json:"repeatOn"`
+}
+
 func NewUpdateRulesFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
 	return factory.NewProviderFactory(factory.MustNewName("update_rules"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
 		return newUpdateRules(ctx, ps, c, sqlstore)

pkg/sqlmigration/094_migrate_recurrence_bounds.go

@@ -0,0 +1,128 @@
+package sqlmigration
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/sqlstore"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/migrate"
+)
+
+type migrateRecurrenceBounds struct {
+	sqlstore sqlstore.SQLStore
+	logger   *slog.Logger
+}
+
+type plannedMaintenanceScheduleRow struct {
+	bun.BaseModel `bun:"table:planned_maintenance"`
+
+	ID       string `bun:"id"`
+	Schedule string `bun:"schedule"`
+}
+
+func NewMigrateRecurrenceBoundsFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
+	return factory.NewProviderFactory(
+		factory.MustNewName("migrate_recurrence_bounds"),
+		func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
+			return &migrateRecurrenceBounds{sqlstore: sqlstore, logger: ps.Logger}, nil
+		},
+	)
+}
+
+func (migration *migrateRecurrenceBounds) Register(migrations *migrate.Migrations) error {
+	if err := migrations.Register(migration.Up, migration.Down); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Up moves the start/end bounds of a recurring planned maintenance from the
+// nested recurrence object up to the schedule level. Until now both the
+// schedule and its recurrence carried their own startTime/endTime, with the
+// recurrence values taking precedence when a recurrence was present. The
+// recurrence fields are being dropped, so the recurrence bounds (the source of
+// truth for recurring maintenances) are promoted to the schedule before the
+// struct loses those fields.
+//
+// We deliberately operate on the raw JSON instead of the Recurrence struct:
+// that struct loses its StartTime/EndTime fields in the same change set, so it
+// can no longer read the values this migration needs to move.
+func (migration *migrateRecurrenceBounds) Up(ctx context.Context, db *bun.DB) error {
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = tx.Rollback()
+	}()
+
+	rows := make([]*plannedMaintenanceScheduleRow, 0)
+	if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
+		return err
+	}
+
+	for _, row := range rows {
+		schedule := make(map[string]json.RawMessage)
+		if err := json.Unmarshal([]byte(row.Schedule), &schedule); err != nil {
+			// A single corrupt row must not abort the whole migration (which would block startup).
+			migration.logger.WarnContext(ctx, "skipping planned maintenance with unreadable schedule", slog.String("maintenance_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		recurrenceRaw, ok := schedule["recurrence"]
+		if !ok || string(recurrenceRaw) == "null" {
+			continue
+		}
+
+		recurrence := make(map[string]json.RawMessage)
+		if err := json.Unmarshal(recurrenceRaw, &recurrence); err != nil {
+			migration.logger.WarnContext(ctx, "skipping planned maintenance with unreadable recurrence", slog.String("maintenance_id", row.ID), errors.Attr(err))
+			continue
+		}
+
+		// Promote the recurrence bounds (source of truth) to the schedule
+		// level, then drop them from the recurrence.
+		if startTime, ok := recurrence["startTime"]; ok {
+			schedule["startTime"] = startTime
+			delete(recurrence, "startTime")
+		}
+		if endTime, ok := recurrence["endTime"]; ok && string(endTime) != "null" {
+			schedule["endTime"] = endTime
+		} else {
+			// The recurrence had no end time, so the schedule must not carry
+			// a stale one duplicated by the UI.
+			delete(schedule, "endTime")
+		}
+		delete(recurrence, "endTime")
+
+		newRecurrence, err := json.Marshal(recurrence)
+		if err != nil {
+			return err
+		}
+		schedule["recurrence"] = newRecurrence
+
+		newSchedule, err := json.Marshal(schedule)
+		if err != nil {
+			return err
+		}
+
+		if _, err := tx.NewUpdate().
+			Model((*plannedMaintenanceScheduleRow)(nil)).
+			Set("schedule = ?", string(newSchedule)).
+			Where("id = ?", row.ID).
+			Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	return tx.Commit()
+}
+
+func (migration *migrateRecurrenceBounds) Down(context.Context, *bun.DB) error {
+	return nil
+}

pkg/types/alertmanagertypes/maintenance.go

@@ -3,6 +3,7 @@ package alertmanagertypes
 import (
 	"context"
 	"encoding/json"
+	"slices"
 	"time"
 
 	"github.com/expr-lang/expr"
@@ -59,11 +60,11 @@ type StorablePlannedMaintenance struct {
 	types.Identifiable
 	types.TimeAuditable
 	types.UserAuditable
-	Name        string    `bun:"name,type:text,notnull"`
-	Description string    `bun:"description,type:text"`
-	Schedule    *Schedule `bun:"schedule,type:text,notnull"`
-	OrgID       string    `bun:"org_id,type:text"`
-	Scope       string    `bun:"scope,type:text"`
+	Name        string `bun:"name,type:text,notnull"`
+	Description string `bun:"description,type:text"`
+	Schedule    string `bun:"schedule,type:text,notnull"`
+	OrgID       string `bun:"org_id,type:text"`
+	Scope       string `bun:"scope,type:text"`
 }
 
 type PlannedMaintenance struct {
@@ -99,18 +100,9 @@ func (p *PostablePlannedMaintenance) Validate() error {
 	if p.Schedule == nil {
 		return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "missing schedule in the payload")
 	}
-	if p.Schedule.Timezone == "" {
-		return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "missing timezone in the payload")
-	}
 
-	if _, err := time.LoadLocation(p.Schedule.Timezone); err != nil {
-		return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "invalid timezone in the payload")
-	}
-
-	if !p.Schedule.StartTime.IsZero() && !p.Schedule.EndTime.IsZero() {
-		if p.Schedule.StartTime.After(p.Schedule.EndTime) {
-			return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "start time cannot be after end time")
-		}
+	if !p.Schedule.EndTime.IsZero() && p.Schedule.StartTime.After(p.Schedule.EndTime) {
+		return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "start time cannot be after end time")
 	}
 
 	if p.Schedule.Recurrence != nil {
@@ -120,9 +112,6 @@ func (p *PostablePlannedMaintenance) Validate() error {
 		if p.Schedule.Recurrence.Duration.IsZero() {
 			return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "missing duration in the payload")
 		}
-		if p.Schedule.Recurrence.EndTime != nil && p.Schedule.Recurrence.EndTime.Before(p.Schedule.Recurrence.StartTime) {
-			return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "end time cannot be before start time")
-		}
 	}
 	if p.Scope != "" {
 		if _, err := expr.Compile(p.Scope, expr.AllowUndefinedVariables(), expr.AsBool()); err != nil {
@@ -148,134 +137,85 @@ type PlannedMaintenanceWithRules struct {
 	Rules                       []*StorablePlannedMaintenanceRule `bun:"rel:has-many,join:id=planned_maintenance_id"`
 }
 
-// HasScheduleRecurrenceBoundsMismatch reports whether a recurring maintenance
-// has different start/end bounds in Schedule and Schedule.Recurrence.
-//
-// This is used to detect if there are any entries with recurrence that don't
-// have the same timestamps stored at the schedule-level.
-// UI payloads duplicated those values in both places, but direct API users may
-// have stored bounds that are missing from, or different than, the schedule-level bounds.
-// We need to observe these before we can safely drop Recurrence.StartTime and
-// Recurrence.EndTime.
-func (m *PlannedMaintenance) HasScheduleRecurrenceBoundsMismatch() bool {
-	recurrence := m.Schedule.Recurrence
-	if recurrence == nil {
-		return false
-	}
-
-	return !recurrence.StartTime.Equal(m.Schedule.StartTime) ||
-		(recurrence.EndTime == nil && !m.Schedule.EndTime.IsZero()) ||
-		(recurrence.EndTime != nil && !recurrence.EndTime.Equal(m.Schedule.EndTime))
+// AppliesTo reports whether this maintenance applies to the given rule.
+// An empty RuleIDs set means the maintenance applies to all rules.
+func (m *PlannedMaintenance) AppliesTo(ruleID string) bool {
+	return len(m.RuleIDs) == 0 || slices.Contains(m.RuleIDs, ruleID)
 }
 
 func (m *PlannedMaintenance) ShouldSkip(ruleID string, now time.Time, lset model.LabelSet) (bool, error) {
-	// Check if the alert ID is in the maintenance window
-	found := false
-	if len(m.RuleIDs) > 0 {
-		for _, alertID := range m.RuleIDs {
-			if alertID == ruleID {
-				found = true
-				break
-			}
-		}
-	}
-	// If no alert ids, then skip all alerts
-	if len(m.RuleIDs) == 0 {
-		found = true
-	}
-
-	if !found {
-		return false, nil
-	}
-
-	if !m.IsActive(now) {
+	if !m.AppliesTo(ruleID) || !m.IsActive(now) {
 		return false, nil
 	}
 
 	if m.Scope != "" {
-		result, err := EvalScopeExpression(m.Scope, lset)
-		if err != nil {
+		skip, err := EvalScopeExpression(m.Scope, lset)
+		if err != nil || !skip {
 			return false, err
 		}
-		if !result {
-			return false, nil
-		}
 	}
 	return true, nil
 }
 
 // IsActive reports whether [now] falls inside the maintenance window's schedule.
 func (m *PlannedMaintenance) IsActive(now time.Time) bool {
-	// If alert is found, we check if it should be skipped based on the schedule
+	// Check if maintenance window has not started yet
+	if now.Before(m.Schedule.StartTime) {
+		return false
+	}
+
+	// Check if maintenance window has expired
+	if !m.Schedule.EndTime.IsZero() && now.After(m.Schedule.EndTime) {
+		return false
+	}
+
+	// Fixed schedule
+	if m.Schedule.Recurrence == nil {
+		return true
+	}
+
 	loc, err := time.LoadLocation(m.Schedule.Timezone)
 	if err != nil {
 		return false
 	}
 
-	startTime := m.Schedule.StartTime
-	endTime := m.Schedule.EndTime
-	recurrence := m.Schedule.Recurrence
-
-	// fixed schedule — only when no recurrence is configured.
-	// When recurrence is set, the recurring check below handles everything;
-	// falling through here would cause the window to match the absolute
-	// StartTime–EndTime range instead of the daily/weekly/monthly pattern.
-	if recurrence == nil && !startTime.IsZero() && !endTime.IsZero() {
-		if now.Equal(startTime) || now.Equal(endTime) ||
-			(now.After(startTime) && now.Before(endTime)) {
-			return true
-		}
+	switch m.Schedule.Recurrence.RepeatType {
+	case RepeatTypeDaily:
+		return m.checkDaily(now, loc)
+	case RepeatTypeWeekly:
+		return m.checkWeekly(now, loc)
+	case RepeatTypeMonthly:
+		return m.checkMonthly(now, loc)
+	default:
+		return false
 	}
-
-	// recurring schedule
-	if recurrence != nil {
-		// Make sure the recurrence has started
-		if now.Before(recurrence.StartTime) {
-			return false
-		}
-
-		// Check if recurrence has expired
-		if recurrence.EndTime != nil {
-			if !recurrence.EndTime.IsZero() && now.After(*recurrence.EndTime) {
-				return false
-			}
-		}
-
-		currentTime := now.In(loc)
-		switch recurrence.RepeatType {
-		case RepeatTypeDaily:
-			return m.checkDaily(currentTime, recurrence, loc)
-		case RepeatTypeWeekly:
-			return m.checkWeekly(currentTime, recurrence, loc)
-		case RepeatTypeMonthly:
-			return m.checkMonthly(currentTime, recurrence, loc)
-		}
-	}
-
-	return false
 }
 
 // checkDaily rebases the recurrence start to today (or yesterday if needed)
 // and returns true if currentTime is within [candidate, candidate+Duration].
-func (m *PlannedMaintenance) checkDaily(currentTime time.Time, rec *Recurrence, loc *time.Location) bool {
+func (m *PlannedMaintenance) checkDaily(currentTime time.Time, loc *time.Location) bool {
+	currentTime = currentTime.In(loc)
 	candidate := time.Date(
 		currentTime.Year(), currentTime.Month(), currentTime.Day(),
-		rec.StartTime.Hour(), rec.StartTime.Minute(), 0, 0,
+		m.Schedule.StartTime.Hour(), m.Schedule.StartTime.Minute(), 0, 0,
 		loc,
 	)
 	if candidate.After(currentTime) {
 		candidate = candidate.AddDate(0, 0, -1)
 	}
-	return currentTime.Sub(candidate) <= rec.Duration.Duration()
+	return currentTime.Sub(candidate) <= m.Schedule.Recurrence.Duration.Duration()
 }
 
 // checkWeekly finds the most recent allowed occurrence by rebasing the recurrence’s
 // time-of-day onto the allowed weekday. It does this for each allowed day and returns true
 // if the current time falls within the candidate window.
-func (m *PlannedMaintenance) checkWeekly(currentTime time.Time, rec *Recurrence, loc *time.Location) bool {
+func (m *PlannedMaintenance) checkWeekly(currentTime time.Time, loc *time.Location) bool {
+	currentTime = currentTime.In(loc)
+	rec := m.Schedule.Recurrence
+
 	// If no days specified, treat as every day (like daily).
 	if len(rec.RepeatOn) == 0 {
-		return m.checkDaily(currentTime, rec, loc)
+		return m.checkDaily(currentTime, loc)
 	}
 
 	for _, day := range rec.RepeatOn {
@@ -288,7 +228,7 @@ func (m *PlannedMaintenance) checkWeekly(currentTime time.Time, rec *Recurrence,
 		// Build a candidate occurrence by rebasing today's date to the allowed weekday.
 		candidate := time.Date(
 			currentTime.Year(), currentTime.Month(), currentTime.Day(),
-			rec.StartTime.Hour(), rec.StartTime.Minute(), 0, 0,
+			m.Schedule.StartTime.Hour(), m.Schedule.StartTime.Minute(), 0, 0,
 			loc,
 		).AddDate(0, 0, delta)
 		// If the candidate is in the future, subtract 7 days.
@@ -304,8 +244,10 @@ func (m *PlannedMaintenance) checkWeekly(currentTime time.Time, rec *Recurrence,
 
 // checkMonthly rebases the candidate occurrence using the recurrence's day-of-month.
 // If the candidate for the current month is in the future, it uses the previous month.
-func (m *PlannedMaintenance) checkMonthly(currentTime time.Time, rec *Recurrence, loc *time.Location) bool {
-	refDay := rec.StartTime.Day()
+func (m *PlannedMaintenance) checkMonthly(currentTime time.Time, loc *time.Location) bool {
+	currentTime = currentTime.In(loc)
+	startTime := m.Schedule.StartTime
+	refDay := startTime.Day()
 	year, month, _ := currentTime.Date()
 	lastDay := time.Date(year, month+1, 0, 0, 0, 0, 0, loc).Day()
 	day := refDay
@@ -313,7 +255,7 @@ func (m *PlannedMaintenance) checkMonthly(currentTime time.Time, rec *Recurrence
 		day = lastDay
 	}
 	candidate := time.Date(year, month, day,
-		rec.StartTime.Hour(), rec.StartTime.Minute(), rec.StartTime.Second(), rec.StartTime.Nanosecond(),
+		startTime.Hour(), startTime.Minute(), startTime.Second(), startTime.Nanosecond(),
 		loc,
 	)
 	if candidate.After(currentTime) {
@@ -323,33 +265,30 @@ func (m *PlannedMaintenance) checkMonthly(currentTime time.Time, rec *Recurrence
 		lastDayPrev := time.Date(y, m+1, 0, 0, 0, 0, 0, loc).Day()
 		if refDay > lastDayPrev {
 			candidate = time.Date(y, m, lastDayPrev,
-				rec.StartTime.Hour(), rec.StartTime.Minute(), rec.StartTime.Second(), rec.StartTime.Nanosecond(),
+				startTime.Hour(), startTime.Minute(), startTime.Second(), startTime.Nanosecond(),
 				loc,
 			)
 		} else {
 			candidate = time.Date(y, m, refDay,
-				rec.StartTime.Hour(), rec.StartTime.Minute(), rec.StartTime.Second(), rec.StartTime.Nanosecond(),
+				startTime.Hour(), startTime.Minute(), startTime.Second(), startTime.Nanosecond(),
 				loc,
 			)
 		}
 	}
-	return currentTime.Sub(candidate) <= rec.Duration.Duration()
+	return currentTime.Sub(candidate) <= m.Schedule.Recurrence.Duration.Duration()
 }
 
 func (m *PlannedMaintenance) IsUpcoming() bool {
-	loc, err := time.LoadLocation(m.Schedule.Timezone)
-	if err != nil {
-		return false
-	}
-	now := time.Now().In(loc)
+	now := time.Now()
 
-	if !m.Schedule.StartTime.IsZero() && !m.Schedule.EndTime.IsZero() {
-		return now.Before(m.Schedule.StartTime)
+	if m.IsRecurring() {
+		// Note: this would return true even if the maintenance is active.
+		// This isn't an issue right now because the only usage happens after the `IsActive` check.
+		return m.Schedule.EndTime.IsZero() || now.Before(m.Schedule.EndTime)
 	}
-	if m.Schedule.Recurrence != nil {
-		return now.Before(m.Schedule.Recurrence.StartTime)
-	}
-	return false
+
+	// Fixed schedule
+	return now.Before(m.Schedule.StartTime)
 }
 
 func (m *PlannedMaintenance) IsRecurring() bool {
@@ -363,19 +302,8 @@ func (m *PlannedMaintenance) Validate() error {
 	if m.Schedule == nil {
 		return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "missing schedule in the payload")
 	}
-	if m.Schedule.Timezone == "" {
-		return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "missing timezone in the payload")
-	}
-
-	_, err := time.LoadLocation(m.Schedule.Timezone)
-	if err != nil {
-		return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "invalid timezone in the payload")
-	}
-
-	if !m.Schedule.StartTime.IsZero() && !m.Schedule.EndTime.IsZero() {
-		if m.Schedule.StartTime.After(m.Schedule.EndTime) {
-			return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "start time cannot be after end time")
-		}
+	if !m.Schedule.EndTime.IsZero() && m.Schedule.StartTime.After(m.Schedule.EndTime) {
+		return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "start time cannot be after end time")
 	}
 
 	if m.Schedule.Recurrence != nil {
@@ -385,28 +313,31 @@ func (m *PlannedMaintenance) Validate() error {
 		if m.Schedule.Recurrence.Duration.IsZero() {
 			return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "missing duration in the payload")
 		}
-		if m.Schedule.Recurrence.EndTime != nil && m.Schedule.Recurrence.EndTime.Before(m.Schedule.Recurrence.StartTime) {
-			return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload, "end time cannot be before start time")
+	}
+	if m.Scope != "" {
+		if _, err := expr.Compile(m.Scope, expr.AllowUndefinedVariables(), expr.AsBool()); err != nil {
+			err := errors.Newf(
+				errors.TypeInvalidInput, ErrCodeInvalidPlannedMaintenancePayload,
+				"invalid scope: %s", err.Error(),
+			)
+			return err.WithUrl(scopeDocUrl)
 		}
 	}
 	return nil
 }
 
 func (m PlannedMaintenance) MarshalJSON() ([]byte, error) {
-	now := time.Now().In(time.FixedZone(m.Schedule.Timezone, 0))
 	var status MaintenanceStatus
-	if m.IsActive(now) {
+	if m.IsActive(time.Now()) {
 		status = MaintenanceStatusActive
 	} else if m.IsUpcoming() {
 		status = MaintenanceStatusUpcoming
 	} else {
 		status = MaintenanceStatusExpired
 	}
-	var kind MaintenanceKind
 
-	if !m.Schedule.StartTime.IsZero() && !m.Schedule.EndTime.IsZero() && m.Schedule.EndTime.After(m.Schedule.StartTime) {
-		kind = MaintenanceKindFixed
-	} else {
+	kind := MaintenanceKindFixed
+	if m.Schedule.Recurrence != nil {
 		kind = MaintenanceKindRecurring
 	}
 
@@ -439,26 +370,29 @@ func (m PlannedMaintenance) MarshalJSON() ([]byte, error) {
 	})
 }
 
-func (m *PlannedMaintenanceWithRules) ToPlannedMaintenance() *PlannedMaintenance {
-	ruleIDs := []string{}
-	if m.Rules != nil {
-		for _, storableMaintenanceRule := range m.Rules {
-			ruleIDs = append(ruleIDs, storableMaintenanceRule.RuleID.StringValue())
-		}
+func (m *PlannedMaintenanceWithRules) ToPlannedMaintenance() (*PlannedMaintenance, error) {
+	schedule := &Schedule{}
+	if err := json.Unmarshal([]byte(m.Schedule), &schedule); err != nil {
+		return nil, err
+	}
+
+	ruleIDs := make([]string, 0, len(m.Rules))
+	for _, storableMaintenanceRule := range m.Rules {
+		ruleIDs = append(ruleIDs, storableMaintenanceRule.RuleID.StringValue())
 	}
 
 	return &PlannedMaintenance{
 		ID:          m.ID,
 		Name:        m.Name,
 		Description: m.Description,
-		Schedule:    m.Schedule,
+		Schedule:    schedule,
 		RuleIDs:     ruleIDs,
 		Scope:       m.Scope,
 		CreatedAt:   m.CreatedAt,
 		UpdatedAt:   m.UpdatedAt,
 		CreatedBy:   m.CreatedBy,
 		UpdatedBy:   m.UpdatedBy,
-	}
+	}, nil
 }
 
 type ListPlannedMaintenanceParams struct {

pkg/types/alertmanagertypes/maintenance_test.go

@@ -8,11 +8,6 @@ import (
 	"github.com/prometheus/common/model"
 )
 
-// Helper function to create a time pointer.
-func timePtr(t time.Time) *time.Time {
-	return &t
-}
-
 func TestShouldSkipMaintenance(t *testing.T) {
 	cases := []struct {
 		name        string
@@ -24,9 +19,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "only-on-saturday",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "Europe/London",
+					Timezone:  "Europe/London",
+					StartTime: time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("24h"),
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday, RepeatOnTuesday, RepeatOnWednesday, RepeatOnThursday, RepeatOnFriday, RepeatOnSunday},
@@ -41,10 +36,10 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "weekly-across-midnight-previous-day",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 22, 0, 0, 0, time.UTC), // Monday 22:00
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 22, 0, 0, 0, time.UTC), // Monday 22:00
-						Duration:   valuer.MustParseTextDuration("4h"),           // Until Tuesday 02:00
+						Duration:   valuer.MustParseTextDuration("4h"), // Until Tuesday 02:00
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday}, // Only Monday
 					},
@@ -58,10 +53,10 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "weekly-across-midnight-previous-day",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 22, 0, 0, 0, time.UTC), // Monday 22:00
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 22, 0, 0, 0, time.UTC), // Monday 22:00
-						Duration:   valuer.MustParseTextDuration("4h"),           // Until Tuesday 02:00
+						Duration:   valuer.MustParseTextDuration("4h"), // Until Tuesday 02:00
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday}, // Only Monday
 					},
@@ -75,10 +70,10 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "weekly-across-midnight-previous-day",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 22, 0, 0, 0, time.UTC), // Monday 22:00
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 22, 0, 0, 0, time.UTC), // Monday 22:00
-						Duration:   valuer.MustParseTextDuration("52h"),          // Until Thursday 02:00
+						Duration:   valuer.MustParseTextDuration("52h"), // Until Thursday 02:00
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday}, // Only Monday
 					},
@@ -92,10 +87,10 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "weekly-across-midnight-previous-day-not-in-repeaton",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 2, 22, 0, 0, 0, time.UTC), // Tuesday 22:00
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 2, 22, 0, 0, 0, time.UTC), // Tuesday 22:00
-						Duration:   valuer.MustParseTextDuration("4h"),           // Until Wednesday 02:00
+						Duration:   valuer.MustParseTextDuration("4h"), // Until Wednesday 02:00
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnTuesday}, // Only Tuesday
 					},
@@ -109,10 +104,10 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "daily-maintenance-across-midnight",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 23, 0, 0, 0, time.UTC), // 23:00
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 23, 0, 0, 0, time.UTC), // 23:00
-						Duration:   valuer.MustParseTextDuration("2h"),           // Until 01:00 next day
+						Duration:   valuer.MustParseTextDuration("2h"), // Until 01:00 next day
 						RepeatType: RepeatTypeDaily,
 					},
 				},
@@ -125,9 +120,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "at-start-time-boundary",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeDaily,
 					},
@@ -141,9 +136,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "at-end-time-boundary",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeDaily,
 					},
@@ -157,9 +152,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "monthly-multi-day-duration",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 28, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 28, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("72h"), // 3 days
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -173,9 +168,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "weekly-multi-day-duration",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 28, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 28, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("72h"), // 3 days
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnSunday},
@@ -190,9 +185,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "monthly-crosses-to-next-month",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 30, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 30, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("48h"), // 2 days, crosses to Feb 1
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -206,9 +201,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "timezone-offset-test",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "America/New_York", // UTC-5 or UTC-4 depending on DST
+					Timezone:  "America/New_York", // UTC-5 or UTC-4 depending on DST
+					StartTime: time.Date(2024, 1, 1, 22, 0, 0, 0, time.FixedZone("America/New_York", -5*3600)),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 22, 0, 0, 0, time.FixedZone("America/New_York", -5*3600)),
 						Duration:   valuer.MustParseTextDuration("4h"),
 						RepeatType: RepeatTypeDaily,
 					},
@@ -222,9 +217,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "daily-maintenance-time-outside-window",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeDaily,
 					},
@@ -238,10 +233,10 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring-maintenance-with-past-end-date",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
+					EndTime:   time.Date(2024, 1, 10, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
-						EndTime:    timePtr(time.Date(2024, 1, 10, 12, 0, 0, 0, time.UTC)),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeDaily,
 					},
@@ -255,10 +250,10 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "monthly-maintenance-spans-month-end",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 3, 31, 22, 0, 0, 0, time.UTC), // March 31, 22:00
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 3, 31, 22, 0, 0, 0, time.UTC), // March 31, 22:00
-						Duration:   valuer.MustParseTextDuration("6h"),            // Until April 1, 04:00
+						Duration:   valuer.MustParseTextDuration("6h"), // Until April 1, 04:00
 						RepeatType: RepeatTypeMonthly,
 					},
 				},
@@ -271,9 +266,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "weekly-empty-repeaton",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{}, // Empty - should apply to all days
@@ -288,9 +283,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "monthly-maintenance-february-fewer-days",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 31, 12, 0, 0, 0, time.UTC), // January 31st
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 31, 12, 0, 0, 0, time.UTC), // January 31st
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -303,9 +298,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "daily-maintenance-crosses-midnight",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 23, 30, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 23, 30, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("1h"), // Crosses to 00:30 next day
 						RepeatType: RepeatTypeDaily,
 					},
@@ -318,9 +313,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "monthly-maintenance-crosses-month-end",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 31, 12, 0, 0, 0, time.UTC), // January 31st
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 31, 12, 0, 0, 0, time.UTC), // January 31st
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -333,9 +328,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "monthly-maintenance-crosses-month-end-and-duration-is-2-days",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 30, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 30, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("48h"), // 2 days duration
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -348,10 +343,10 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "weekly-maintenance-crosses-midnight",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 23, 0, 0, 0, time.UTC), // Monday 23:00
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 23, 0, 0, 0, time.UTC), // Monday 23:00
-						Duration:   valuer.MustParseTextDuration("2h"),           // Until Tuesday 01:00
+						Duration:   valuer.MustParseTextDuration("2h"), // Until Tuesday 01:00
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday}, // Only Monday
 					},
@@ -364,9 +359,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "monthly-maintenance-crosses-month-end-and-duration-is-2-days",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 31, 12, 0, 0, 0, time.UTC), // January 31st
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 31, 12, 0, 0, 0, time.UTC), // January 31st
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -379,9 +374,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "daily-maintenance-crosses-midnight",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 22, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 22, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("4h"), // Until 02:00 next day
 						RepeatType: RepeatTypeDaily,
 					},
@@ -394,9 +389,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "monthly-maintenance-crosses-month-end-and-duration-is-2-hours",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 31, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 31, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -445,9 +440,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring maintenance, repeat sunday, saturday, weekly for 24 hours, in Us/Eastern timezone",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "US/Eastern",
+					Timezone:  "US/Eastern",
+					StartTime: time.Date(2025, 3, 29, 20, 0, 0, 0, time.FixedZone("US/Eastern", -4*3600)),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2025, 3, 29, 20, 0, 0, 0, time.FixedZone("US/Eastern", -4*3600)),
 						Duration:   valuer.MustParseTextDuration("24h"),
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnSunday, RepeatOnSaturday},
@@ -458,57 +453,57 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			skip: true,
 		},
 		{
-			name: "recurring maintenance, repeat daily from 12:00 to 14:00",
+			name: "recurring maintenance, repeat daily from 12:00 to 14:00, ts < start",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeDaily,
 					},
 				},
 			},
-			ts:   time.Date(2024, 1, 1, 12, 10, 0, 0, time.UTC),
+			ts:   time.Date(2024, 1, 10, 11, 0, 0, 0, time.UTC),
+			skip: false,
+		},
+		{
+			name: "recurring maintenance, repeat daily from 12:00 to 14:00, start <= ts <= end",
+			maintenance: &PlannedMaintenance{
+				Schedule: &Schedule{
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
+					Recurrence: &Recurrence{
+						Duration:   valuer.MustParseTextDuration("2h"),
+						RepeatType: RepeatTypeDaily,
+					},
+				},
+			},
+			ts:   time.Date(2024, 1, 10, 13, 0, 0, 0, time.UTC),
 			skip: true,
 		},
 		{
-			name: "recurring maintenance, repeat daily from 12:00 to 14:00",
+			name: "recurring maintenance, repeat daily from 12:00 to 14:00, start > end",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeDaily,
 					},
 				},
 			},
-			ts:   time.Date(2024, 1, 1, 14, 0, 0, 0, time.UTC),
-			skip: true,
-		},
-		{
-			name: "recurring maintenance, repeat daily from 12:00 to 14:00",
-			maintenance: &PlannedMaintenance{
-				Schedule: &Schedule{
-					Timezone: "UTC",
-					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
-						Duration:   valuer.MustParseTextDuration("2h"),
-						RepeatType: RepeatTypeDaily,
-					},
-				},
-			},
-			ts:   time.Date(2024, 4, 1, 12, 10, 0, 0, time.UTC),
-			skip: true,
+			ts:   time.Date(2024, 1, 10, 15, 0, 0, 0, time.UTC),
+			skip: false,
 		},
 		{
 			name: "recurring maintenance, repeat weekly on monday from 12:00 to 14:00",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday},
@@ -522,9 +517,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring maintenance, repeat weekly on monday from 12:00 to 14:00",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday},
@@ -538,9 +533,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring maintenance, repeat weekly on monday from 12:00 to 14:00",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday},
@@ -554,9 +549,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring maintenance, repeat weekly on monday from 12:00 to 14:00",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday},
@@ -570,9 +565,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring maintenance, repeat weekly on monday from 12:00 to 14:00",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 1, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeWeekly,
 						RepeatOn:   []RepeatOn{RepeatOnMonday},
@@ -586,9 +581,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring maintenance, repeat monthly on 4th from 12:00 to 14:00",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 4, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 4, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -601,9 +596,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring maintenance, repeat monthly on 4th from 12:00 to 14:00",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 4, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 4, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -616,9 +611,9 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			name: "recurring maintenance, repeat monthly on 4th from 12:00 to 14:00",
 			maintenance: &PlannedMaintenance{
 				Schedule: &Schedule{
-					Timezone: "UTC",
+					Timezone:  "UTC",
+					StartTime: time.Date(2024, 4, 4, 12, 0, 0, 0, time.UTC),
 					Recurrence: &Recurrence{
-						StartTime:  time.Date(2024, 4, 4, 12, 0, 0, 0, time.UTC),
 						Duration:   valuer.MustParseTextDuration("2h"),
 						RepeatType: RepeatTypeMonthly,
 					},
@@ -627,45 +622,6 @@ func TestShouldSkipMaintenance(t *testing.T) {
 			ts:   time.Date(2024, 5, 4, 12, 10, 0, 0, time.UTC),
 			skip: true,
 		},
-		// The recurrence should govern, when set. Not the fixed range.
-		{
-			name: "recurring-daily-with-fixed-times-outside-daily-window",
-			maintenance: &PlannedMaintenance{
-				Schedule: &Schedule{
-					Timezone: "UTC",
-					// These fixed fields should be ignored when Recurrence is set.
-					StartTime: time.Date(2026, 4, 1, 14, 0, 0, 0, time.UTC),
-					EndTime:   time.Date(2026, 4, 30, 18, 0, 0, 0, time.UTC),
-					Recurrence: &Recurrence{
-						StartTime:  time.Date(2026, 4, 1, 14, 0, 0, 0, time.UTC), // daily at 14:00
-						Duration:   valuer.MustParseTextDuration("2h"),           // until 16:00
-						RepeatType: RepeatTypeDaily,
-					},
-				},
-			},
-			// 2026-04-15 11:00 is inside the fixed range but outside the daily 14:00-16:00 window.
-			ts:   time.Date(2026, 4, 15, 11, 0, 0, 0, time.UTC),
-			skip: false,
-		},
-		{
-			name: "recurring-daily-with-fixed-times-inside-daily-window",
-			maintenance: &PlannedMaintenance{
-				Schedule: &Schedule{
-					Timezone:  "UTC",
-					StartTime: time.Date(2026, 4, 1, 14, 0, 0, 0, time.UTC),
-					EndTime:   time.Date(2026, 4, 30, 18, 0, 0, 0, time.UTC),
-					Recurrence: &Recurrence{
-						StartTime:  time.Date(2026, 4, 1, 14, 0, 0, 0, time.UTC),
-						EndTime:    timePtr(time.Date(2026, 4, 30, 18, 0, 0, 0, time.UTC)),
-						Duration:   valuer.MustParseTextDuration("2h"),
-						RepeatType: RepeatTypeDaily,
-					},
-				},
-			},
-			// 15:00 is inside the daily 14:00-16:00 window. Should skip.
-			ts:   time.Date(2026, 4, 15, 15, 0, 0, 0, time.UTC),
-			skip: true,
-		},
 	}
 
 	for idx, c := range cases {
@@ -679,13 +635,211 @@ func TestShouldSkipMaintenance(t *testing.T) {
 	}
 }
 
+func TestIsActiveFixedSchedule(t *testing.T) {
+	start := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
+	end := time.Date(2024, 1, 10, 12, 0, 0, 0, time.UTC)
+
+	cases := []struct {
+		name      string
+		startTime time.Time
+		endTime   time.Time
+		now       time.Time
+		active    bool
+	}{
+		{
+			name:      "no end, t < start",
+			startTime: start,
+			now:       start.Add(-time.Hour),
+			active:    false,
+		},
+		{
+			name:      "no end, start == t",
+			startTime: start,
+			now:       start,
+			active:    true,
+		},
+		{
+			// A fixed schedule with no end time stays active indefinitely.
+			name:      "no end, start << t",
+			startTime: start,
+			now:       start.AddDate(10, 0, 0),
+			active:    true,
+		},
+		{
+			name:      "with end, start < t < end",
+			startTime: start,
+			endTime:   end,
+			now:       start.Add(24 * time.Hour),
+			active:    true,
+		},
+		{
+			name:      "with end, t == end",
+			startTime: start,
+			endTime:   end,
+			now:       end,
+			active:    true,
+		},
+		{
+			name:      "with end, end < t",
+			startTime: start,
+			endTime:   end,
+			now:       end.Add(time.Hour),
+			active:    false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			m := &PlannedMaintenance{
+				Schedule: &Schedule{
+					Timezone:  "UTC",
+					StartTime: c.startTime,
+					EndTime:   c.endTime,
+				},
+			}
+			if got := m.IsActive(c.now); got != c.active {
+				t.Errorf("IsActive() = %v, want %v", got, c.active)
+			}
+		})
+	}
+}
+
+func TestIsActiveRecurringSchedule(t *testing.T) {
+	// Daily window 12:00-14:00, starting 2024-01-01 (a Monday).
+	start := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
+
+	daily := &Recurrence{
+		Duration:   valuer.MustParseTextDuration("2h"),
+		RepeatType: RepeatTypeDaily,
+	}
+
+	cases := []struct {
+		name       string
+		startTime  time.Time
+		endTime    time.Time
+		recurrence *Recurrence
+		now        time.Time
+		active     bool
+	}{
+		{
+			// The recurrence has not begun yet, even though the time-of-day matches.
+			name:       "daily: t < recurrence start",
+			startTime:  start,
+			recurrence: daily,
+			now:        time.Date(2023, 12, 31, 13, 0, 0, 0, time.UTC),
+			active:     false,
+		},
+		{
+			name:       "daily: no end, within window",
+			startTime:  start,
+			recurrence: daily,
+			now:        time.Date(2024, 6, 15, 13, 0, 0, 0, time.UTC),
+			active:     true,
+		},
+		{
+			name:       "daily: no end, outside window",
+			startTime:  start,
+			recurrence: daily,
+			now:        time.Date(2024, 6, 15, 15, 0, 0, 0, time.UTC),
+			active:     false,
+		},
+		{
+			name:       "daily: at window start boundary",
+			startTime:  start,
+			recurrence: daily,
+			now:        time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC),
+			active:     true,
+		},
+		{
+			name:       "daily: at window end boundary",
+			startTime:  start,
+			recurrence: daily,
+			now:        time.Date(2024, 6, 15, 14, 0, 0, 0, time.UTC),
+			active:     true,
+		},
+		{
+			// Past the recurrence end, the time-of-day match no longer applies.
+			name:       "daily: t > recurrence end",
+			startTime:  start,
+			endTime:    time.Date(2024, 1, 10, 12, 0, 0, 0, time.UTC),
+			recurrence: daily,
+			now:        time.Date(2024, 1, 15, 13, 0, 0, 0, time.UTC),
+			active:     false,
+		},
+		{
+			name:       "daily: before recurrence end, within window",
+			startTime:  start,
+			endTime:    time.Date(2024, 1, 10, 23, 0, 0, 0, time.UTC),
+			recurrence: daily,
+			now:        time.Date(2024, 1, 10, 13, 0, 0, 0, time.UTC),
+			active:     true,
+		},
+		{
+			name:      "weekly: on allowed day, within window",
+			startTime: start, // Monday
+			recurrence: &Recurrence{
+				Duration:   valuer.MustParseTextDuration("2h"),
+				RepeatType: RepeatTypeWeekly,
+				RepeatOn:   []RepeatOn{RepeatOnMonday},
+			},
+			now:    time.Date(2024, 4, 15, 13, 0, 0, 0, time.UTC), // a Monday
+			active: true,
+		},
+		{
+			name:      "weekly: on non-allowed day",
+			startTime: start,
+			recurrence: &Recurrence{
+				Duration:   valuer.MustParseTextDuration("2h"),
+				RepeatType: RepeatTypeWeekly,
+				RepeatOn:   []RepeatOn{RepeatOnMonday},
+			},
+			now:    time.Date(2024, 4, 16, 13, 0, 0, 0, time.UTC), // a Tuesday
+			active: false,
+		},
+		{
+			name:      "monthly: on day-of-month, within window",
+			startTime: time.Date(2024, 1, 4, 12, 0, 0, 0, time.UTC),
+			recurrence: &Recurrence{
+				Duration:   valuer.MustParseTextDuration("2h"),
+				RepeatType: RepeatTypeMonthly,
+			},
+			now:    time.Date(2024, 5, 4, 13, 0, 0, 0, time.UTC),
+			active: true,
+		},
+		{
+			name:      "monthly: on different day-of-month",
+			startTime: time.Date(2024, 1, 4, 12, 0, 0, 0, time.UTC),
+			recurrence: &Recurrence{
+				Duration:   valuer.MustParseTextDuration("2h"),
+				RepeatType: RepeatTypeMonthly,
+			},
+			now:    time.Date(2024, 5, 5, 13, 0, 0, 0, time.UTC),
+			active: false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			m := &PlannedMaintenance{
+				Schedule: &Schedule{
+					Timezone:   "UTC",
+					StartTime:  c.startTime,
+					EndTime:    c.endTime,
+					Recurrence: c.recurrence,
+				},
+			}
+			if got := m.IsActive(c.now); got != c.active {
+				t.Errorf("IsActive() = %v, want %v", got, c.active)
+			}
+		})
+	}
+}
+
 func TestShouldSkip_Scope(t *testing.T) {
-	activeSchedule := func() *Schedule {
-		return &Schedule{
-			Timezone:  "UTC",
-			StartTime: time.Now().UTC().Add(-time.Hour),
-			EndTime:   time.Now().UTC().Add(time.Hour),
-		}
+	activeSchedule := &Schedule{
+		Timezone:  "UTC",
+		StartTime: time.Now().UTC().Add(-time.Hour),
+		EndTime:   time.Now().UTC().Add(time.Hour),
 	}
 	now := time.Now().UTC()
 
@@ -699,7 +853,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 	}{
 		{
 			name:        "empty scope - no label filtering applied",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule()},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "production"},
@@ -707,7 +861,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "scope matches labels",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env = "production"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env = "production"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "production"},
@@ -715,7 +869,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "scope does not match labels",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env = "production"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env = "production"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "staging"},
@@ -723,7 +877,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "AND expression - both conditions match",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env = "production" AND service = "api"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env = "production" AND service = "api"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "production", "service": "api"},
@@ -731,7 +885,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "AND expression - one condition does not match",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env = "production" AND service = "api"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env = "production" AND service = "api"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "production", "service": "worker"},
@@ -739,7 +893,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "OR expression - first alternative matches",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env = "production" OR env = "staging"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env = "production" OR env = "staging"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "production"},
@@ -747,7 +901,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "OR expression - second alternative matches",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env = "production" OR env = "staging"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env = "production" OR env = "staging"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "staging"},
@@ -755,7 +909,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "OR expression - neither alternative matches",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env = "production" OR env = "staging"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env = "production" OR env = "staging"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "development"},
@@ -763,7 +917,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "scope references label absent from lset",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env = "production"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env = "production"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"service": "api"},
@@ -771,7 +925,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "in expression - value is in list",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env in ["production", "staging"]`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env in ["production", "staging"]`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "staging"},
@@ -779,7 +933,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "in expression - value not in list",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), Scope: `env in ["production", "staging"]`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, Scope: `env in ["production", "staging"]`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "development"},
@@ -787,7 +941,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "ruleID in list and scope matches - should skip",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), RuleIDs: []string{"rule-1", "rule-2"}, Scope: `env = "production"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, RuleIDs: []string{"rule-1", "rule-2"}, Scope: `env = "production"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "production"},
@@ -795,7 +949,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "ruleID not in list and scope matches - ruleID gate prevents skip",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), RuleIDs: []string{"rule-2"}, Scope: `env = "production"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, RuleIDs: []string{"rule-2"}, Scope: `env = "production"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "production"},
@@ -803,7 +957,7 @@ func TestShouldSkip_Scope(t *testing.T) {
 		},
 		{
 			name:        "ruleID in list but scope does not match - should not skip",
-			maintenance: &PlannedMaintenance{Schedule: activeSchedule(), RuleIDs: []string{"rule-1"}, Scope: `env = "production"`},
+			maintenance: &PlannedMaintenance{Schedule: activeSchedule, RuleIDs: []string{"rule-1"}, Scope: `env = "production"`},
 			ruleID:      "rule-1",
 			ts:          now,
 			lset:        model.LabelSet{"env": "staging"},

pkg/types/alertmanagertypes/recurrence.go

@@ -66,9 +66,9 @@ var RepeatOnAllMap = map[RepeatOn]time.Weekday{
 	RepeatOnSaturday:  time.Saturday,
 }
 
+// Recurrence describes the repeat pattern of a planned maintenance.
+// The window bounds (start/end) live on the enclosing Schedule.
 type Recurrence struct {
-	StartTime  time.Time           `json:"startTime" required:"true"`
-	EndTime    *time.Time          `json:"endTime,omitempty"`
 	Duration   valuer.TextDuration `json:"duration" required:"true"`
 	RepeatType RepeatType          `json:"repeatType" required:"true"`
 	RepeatOn   []RepeatOn          `json:"repeatOn"`

pkg/types/alertmanagertypes/schedule.go

@@ -11,7 +11,7 @@ import (
 
 type Schedule struct {
 	Timezone   string      `json:"timezone" required:"true"`
-	StartTime  time.Time   `json:"startTime,omitempty"`
+	StartTime  time.Time   `json:"startTime" required:"true"`
 	EndTime    time.Time   `json:"endTime,omitzero"`
 	Recurrence *Recurrence `json:"recurrence"`
 }
@@ -39,29 +39,12 @@ func (s Schedule) MarshalJSON() ([]byte, error) {
 		return nil, err
 	}
 
-	var startTime, endTime time.Time
-	if !s.StartTime.IsZero() {
-		startTime = time.Date(s.StartTime.Year(), s.StartTime.Month(), s.StartTime.Day(), s.StartTime.Hour(), s.StartTime.Minute(), s.StartTime.Second(), s.StartTime.Nanosecond(), loc)
-	}
+	// Marshal times in the selected timezone.
+	// This ensures that recurring events are handled correctly when DST is involved.
+	startTime := s.StartTime.In(loc)
+	var endTime time.Time
 	if !s.EndTime.IsZero() {
-		endTime = time.Date(s.EndTime.Year(), s.EndTime.Month(), s.EndTime.Day(), s.EndTime.Hour(), s.EndTime.Minute(), s.EndTime.Second(), s.EndTime.Nanosecond(), loc)
-	}
-
-	var recurrence *Recurrence
-	if s.Recurrence != nil {
-		recStartTime := time.Date(s.Recurrence.StartTime.Year(), s.Recurrence.StartTime.Month(), s.Recurrence.StartTime.Day(), s.Recurrence.StartTime.Hour(), s.Recurrence.StartTime.Minute(), s.Recurrence.StartTime.Second(), s.Recurrence.StartTime.Nanosecond(), loc)
-		var recEndTime *time.Time
-		if s.Recurrence.EndTime != nil {
-			end := time.Date(s.Recurrence.EndTime.Year(), s.Recurrence.EndTime.Month(), s.Recurrence.EndTime.Day(), s.Recurrence.EndTime.Hour(), s.Recurrence.EndTime.Minute(), s.Recurrence.EndTime.Second(), s.Recurrence.EndTime.Nanosecond(), loc)
-			recEndTime = &end
-		}
-		recurrence = &Recurrence{
-			StartTime:  recStartTime,
-			EndTime:    recEndTime,
-			Duration:   s.Recurrence.Duration,
-			RepeatType: s.Recurrence.RepeatType,
-			RepeatOn:   s.Recurrence.RepeatOn,
-		}
+		endTime = s.EndTime.In(loc)
 	}
 
 	return json.Marshal(&struct {
@@ -73,7 +56,7 @@ func (s Schedule) MarshalJSON() ([]byte, error) {
 		Timezone:   s.Timezone,
 		StartTime:  startTime,
 		EndTime:    endTime,
-		Recurrence: recurrence,
+		Recurrence: s.Recurrence,
 	})
 }
 
@@ -88,55 +71,35 @@ func (s *Schedule) UnmarshalJSON(data []byte) error {
 		return err
 	}
 
-	loc, err := time.LoadLocation(aux.Timezone)
-	if err != nil {
-		return err
+	if aux.Timezone == "" {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "missing timezone")
 	}
 
-	var startTime time.Time
-	if aux.StartTime != "" {
-		startTime, err = time.Parse(time.RFC3339, aux.StartTime)
-		if err != nil {
-			return err
-		}
-		s.StartTime = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), startTime.Second(), startTime.Nanosecond(), loc)
+	loc, err := time.LoadLocation(aux.Timezone)
+	if err != nil {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, `invalid timezone "%s"`, aux.Timezone)
 	}
 
+	startTime, err := time.Parse(time.RFC3339, aux.StartTime)
+	if err != nil {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, `invalid start time "%s"`, aux.StartTime)
+	}
+	startTime = startTime.In(loc)
+
 	var endTime time.Time
 	if aux.EndTime != "" {
 		endTime, err = time.Parse(time.RFC3339, aux.EndTime)
 		if err != nil {
-			return err
+			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, `invalid end time "%s"`, aux.EndTime)
+		}
+		if !endTime.IsZero() {
+			endTime = endTime.In(loc)
 		}
-		// TODO(jatinderjit): if endTime.IsZero() then we should not set the endTime
-		s.EndTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), endTime.Hour(), endTime.Minute(), endTime.Second(), endTime.Nanosecond(), loc)
 	}
 
 	s.Timezone = aux.Timezone
-
-	if aux.Recurrence != nil {
-		recStartTime, err := time.Parse(time.RFC3339, aux.Recurrence.StartTime.Format(time.RFC3339))
-		if err != nil {
-			return err
-		}
-
-		var recEndTime *time.Time
-		if aux.Recurrence.EndTime != nil {
-			end, err := time.Parse(time.RFC3339, aux.Recurrence.EndTime.Format(time.RFC3339))
-			if err != nil {
-				return err
-			}
-			endConverted := time.Date(end.Year(), end.Month(), end.Day(), end.Hour(), end.Minute(), end.Second(), end.Nanosecond(), loc)
-			recEndTime = &endConverted
-		}
-
-		s.Recurrence = &Recurrence{
-			StartTime:  time.Date(recStartTime.Year(), recStartTime.Month(), recStartTime.Day(), recStartTime.Hour(), recStartTime.Minute(), recStartTime.Second(), recStartTime.Nanosecond(), loc),
-			EndTime:    recEndTime,
-			Duration:   aux.Recurrence.Duration,
-			RepeatType: aux.Recurrence.RepeatType,
-			RepeatOn:   aux.Recurrence.RepeatOn,
-		}
-	}
+	s.StartTime = startTime
+	s.EndTime = endTime
+	s.Recurrence = aux.Recurrence
 	return nil
 }

tests/conftest.py

@@ -28,6 +28,7 @@ pytest_plugins = [
     "fixtures.serviceaccount",
     "fixtures.role",
     "fixtures.seed_golden_dataset",
+    "fixtures.maildev",
 ]
 
 

tests/fixtures/alerts.py

@@ -4,6 +4,7 @@ import time
 from collections.abc import Callable
 from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
+from urllib.parse import urlparse
 
 import pytest
 import requests
@@ -13,6 +14,7 @@ from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.fs import get_testdata_file_path
 from fixtures.logger import setup_logger
 from fixtures.logs import Logs
+from fixtures.maildev import verify_email_received
 from fixtures.metrics import Metrics
 from fixtures.traces import Traces
 
@@ -218,3 +220,131 @@ def update_rule_channel_name(rule_data: dict, channel_name: str):
         # loop over all the sepcs and update the channels
         for spec in thresholds["spec"]:
             spec["channels"] = [channel_name]
+
+
+def _is_json_subset(subset, superset) -> bool:
+    """Check if subset is contained within superset recursively.
+    - For dicts: all keys in subset must exist in superset with matching values
+    - For lists: all items in subset must be present in superset
+    - For scalars: exact equality
+    """
+    if isinstance(subset, dict):
+        if not isinstance(superset, dict):
+            return False
+        return all(key in superset and _is_json_subset(value, superset[key]) for key, value in subset.items())
+    if isinstance(subset, list):
+        if not isinstance(superset, list):
+            return False
+        return all(any(_is_json_subset(sub_item, sup_item) for sup_item in superset) for sub_item in subset)
+    return subset == superset
+
+
+def verify_webhook_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    validation_data: dict,
+) -> bool:
+    """Check if wiremock received a request at the given path
+    whose JSON body is a superset of the expected json_body."""
+    path = validation_data["path"]
+    json_body = validation_data["json_body"]
+
+    url = notification_channel.host_configs["8080"].get("__admin/requests/find")
+    try:
+        res = requests.post(url, json={"method": "POST", "url": path}, timeout=10)
+    except requests.exceptions.RequestException:
+        return False
+    if res.status_code != HTTPStatus.OK:
+        return False
+
+    for req in res.json()["requests"]:
+        body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+        # logger.info("Webhook request body: %s", json.dumps(body, indent=2))
+        if _is_json_subset(json_body, body):
+            return True
+    return False
+
+
+def _check_notification_validation(
+    validation: types.NotificationValidation,
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+) -> bool:
+    """Dispatch a single validation check to the appropriate verifier."""
+    if validation.destination_type == "webhook":
+        return verify_webhook_notification_expectation(notification_channel, validation.validation_data)
+    if validation.destination_type == "email":
+        return verify_email_received(maildev, validation.validation_data)
+    raise ValueError(f"Invalid destination type: {validation.destination_type}")
+
+
+def verify_notification_expectation(
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker,
+    expected_notification: types.AMNotificationExpectation,
+) -> bool:
+    """Poll for expected notifications across webhook and email channels."""
+    time_to_wait = datetime.now() + timedelta(seconds=expected_notification.wait_time_seconds)
+
+    while datetime.now() < time_to_wait:
+        all_found = all(_check_notification_validation(v, notification_channel, maildev) for v in expected_notification.notification_validations)
+
+        if expected_notification.should_notify and all_found:
+            logger.info("All expected notifications found")
+            return True
+
+        time.sleep(1)
+
+    # Timeout reached
+    if not expected_notification.should_notify:
+        # Verify no notifications were received
+        for validation in expected_notification.notification_validations:
+            found = _check_notification_validation(validation, notification_channel, maildev)
+            assert not found, f"Expected no notification but found one for {validation.destination_type} with data {validation.validation_data}"
+        logger.info("No notifications found, as expected")
+        return True
+
+    # Expected notifications but didn't get them all — report missing
+    missing = [v for v in expected_notification.notification_validations if not _check_notification_validation(v, notification_channel, maildev)]
+    assert len(missing) == 0, f"Expected all notifications to be found but missing: {missing}"
+    return True
+
+
+def update_raw_channel_config(
+    channel_config: dict,
+    channel_name: str,
+    notification_channel: types.TestContainerDocker,
+    maildev: types.TestContainerDocker | None = None,
+) -> dict:
+    """
+    Updates the channel config to point to the given wiremock
+    notification_channel container to receive notifications.
+    """
+    config = channel_config.copy()
+
+    config["name"] = channel_name
+
+    url_field_map = {
+        "slack_configs": "api_url",
+        "msteamsv2_configs": "webhook_url",
+        "webhook_configs": "url",
+        "pagerduty_configs": "url",
+        "opsgenie_configs": "api_url",
+    }
+
+    for config_key, url_field in url_field_map.items():
+        if config_key in config:
+            for entry in config[config_key]:
+                if url_field in entry:
+                    original_url = entry[url_field]
+                    path = urlparse(original_url).path
+                    entry[url_field] = notification_channel.container_configs["8080"].get(path)
+
+    # Inject live maildev SMTP address into email configs so the test works
+    if maildev is not None and "email_configs" in config:
+        smtp_host = maildev.container_configs["1025"].address
+        smtp_port = str(maildev.container_configs["1025"].port)
+        for entry in config["email_configs"]:
+            entry["smarthost"] = f"{smtp_host}:{smtp_port}"
+            entry["require_tls"] = False
+
+    return config

tests/fixtures/http.py

@@ -124,14 +124,19 @@ def gateway(
 
 
 @pytest.fixture(name="make_http_mocks", scope="function")
-def make_http_mocks() -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
+def make_http_mocks(
+    request: pytest.FixtureRequest,
+) -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
     def _make_http_mocks(container: types.TestContainerDocker, mappings: list[Mapping]) -> None:
         Config.base_url = container.host_configs["8080"].get("/__admin")
 
         for mapping in mappings:
             Mappings.create_mapping(mapping=mapping)
 
-    yield _make_http_mocks
+        def cleanup():
+            Mappings.delete_all_mappings()
+            Requests.reset_request_journal()
 
-    Mappings.delete_all_mappings()
-    Requests.reset_request_journal()
+        request.addfinalizer(cleanup)
+
+    return _make_http_mocks

tests/fixtures/maildev.py

@@ -0,0 +1,122 @@
+import json
+from http import HTTPStatus
+
+import docker
+import docker.errors
+import pytest
+import requests
+from testcontainers.core.container import DockerContainer, Network
+
+from fixtures import reuse, types
+from fixtures.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+
+@pytest.fixture(name="maildev", scope="package")
+def maildev(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
+    """
+    Package-scoped fixture for MailDev container.
+    Provides SMTP (port 1025) and HTTP API (port 1080) for email testing.
+    """
+
+    def create() -> types.TestContainerDocker:
+        container = DockerContainer(image="maildev/maildev:2.2.1")
+        container.with_exposed_ports(1025, 1080)
+        container.with_network(network=network)
+        container.start()
+
+        return types.TestContainerDocker(
+            id=container.get_wrapped_container().id,
+            host_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1025),
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(1080),
+                ),
+            },
+            container_configs={
+                "1025": types.TestContainerUrlConfig(
+                    scheme="smtp",
+                    address=container.get_wrapped_container().name,
+                    port=1025,
+                ),
+                "1080": types.TestContainerUrlConfig(
+                    scheme="http",
+                    address=container.get_wrapped_container().name,
+                    port=1080,
+                ),
+            },
+        )
+
+    def delete(container: types.TestContainerDocker):
+        client = docker.from_env()
+        try:
+            client.containers.get(container_id=container.id).stop()
+            client.containers.get(container_id=container.id).remove(v=True)
+        except docker.errors.NotFound:
+            logger.info(
+                "Skipping removal of MailDev, MailDev(%s) not found. Maybe it was manually removed?",
+                {"id": container.id},
+            )
+
+    def restore(cache: dict) -> types.TestContainerDocker:
+        return types.TestContainerDocker.from_cache(cache)
+
+    return reuse.wrap(
+        request,
+        pytestconfig,
+        "maildev",
+        lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
+        create,
+        delete,
+        restore,
+    )
+
+
+def get_all_mails(_maildev: types.TestContainerDocker) -> list[dict]:
+    """
+    Fetches all emails from the MailDev HTTP API.
+    Returns list of dicts with keys: subject, html, text.
+    """
+    url = _maildev.host_configs["1080"].get("/email")
+    response = requests.get(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to fetch emails from MailDev, status code: {response.status_code}, response: {response.text}"
+    emails = response.json()
+    # logger.info("Emails: %s", json.dumps(emails, indent=2))
+    return [
+        {
+            "subject": email.get("subject", ""),
+            "html": email.get("html", ""),
+            "text": email.get("text", ""),
+        }
+        for email in emails
+    ]
+
+
+def verify_email_received(_maildev: types.TestContainerDocker, filters: dict) -> bool:
+    """
+    Checks if any email in MailDev matches all the given filters.
+    Filters are matched with exact equality against the email fields (subject, html, text).
+    Returns True if at least one matching email is found.
+    """
+    emails = get_all_mails(_maildev)
+    for email in emails:
+        logger.info("Email: %s", json.dumps(email, indent=2))
+        if all(key in email and filter_value == email[key] for key, filter_value in filters.items()):
+            return True
+    return False
+
+
+def delete_all_mails(_maildev: types.TestContainerDocker) -> None:
+    """
+    Deletes all emails from the MailDev inbox.
+    """
+    url = _maildev.host_configs["1080"].get("/email/all")
+    response = requests.delete(url, timeout=5)
+    assert response.status_code == HTTPStatus.OK, f"Failed to delete emails from MailDev, status code: {response.status_code}, response: {response.text}"

tests/fixtures/notification_channel.py

@@ -1,3 +1,4 @@
+# pylint: disable=line-too-long
 from collections.abc import Callable
 from http import HTTPStatus
 
@@ -15,6 +16,87 @@ from fixtures.logger import setup_logger
 logger = setup_logger(__name__)
 
 
+"""
+Default notification channel configs shared across alertmanager tests.
+"""
+slack_default_config = {
+    # channel name configured on runtime
+    "slack_configs": [
+        {
+            "api_url": "services/TEAM_ID/BOT_ID/TOKEN_ID",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+# MSTeams default config
+msteams_default_config = {
+    "msteamsv2_configs": [
+        {
+            "webhook_url": "msteams/webhook_url",  # base_url configured on runtime
+            "title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
+            "text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
+        }
+    ],
+}
+
+# pagerduty default config
+pagerduty_default_config = {
+    "pagerduty_configs": [
+        {
+            "routing_key": "PagerDutyRoutingKey",
+            "url": "v2/enqueue",  # base_url configured on runtime
+            "client": "SigNoz Alert Manager",
+            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+            "description": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n\t{{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n\t {{" "}}(\n\t {{- with .CommonLabels.Remove .GroupLabels.Names }}\n\t\t{{- range $index, $label := .SortedPairs -}}\n\t\t {{ if $index }}, {{ end }}\n\t\t {{- $label.Name }}="{{ $label.Value -}}"\n\t\t{{- end }}\n\t {{- end -}}\n\t )\n\t{{- end }}',
+            "details": {
+                "firing": '{{ template "pagerduty.default.instances" .Alerts.Firing }}',
+                "num_firing": "{{ .Alerts.Firing | len }}",
+                "num_resolved": "{{ .Alerts.Resolved | len }}",
+                "resolved": '{{ template "pagerduty.default.instances" .Alerts.Resolved }}',
+            },
+            "source": "SigNoz Alert Manager",
+            "severity": "{{ (index .Alerts 0).Labels.severity }}",
+        }
+    ],
+}
+# opsgenie default config
+opsgenie_default_config = {
+    "opsgenie_configs": [
+        {
+            "api_key": "OpsGenieAPIKey",
+            "api_url": "/",  # base_url configured on runtime
+            "description": '{{ if gt (len .Alerts.Firing) 0 -}}\r\n\tAlerts Firing:\r\n\t{{ range .Alerts.Firing }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n\tAlerts Resolved:\r\n\t{{ range .Alerts.Resolved }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }}   Annotations:\r\n\t{{ range .Annotations.SortedPairs }}   - {{ .Name }} = {{ .Value }}\r\n\t{{ end }}   Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}',
+            "priority": '{{ if eq (index .Alerts 0).Labels.severity "critical" }}P1{{ else if eq (index .Alerts 0).Labels.severity "warning" }}P2{{ else if eq (index .Alerts 0).Labels.severity "info" }}P3{{ else }}P4{{ end }}',
+            "message": "{{ .CommonLabels.alertname }}",
+            "details": {},
+        }
+    ]
+}
+
+# webhook default config
+webhook_default_config = {
+    "webhook_configs": [
+        {
+            "url": "webhook/webhook_url",  # base_url configured on runtime
+        }
+    ],
+}
+# email default config
+email_default_config = {
+    "email_configs": [
+        {
+            "to": "test@example.com",
+            "html": '<html><body>{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n   {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n   {{ end -}}\r\n {{ end -}}\r\n{{ end }}</body></html>',
+            "headers": {
+                "Subject": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}'
+            },
+        }
+    ],
+}
+
+
 @pytest.fixture(name="notification_channel", scope="package")
 def notification_channel(
     network: Network,
@@ -67,6 +149,27 @@ def notification_channel(
     )
 
 
+@pytest.fixture(name="create_notification_channel", scope="function")
+def create_notification_channel(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> Callable[[dict], str]:
+    admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    def _create_notification_channel(channel_config: dict) -> str:
+        response = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels"),
+            json=channel_config,
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=5,
+        )
+        assert response.status_code == HTTPStatus.CREATED, f"Failed to create channel, Response: {response.text} Response status: {response.status_code}"
+        return response.json()["data"]["id"]
+
+    return _create_notification_channel
+
+
 @pytest.fixture(name="create_webhook_notification_channel", scope="function")
 def create_webhook_notification_channel(
     signoz: types.SigNoz,

tests/fixtures/types.py

@@ -192,3 +192,40 @@ class AlertTestCase:
     alert_data: list[AlertData]
     # list of alert expectations for the test case
     alert_expectation: AlertExpectation
+
+
+@dataclass(frozen=True)
+class NotificationValidation:
+    # destination type of the notification, either webhook or email
+    # slack, msteams, pagerduty, opsgenie, webhook channels send notifications through webhook
+    # email channels send notifications through email
+    destination_type: Literal["webhook", "email"]
+    # validation data for validating the received notification payload
+    validation_data: dict[str, any]
+
+
+@dataclass(frozen=True)
+class AMNotificationExpectation:
+    # whether we expect any notifications to be fired or not, false when testing downtime scenarios
+    # or don't expect any notifications to be fired in given time period
+    should_notify: bool
+    # seconds to wait for the notifications to be fired, if no
+    # notifications are fired in the expected time, the test will fail
+    wait_time_seconds: int
+    # list of notifications to expect, as a single rule can trigger multiple notifications
+    # spanning across different notifiers
+    notification_validations: list[NotificationValidation]
+
+
+@dataclass(frozen=True)
+class AlertManagerNotificationTestCase:
+    # name of the test case
+    name: str
+    # path to the rule file in testdata directory
+    rule_path: str
+    # list of alert data that will be inserted into the database for the rule to be triggered
+    alert_data: list[AlertData]
+    # configuration for the notification channel
+    channel_config: dict[str, any]
+    # notification expectations for the test case
+    notification_expectation: AMNotificationExpectation

tests/integration/bootstrap/setup.py

@@ -39,5 +39,7 @@ def test_teardown(
     idp: types.TestContainerIDP,  # pylint: disable=unused-argument
     create_user_admin: types.Operation,  # pylint: disable=unused-argument
     migrator: types.Operation,  # pylint: disable=unused-argument
+    maildev: types.TestContainerDocker,  # pylint: disable=unused-argument
+    notification_channel: types.TestContainerDocker,  # pylint: disable=unused-argument
 ) -> None:
     pass

tests/integration/testdata/alertmanager/content_templating/logs_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "User login successful", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Database connection established", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "API request received", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Response sent to client", "severity_text": "INFO" }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }

tests/integration/testdata/alertmanager/content_templating/logs_rule.json

@@ -0,0 +1,69 @@
+{
+  "alert": "content_templating_logs",
+  "ruleType": "threshold_rule",
+  "alertType": "LOGS_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 0,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "logs",
+            "filter": {
+              "expression": "body CONTAINS 'payment failure'"
+            },
+            "aggregations": [
+              {
+                "expression": "count()"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Payment failure spike detected on $service_name",
+    "summary": "Payment failures elevated on $service_name"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/metrics_data.jsonl

@@ -0,0 +1,12 @@
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:01:00+00:00","value":80,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:02:00+00:00","value":95,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:03:00+00:00","value":110,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:04:00+00:00","value":120,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:05:00+00:00","value":125,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:06:00+00:00","value":130,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:07:00+00:00","value":135,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:08:00+00:00","value":140,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:09:00+00:00","value":145,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:10:00+00:00","value":150,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:11:00+00:00","value":155,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:12:00+00:00","value":160,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}

tests/integration/testdata/alertmanager/content_templating/metrics_rule.json

@@ -0,0 +1,72 @@
+{
+  "alert": "content_templating_metrics",
+  "ruleType": "threshold_rule",
+  "alertType": "METRIC_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 100,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "metrics",
+            "aggregations": [
+              {
+                "metricName": "container_memory_bytes_content_templating",
+                "timeAggregation": "avg",
+                "spaceAggregation": "max"
+              }
+            ],
+            "groupBy": [
+              {"name": "namespace", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "pod", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "container", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "node", "fieldContext": "attribute", "fieldDataType": "string"},
+              {"name": "severity", "fieldContext": "attribute", "fieldDataType": "string"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "Container $container in pod $pod ($namespace) exceeded memory threshold",
+    "summary": "High container memory in $namespace/$pod"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/testdata/alertmanager/content_templating/traces_data.jsonl

@@ -0,0 +1,20 @@
+{ "timestamp": "2026-01-29T10:00:00.000000Z", "duration": "PT1.2S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a1", "span_id": "c1b2c3d4e5f6a7b8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:00:30.000000Z", "duration": "PT1.4S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a2", "span_id": "c2b3c4d5e6f7a8b9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:00.000000Z", "duration": "PT1.6S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a3", "span_id": "c3b4c5d6e7f8a9b0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:01:30.000000Z", "duration": "PT1.8S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a4", "span_id": "c4b5c6d7e8f9a0b1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:00.000000Z", "duration": "PT2.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a5", "span_id": "c5b6c7d8e9f0a1b2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:02:30.000000Z", "duration": "PT2.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a6", "span_id": "c6b7c8d9e0f1a2b3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:00.000000Z", "duration": "PT2.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a7", "span_id": "c7b8c9d0e1f2a3b4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:03:30.000000Z", "duration": "PT2.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a8", "span_id": "c8b9c0d1e2f3a4b5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:00.000000Z", "duration": "PT2.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a9", "span_id": "c9b0c1d2e3f4a5b6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:04:30.000000Z", "duration": "PT3.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b1", "span_id": "d1c2d3e4f5a6b7c8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:00.000000Z", "duration": "PT3.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b2", "span_id": "d2c3d4e5f6a7b8c9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:05:30.000000Z", "duration": "PT3.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b3", "span_id": "d3c4d5e6f7a8b9c0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:00.000000Z", "duration": "PT3.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b4", "span_id": "d4c5d6e7f8a9b0c1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:06:30.000000Z", "duration": "PT3.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b5", "span_id": "d5c6d7e8f9a0b1c2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:00.000000Z", "duration": "PT4.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b6", "span_id": "d6c7d8e9f0a1b2c3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:07:30.000000Z", "duration": "PT4.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b7", "span_id": "d7c8d9e0f1a2b3c4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:00.000000Z", "duration": "PT4.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b8", "span_id": "d8c9d0e1f2a3b4c5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:08:30.000000Z", "duration": "PT4.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b9", "span_id": "d9c0d1e2f3a4b5c6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:09:00.000000Z", "duration": "PT4.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c1", "span_id": "e1d2e3f4a5b6c7d8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
+{ "timestamp": "2026-01-29T10:10:00.000000Z", "duration": "PT5.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c2", "span_id": "e2d3e4f5a6b7c8d9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }

tests/integration/testdata/alertmanager/content_templating/traces_rule.json

@@ -0,0 +1,71 @@
+{
+  "alert": "content_templating_traces",
+  "ruleType": "threshold_rule",
+  "alertType": "TRACES_BASED_ALERT",
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 1,
+          "matchType": "1",
+          "op": "1",
+          "channels": [
+            "test channel"
+          ],
+          "targetUnit": "s"
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "builder",
+      "unit": "ns",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "builder_query",
+          "spec": {
+            "name": "A",
+            "signal": "traces",
+            "filter": {
+              "expression": "http.request.path = '/checkout'"
+            },
+            "aggregations": [
+              {
+                "expression": "p90(duration_nano)"
+              }
+            ],
+            "groupBy": [
+              {"name": "service.name", "fieldContext": "resource"}
+            ]
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "p90 latency high on $service_name",
+    "summary": "p90 latency exceeded threshold on $service_name"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/tests/alertmanager/01_notifers.py

@@ -0,0 +1,372 @@
+import json
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    email_default_config,
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+# tests to verify the notifiers sending out the notifications with expected content
+NOTIFIERS_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="slack_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            # extra wait for alertmanager server setup
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="msteams_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                                        "type": "AdaptiveCard",
+                                        "version": "1.2",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Alerts",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                                "wrap": True,
+                                                "color": "Attention",
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Labels",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "threshold.name",
+                                                        "value": "critical",
+                                                    }
+                                                ],
+                                            },
+                                            {
+                                                "type": "TextBlock",
+                                                "text": "Annotations",
+                                                "weight": "Bolder",
+                                                "size": "Medium",
+                                            },
+                                            {
+                                                "type": "FactSet",
+                                                "text": "",
+                                                "facts": [
+                                                    {
+                                                        "title": "description",
+                                                        "value": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                                    },
+                                                ],
+                                            },
+                                        ],
+                                        "msteams": {"width": "full"},
+                                        "actions": [
+                                            {
+                                                "type": "Action.OpenUrl",
+                                                "title": "View Alert",
+                                            }
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "event_action": "trigger",
+                            "payload": {
+                                "source": "SigNoz Alert Manager",
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Annotations": [
+                                            {"description = This alert is fired when the defined metric (current value": "15) crosses the threshold (10)"},
+                                        ],
+                                        "Labels": [
+                                            "alertname = threshold_above_at_least_once",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "threshold_above_at_least_once",
+                            "details": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "threshold_above_at_least_once",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                        "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "threshold_above_at_least_once",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "commonAnnotations": {
+                                "description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                                "summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="email_notifier_default_templating",
+        rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
+            ),
+        ],
+        channel_config=email_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="email",
+                    validation_data={
+                        "subject": '[FIRING:1] threshold_above_at_least_once for  (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+logger = setup_logger(__name__)
+
+
+@pytest.mark.parametrize(
+    "notifier_test_case",
+    NOTIFIERS_TEST,
+    ids=lambda notifier_test_case: notifier_test_case.name,
+)
+def test_notifier_templating(
+    # wiremock container for webhook notifications
+    notification_channel: types.TestContainerDocker,
+    # function to create wiremock mocks
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    # function to create alert rule
+    create_alert_rule: Callable[[dict], str],
+    # Alert data insertion related fixture
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    # Mail dev container for email verification
+    maildev: types.TestContainerDocker,
+    # test case from parametrize
+    notifier_test_case: types.AlertManagerNotificationTestCase,
+):
+    # generate unique channel name
+    channel_name = str(uuid.uuid4())
+
+    # update channel config: set name and rewrite URLs to wiremock
+    channel_config = update_raw_channel_config(notifier_test_case.channel_config, channel_name, notification_channel, maildev)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    # setup wiremock mocks for webhook-based notification validations
+    webhook_validations = [v for v in notifier_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    # clear mails if any destination is email
+    if any(v.destination_type == "email" for v in notifier_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    # create notification channel
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    # insert alert data
+    insert_alert_data(
+        notifier_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=5),
+    )
+
+    # create alert rule
+    rule_path = get_testdata_file_path(notifier_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    # verify notification expectations
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        notifier_test_case.notification_expectation,
+    )

tests/integration/tests/alertmanager/02_content_templating.py

@@ -0,0 +1,341 @@
+import json
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.maildev import delete_all_mails
+from fixtures.notification_channel import (
+    msteams_default_config,
+    opsgenie_default_config,
+    pagerduty_default_config,
+    slack_default_config,
+    webhook_default_config,
+)
+
+logger = setup_logger(__name__)
+
+
+# Test cases verifying default notification content
+CONTENT_TEMPLATING_TEST = [
+    types.AlertManagerNotificationTestCase(
+        name="msteams_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=msteams_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/msteams/webhook_url",
+                        "json_body": {
+                            "type": "message",
+                            "attachments": [
+                                {
+                                    "contentType": "application/vnd.microsoft.card.adaptive",
+                                    "content": {
+                                        "type": "AdaptiveCard",
+                                        "body": [
+                                            {
+                                                "type": "TextBlock",
+                                                "text": '[FIRING:1] content_templating_metrics for  (alertname="content_templating_metrics", container="checkout", namespace="production", node="ip-10-0-1-23", pod="checkout-7d9c8b5f4-x2k9p", severity="critical", threshold.name="critical")',
+                                            },
+                                        ],
+                                    },
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="opsgenie_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=opsgenie_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/alerts",
+                        "json_body": {
+                            "message": "content_templating_metrics",
+                            "details": {
+                                "alertname": "content_templating_metrics",
+                                "container": "checkout",
+                                "namespace": "production",
+                                "node": "ip-10-0-1-23",
+                                "pod": "checkout-7d9c8b5f4-x2k9p",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                            "priority": "P1",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="pagerduty_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=pagerduty_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v2/enqueue",
+                        "json_body": {
+                            "routing_key": "PagerDutyRoutingKey",
+                            "payload": {
+                                "severity": "critical",
+                                "custom_details": {
+                                    "firing": {
+                                        "Labels": [
+                                            "alertname = content_templating_metrics",
+                                            "container = checkout",
+                                            "namespace = production",
+                                            "node = ip-10-0-1-23",
+                                            "pod = checkout-7d9c8b5f4-x2k9p",
+                                            "severity = critical",
+                                            "threshold.name = critical",
+                                        ],
+                                    }
+                                },
+                            },
+                            "client": "SigNoz Alert Manager",
+                            "client_url": "https://enter-signoz-host-n-port-here/alerts",
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="slack_logs_default_templating",
+        rule_path="alertmanager/content_templating/logs_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="logs",
+                data_path="alertmanager/content_templating/logs_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="slack_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=slack_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
+                        "json_body": {
+                            "username": "Alertmanager",
+                            "attachments": [
+                                {
+                                    "color": "danger",
+                                    "mrkdwn_in": ["fallback", "pretext", "text"],
+                                }
+                            ],
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="webhook_metrics_default_templating",
+        rule_path="alertmanager/content_templating/metrics_rule.json",
+        alert_data=[
+            types.AlertData(
+                type="metrics",
+                data_path="alertmanager/content_templating/metrics_data.jsonl",
+            ),
+        ],
+        channel_config=webhook_default_config,
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=120,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/webhook/webhook_url",
+                        "json_body": {
+                            "status": "firing",
+                            "alerts": [
+                                {
+                                    "status": "firing",
+                                    "labels": {
+                                        "alertname": "content_templating_metrics",
+                                        "container": "checkout",
+                                        "namespace": "production",
+                                        "node": "ip-10-0-1-23",
+                                        "pod": "checkout-7d9c8b5f4-x2k9p",
+                                        "severity": "critical",
+                                        "threshold.name": "critical",
+                                    },
+                                    "annotations": {
+                                        "description": "Container checkout in pod checkout-7d9c8b5f4-x2k9p (production) exceeded memory threshold",
+                                        "summary": "High container memory in production/checkout-7d9c8b5f4-x2k9p",
+                                    },
+                                }
+                            ],
+                            "commonLabels": {
+                                "alertname": "content_templating_metrics",
+                                "container": "checkout",
+                                "namespace": "production",
+                                "node": "ip-10-0-1-23",
+                                "pod": "checkout-7d9c8b5f4-x2k9p",
+                                "severity": "critical",
+                                "threshold.name": "critical",
+                            },
+                        },
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "default_templating_test_case",
+    CONTENT_TEMPLATING_TEST,
+    ids=lambda default_templating_test_case: default_templating_test_case.name,
+)
+def test_content_templating(
+    # wiremock container for webhook notifications
+    notification_channel: types.TestContainerDocker,
+    # function to create wiremock mocks
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    # function to create alert rule
+    create_alert_rule: Callable[[dict], str],
+    # Alert data insertion related fixture
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    # Mail dev container for email verification
+    maildev: types.TestContainerDocker,
+    # test case from parametrize
+    default_templating_test_case: types.AlertManagerNotificationTestCase,
+):
+    # generate unique channel name
+    channel_name = str(uuid.uuid4())
+
+    # update channel config: set name and rewrite URLs to wiremock
+    channel_config = update_raw_channel_config(default_templating_test_case.channel_config, channel_name, notification_channel, maildev)
+    logger.info("Channel config: %s", {"channel_config": channel_config})
+
+    # setup wiremock mocks for webhook-based notification validations
+    webhook_validations = [v for v in default_templating_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
+    if len(webhook_validations) > 0:
+        mock_mappings = [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
+                response=MappingResponse(status=200, json_body={}),
+                persistent=False,
+            )
+            for v in webhook_validations
+        ]
+
+        make_http_mocks(notification_channel, mock_mappings)
+        logger.info("Mock mappings created")
+
+    # clear mails if any destination is email
+    if any(v.destination_type == "email" for v in default_templating_test_case.notification_expectation.notification_validations):
+        delete_all_mails(maildev)
+        logger.info("Mails deleted")
+
+    # create notification channel
+    create_notification_channel(channel_config)
+    logger.info("Channel created with name: %s", {"channel_name": channel_name})
+
+    # insert alert data
+    insert_alert_data(
+        default_templating_test_case.alert_data,
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=10),
+    )
+
+    # create alert rule
+    rule_path = get_testdata_file_path(default_templating_test_case.rule_path)
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
+
+    # verify notification expectations
+    verify_notification_expectation(
+        notification_channel,
+        maildev,
+        default_templating_test_case.notification_expectation,
+    )

tests/integration/tests/alertmanager/conftest.py

@@ -0,0 +1,42 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    maildev: types.TestContainerDocker,
+    notification_channel: types.TestContainerDocker,
+) -> types.SigNoz:
+    """
+    Package-scoped fixture for setting up SigNoz.
+    Overrides SMTP, PagerDuty, and OpsGenie URLs to point to test containers.
+    """
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        env_overrides={
+            # SMTP config for email notifications via maildev
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__SMARTHOST": f"{maildev.container_configs['1025'].address}:{maildev.container_configs['1025'].port}",
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__REQUIRE__TLS": "false",
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__FROM": "alertmanager@signoz.io",
+            # PagerDuty API URL -> wiremock (default: https://events.pagerduty.com/v2/enqueue)
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_PAGERDUTY__URL": notification_channel.container_configs["8080"].get("/v2/enqueue"),
+            # OpsGenie API URL -> wiremock (default: https://api.opsgenie.com/)
+            "SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_OPSGENIE__API__URL": notification_channel.container_configs["8080"].get("/"),
+        },
+    )

tests/pyproject.toml

@@ -75,3 +75,7 @@ ignore = [
 
 [tool.ruff.format]
 # Defaults align with black (double quotes, 4-space indent).
+
+[tool.ruff.lint.per-file-ignores]
+"integration/src/alertmanager/*" = ["E501"]
+"fixtures/notification_channel.py" = ["E501"]