Merge branch 'main' of github.com:SigNoz/signoz into feat/alert-template-variables

mapstructure turned time.Time into empty maps, so date, tz and since failed on $alert.starts_at.
buildDataMap now uses toTemplateMap, a tag-driven converter that keeps times; output is unchanged.
Adds now (time.Now) to the shared function map, so channel templates get it too.
Adds engine tests for every function in the Functions API list.
A bare $alert.starts_at now prints the time, and {{ if $alert.ends_at }} is always true.

Assisted-by: Claude Opus 5.5

.devenv/docker/clickhouse/compose.yaml

@@ -67,7 +67,7 @@ services:
       timeout: 5s
       retries: 3
   telemetrystore-migrator:
-    image: signoz/signoz-otel-collector:v0.144.15
+    image: signoz/signoz-otel-collector:v0.144.6
     container_name: telemetrystore-migrator
     environment:
       - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://clickhouse:9000

.devenv/docker/signoz-otel-collector/compose.yaml

@@ -1,6 +1,6 @@
 services:
   signoz-otel-collector:
-    image: signoz/signoz-otel-collector:v0.144.15
+    image: signoz/signoz-otel-collector:v0.142.0
     container_name: signoz-otel-collector-dev
     entrypoint:
       - /bin/sh

.github/workflows/e2eci.yaml

@@ -88,7 +88,7 @@ jobs:
             postgres:15 \
             clickhouse/clickhouse-server:25.12.5 \
             clickhouse/clickhouse-keeper:25.12.5 \
-            signoz/signoz-otel-collector:v0.144.16-rc.1 \
+            signoz/signoz-schema-migrator:v0.144.9 \
             wiremock/wiremock:2.35.1-1 \
             | xargs -P 0 -n 1 docker pull -q
       - name: python

.github/workflows/integrationci.yaml

@@ -40,7 +40,6 @@ jobs:
         suite:
           - alertmanager
           - alertmanagerrotation
-          - auditquerier
           - basepath
           - callbackauthn
           - cloudintegrations
@@ -84,8 +83,8 @@ jobs:
           - wal
         clickhouse-version:
           - 25.12.5
-        signoz-otel-collector-version:
-          - v0.144.16-rc.1
+        schema-migrator-version:
+          - v0.144.6
         postgres-version:
           - 15
     if: |
@@ -156,4 +155,4 @@ jobs:
             --sqlite-mode ${{matrix.sqlite-mode}} \
             --postgres-version ${{matrix.postgres-version}} \
             --clickhouse-version ${{matrix.clickhouse-version}} \
-            --signoz-otel-collector-version ${{matrix.signoz-otel-collector-version}}
+            --schema-migrator-version ${{matrix.schema-migrator-version}}

docs/contributing/tests/integration.md

@@ -256,7 +256,7 @@ Tests can be configured using pytest options:
 - `--sqlite-mode` — SQLite journal mode: `delete` or `wal` (default: `delete`). Only relevant when `--sqlstore-provider=sqlite`.
 - `--postgres-version` — PostgreSQL version (default: `15`)
 - `--clickhouse-version` — ClickHouse version, also used for ClickHouse Keeper (default: `25.12.5`)
-- `--signoz-otel-collector-version`: SigNoz OTel collector version, used to run the ClickHouse migrations (default: `v0.144.16-rc.1`)
+- `--schema-migrator-version` — SigNoz schema migrator version (default: `v0.144.9`)
 - `--with-web` — Build the frontend into the SigNoz image (required for e2e)
 
 Example:

ee/query-service/rules/anomaly.go

@@ -275,7 +275,7 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 		lb.Set(ruletypes.AlertRuleIDLabel, r.ID())
 		lb.Set(ruletypes.RuleSourceLabel, r.GeneratorURL())
 
-		annotations := r.ExpandAnnotations(expand)
+		annotations := r.AlertAnnotations(expand, smpl, value, threshold)
 		if smpl.IsMissing {
 			lb.Set(ruletypes.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(ruletypes.NoDataLabel, "true")

ee/query-service/rules/anomaly_test.go

@@ -129,10 +129,97 @@ func TestAnomalyRule_NoData_AlertOnAbsent(t *testing.T) {
 			alertsFound, err := rule.Eval(context.Background(), evalTime)
 			require.NoError(t, err)
 			assert.Equal(t, c.expectAlerts, alertsFound)
+			for _, item := range rule.Active {
+				assert.Empty(t, item.Annotations.Get(ruletypes.AnnotationValue))
+				assert.Empty(t, item.Annotations.Get(ruletypes.AnnotationThresholdValue))
+			}
 		})
 	}
 }
 
+func TestAnomalyRule_ValueAndThresholdAnnotations(t *testing.T) {
+	baseTime := time.Unix(1700000000, 0)
+	evalTime := baseTime.Add(5 * time.Minute)
+
+	target := 2.0
+
+	postableRule := ruletypes.PostableRule{
+		AlertName: valuer.NewDisplayName("Test anomaly value"),
+		AlertType: ruletypes.AlertTypeMetric,
+		RuleType:  ruletypes.RuleTypeAnomaly,
+		Evaluation: &ruletypes.EvaluationEnvelope{Kind: ruletypes.RollingEvaluation, Spec: ruletypes.RollingWindow{
+			EvalWindow: valuer.MustParseTextDuration("5m"),
+			Frequency:  valuer.MustParseTextDuration("1m"),
+		}},
+		RuleCondition: &ruletypes.RuleCondition{
+			CompareOperator: ruletypes.ValueIsAbove,
+			MatchType:       ruletypes.AtleastOnce,
+			Target:          &target,
+			CompositeQuery: &ruletypes.AlertCompositeQuery{
+				QueryType: ruletypes.QueryTypeBuilder,
+				Queries: []qbtypes.QueryEnvelope{{
+					Type: qbtypes.QueryTypeBuilder,
+					Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+						Name:   "A",
+						Signal: telemetrytypes.SignalMetrics,
+					},
+				}},
+			},
+			SelectedQuery: "A",
+			Seasonality:   ruletypes.SeasonalityDaily,
+			Thresholds: &ruletypes.RuleThresholdData{
+				Kind: ruletypes.BasicThresholdKind,
+				Spec: ruletypes.BasicRuleThresholds{{
+					Name:            "Test anomaly value",
+					TargetValue:     &target,
+					MatchType:       ruletypes.AtleastOnce,
+					CompareOperator: ruletypes.ValueIsAbove,
+				}},
+			},
+		},
+		Annotations: map[string]string{
+			"summary":     "{{$value}}",
+			"description": "{{$threshold}}",
+		},
+	}
+
+	response := &anomaly.AnomaliesResponse{
+		Results: []*qbtypes.TimeSeriesData{
+			{
+				QueryName: "A",
+				Aggregations: []*qbtypes.AggregationBucket{{
+					AnomalyScores: []*qbtypes.TimeSeries{{
+						Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "service.name"}, Value: "checkout"}},
+						Values: []*qbtypes.TimeSeriesValue{
+							{Timestamp: baseTime.Add(1 * time.Minute).UnixMilli(), Value: 1.5},
+							{Timestamp: baseTime.Add(2 * time.Minute).UnixMilli(), Value: 3.5},
+						},
+					}},
+				}},
+			},
+		},
+	}
+
+	rule, err := NewAnomalyRule("test-anomaly-rule", valuer.GenerateUUID(), &postableRule, nil, instrumentationtest.New().Logger(), mustParseURL(t, "http://localhost:8000"))
+	require.NoError(t, err)
+
+	rule.provider = &mockAnomalyProvider{responses: []*anomaly.AnomaliesResponse{response}}
+
+	alertsFound, err := rule.Eval(context.Background(), evalTime)
+	require.NoError(t, err)
+	require.Equal(t, 1, alertsFound)
+
+	for _, item := range rule.Active {
+		value := item.Annotations.Get(ruletypes.AnnotationValue)
+		assert.NotEmpty(t, value)
+		assert.Equal(t, item.Annotations.Get("summary"), value)
+
+		threshold := item.Annotations.Get(ruletypes.AnnotationThresholdValue)
+		assert.NotEmpty(t, threshold)
+		assert.Equal(t, item.Annotations.Get("description"), threshold)
+	}
+}
+
 func TestAnomalyRule_NoData_AbsentFor(t *testing.T) {
 	// Test missing data alert with AbsentFor grace period
 	// 1. Call Eval with data at time t1, to populate lastTimestampWithDatapoints

frontend/src/container/AIAssistant/components/ActionsSection/ActionsSection.tsx

@@ -1,6 +1,4 @@
-import { useCallback, useEffect, useState } from 'react';
-// eslint-disable-next-line no-restricted-imports
-import { useDispatch } from 'react-redux';
+import { useEffect, useState } from 'react';
 import { convertFiltersToExpression } from 'components/QueryBuilderV2/utils';
 import { matchRoute } from 'lib/router/matchRoute';
 import { navigate } from 'lib/router/navigation';
@@ -28,7 +26,6 @@ import {
 } from 'api/ai-assistant/chat';
 import ROUTES from 'constants/routes';
 import { QueryParams } from 'constants/query';
-import { UpdateTimeInterval } from 'store/actions';
 import { buildExplorerNavigationUrl } from 'utils/buildExplorerNavigationUrl';
 import { openInNewTab } from 'utils/navigation';
 import {
@@ -56,7 +53,6 @@ import {
 	getPanelTypeForRequestType,
 	requestTypeFromActionQuery,
 } from './utils/getPanelTypeForRequestType';
-import { getTimeRangeFromActionQuery } from './utils/getTimeRangeFromActionQuery';
 import { isOnResourcePage } from './utils/isOnResourcePage';
 import { openSavedViewByKey } from './utils/openSavedView';
 import {
@@ -283,7 +279,6 @@ interface ApplyFilterDeps {
 		typeof useQueryBuilder
 	>['redirectWithQueryBuilderData'];
 	handleSetQueryData: ReturnType<typeof useQueryBuilder>['handleSetQueryData'];
-	setCustomTimeRange: (start: number, end: number) => void;
 }
 
 /**
@@ -349,35 +344,18 @@ function applyFilter(action: MessageActionDTO, deps: ApplyFilterDeps): void {
 		requestTypeFromActionQuery(action.query),
 		action.signal,
 	);
-	const timeRange = getTimeRangeFromActionQuery(action.query);
 	const normalized = withDerivedFilterExpressions(urlQuery as unknown as Query);
 	// eslint-disable-next-line no-console
 	console.log('[apply_filter] normalized', normalized);
 	if (signalMatchesPathname(action.signal, deps.pathname)) {
 		// eslint-disable-next-line no-console
 		console.log('[apply_filter] on-page → handleSetQueryData + redirect');
-		if (timeRange) {
-			deps.setCustomTimeRange(timeRange.start, timeRange.end);
-		}
 		normalized.builder.queryData.forEach((q, i) => {
 			deps.handleSetQueryData(i, q);
 		});
-		deps.redirectWithQueryBuilderData(
-			normalized,
-			{
-				[QueryParams.panelTypes]: panelType,
-				...(timeRange && {
-					[QueryParams.startTime]: timeRange.start,
-					[QueryParams.endTime]: timeRange.end,
-					// relativeTime outranks startTime/endTime in the time picker, so drop it.
-					[QueryParams.relativeTime]: null,
-				}),
-			},
-			undefined,
-			undefined,
-			// New params like startTime would otherwise make safeNavigate replace the entry.
-			{ replace: false },
-		);
+		deps.redirectWithQueryBuilderData(normalized, {
+			[QueryParams.panelTypes]: panelType,
+		});
 		return;
 	}
 	const base = explorerRouteForSignal(action.signal);
@@ -390,10 +368,6 @@ function applyFilter(action: MessageActionDTO, deps: ApplyFilterDeps): void {
 	console.log('[apply_filter] off-page → navigate', base);
 	const url = buildExplorerNavigationUrl(base, normalized, {
 		[QueryParams.panelTypes]: panelType,
-		...(timeRange && {
-			[QueryParams.startTime]: timeRange.start,
-			[QueryParams.endTime]: timeRange.end,
-		}),
 	});
 	navigate(url);
 }
@@ -428,13 +402,6 @@ export default function ActionsSection({
 	const sendMessage = useAIAssistantStore((s) => s.sendMessage);
 	const { threadId, page, mode } = useAIAssistantAnalyticsContext();
 	const { redirectWithQueryBuilderData, handleSetQueryData } = useQueryBuilder();
-	const dispatch = useDispatch();
-	const setCustomTimeRange = useCallback(
-		(start: number, end: number): void => {
-			dispatch(UpdateTimeInterval('custom', [start, end]));
-		},
-		[dispatch],
-	);
 
 	// Per-chip click state, keyed by chip key (see `key` below). Persists
 	// loading/success/error so the chip reflects the rollback outcome until
@@ -466,16 +433,9 @@ export default function ActionsSection({
 				pathname,
 				redirectWithQueryBuilderData,
 				handleSetQueryData,
-				setCustomTimeRange,
 			});
 		});
-	}, [
-		actions,
-		pathname,
-		redirectWithQueryBuilderData,
-		handleSetQueryData,
-		setCustomTimeRange,
-	]);
+	}, [actions, pathname, redirectWithQueryBuilderData, handleSetQueryData]);
 
 	if (actions.length === 0) {
 		return null;
@@ -608,7 +568,6 @@ export default function ActionsSection({
 					pathname,
 					redirectWithQueryBuilderData,
 					handleSetQueryData,
-					setCustomTimeRange,
 				});
 				break;
 			}

frontend/src/container/AIAssistant/components/ActionsSection/__tests__/ActionsSection.test.tsx

@@ -9,7 +9,6 @@ import { QueryParams } from 'constants/query';
 import { PANEL_TYPES } from 'constants/queryBuilder';
 import ROUTES from 'constants/routes';
 import { getCurrentLocation } from 'lib/router/navigation';
-import { UpdateTimeInterval } from 'store/actions';
 import { render, screen, userEvent, waitFor } from 'tests/test-utils';
 import { QueryBuilderContextType } from 'types/common/queryBuilder';
 
@@ -25,17 +24,7 @@ jest.mock('api/common/logEvent', () => ({
 	default: jest.fn(),
 }));
 
-jest.mock('store/actions', () => {
-	const actual =
-		jest.requireActual<typeof import('store/actions')>('store/actions');
-	return { ...actual, UpdateTimeInterval: jest.fn(actual.UpdateTimeInterval) };
-});
-
 const mockUndo = undoExecution as jest.MockedFunction<typeof undoExecution>;
-const mockUpdateTimeInterval = jest.mocked(UpdateTimeInterval);
-
-const START_MS = Date.UTC(2026, 6, 26, 10, 34);
-const END_MS = Date.UTC(2026, 6, 26, 10, 49);
 
 const UNDO_ACTION = {
 	kind: MessageActionKindDTO.undo,
@@ -49,18 +38,13 @@ function renderUndoChip(): HTMLElement {
 }
 
 // Labels differ per test: auto-applied filters are deduped module-wide by label.
-function countByServiceAction(
-	label: string,
-	{ start, end }: { start?: number; end?: number } = {},
-): MessageActionDTO {
+function countByServiceAction(label: string): MessageActionDTO {
 	return {
 		kind: MessageActionKindDTO.apply_filter,
 		label,
 		signal: ApplyFilterSignalDTO.logs,
 		query: {
 			requestType: Querybuildertypesv5RequestTypeDTO.scalar,
-			start,
-			end,
 			compositeQuery: {
 				queries: [
 					{
@@ -78,8 +62,10 @@ function countByServiceAction(
 	};
 }
 
-function currentParam(param: string): string | null {
-	return new URLSearchParams(getCurrentLocation().search).get(param);
+function currentPanelTypesParam(): string | null {
+	return new URLSearchParams(getCurrentLocation().search).get(
+		QueryParams.panelTypes,
+	);
 }
 
 describe('ActionsSection rollback chip', () => {
@@ -153,34 +139,7 @@ describe('ActionsSection apply_filter', () => {
 		await waitFor(() =>
 			expect(getCurrentLocation().pathname).toBe(ROUTES.LOGS_EXPLORER),
 		);
-		expect(currentParam(QueryParams.panelTypes)).toBe(
-			JSON.stringify(PANEL_TYPES.TABLE),
-		);
-	});
-
-	it('opens the explorer on the time range that produced the answer', async () => {
-		const user = userEvent.setup({ pointerEventsCheck: 0 });
-		render(
-			<ActionsSection
-				actions={[
-					countByServiceAction('Open with time', {
-						start: START_MS,
-						end: END_MS,
-					}),
-				]}
-				messageId="message-with-time"
-			/>,
-			undefined,
-			{ initialRoute: ROUTES.HOME },
-		);
-
-		await user.click(screen.getByTestId('ai-action-chip-apply_filter'));
-
-		await waitFor(() =>
-			expect(getCurrentLocation().pathname).toBe(ROUTES.LOGS_EXPLORER),
-		);
-		expect(currentParam(QueryParams.startTime)).toBe(String(START_MS));
-		expect(currentParam(QueryParams.endTime)).toBe(String(END_MS));
+		expect(currentPanelTypesParam()).toBe(JSON.stringify(PANEL_TYPES.TABLE));
 	});
 
 	it('switches the open explorer to the Table view when applied on it', async () => {
@@ -208,54 +167,6 @@ describe('ActionsSection apply_filter', () => {
 		expect(redirectWithQueryBuilderData).toHaveBeenCalledWith(
 			expect.objectContaining({ builder: expect.anything() }),
 			{ [QueryParams.panelTypes]: PANEL_TYPES.TABLE },
-			undefined,
-			undefined,
-			{ replace: false },
 		);
 	});
-
-	it('switches the open explorer to the answer time range', async () => {
-		const redirectWithQueryBuilderData = jest.fn() as jest.MockedFunction<
-			QueryBuilderContextType['redirectWithQueryBuilderData']
-		>;
-		const handleSetQueryData = jest.fn() as jest.MockedFunction<
-			QueryBuilderContextType['handleSetQueryData']
-		>;
-		render(
-			<ActionsSection
-				actions={[
-					countByServiceAction('Apply with time', {
-						start: START_MS,
-						end: END_MS,
-					}),
-				]}
-				messageId="message-on-page-with-time"
-			/>,
-			undefined,
-			{
-				initialRoute: ROUTES.LOGS_EXPLORER,
-				queryBuilderOverrides: { redirectWithQueryBuilderData, handleSetQueryData },
-			},
-		);
-
-		await waitFor(() =>
-			expect(redirectWithQueryBuilderData).toHaveBeenCalledTimes(1),
-		);
-		expect(redirectWithQueryBuilderData).toHaveBeenCalledWith(
-			expect.objectContaining({ builder: expect.anything() }),
-			{
-				[QueryParams.panelTypes]: PANEL_TYPES.TABLE,
-				[QueryParams.startTime]: START_MS,
-				[QueryParams.endTime]: END_MS,
-				[QueryParams.relativeTime]: null,
-			},
-			undefined,
-			undefined,
-			{ replace: false },
-		);
-		expect(mockUpdateTimeInterval).toHaveBeenCalledWith('custom', [
-			START_MS,
-			END_MS,
-		]);
-	});
 });

frontend/src/container/AIAssistant/components/ActionsSection/utils/getTimeRangeFromActionQuery.ts

@@ -1,25 +0,0 @@
-import type { MessageActionDTOQuery } from 'api/ai-assistant/sigNozAIAssistantAPI.schemas';
-import { normalizeTimeToMs } from 'utils/timeUtils';
-
-export interface ActionTimeRange {
-	start: number;
-	end: number;
-}
-
-function toTimestampMs(value: unknown): number | null {
-	if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
-		return null;
-	}
-	return normalizeTimeToMs(Math.trunc(value));
-}
-
-export function getTimeRangeFromActionQuery(
-	query: MessageActionDTOQuery | undefined,
-): ActionTimeRange | null {
-	const start = toTimestampMs(query?.start);
-	const end = toTimestampMs(query?.end);
-	if (start === null || end === null || end <= start) {
-		return null;
-	}
-	return { start, end };
-}

frontend/src/hooks/useHandleExplorerTabChange.ts

@@ -90,7 +90,7 @@ export const useHandleExplorerTabChange = (): {
 				},
 				redirectToUrl,
 				undefined,
-				{ newTab },
+				newTab,
 			);
 		},
 		[panelType, getUpdateQuery, redirectWithQueryBuilderData, viewKey],

frontend/src/providers/QueryBuilder.tsx

@@ -949,7 +949,7 @@ export function QueryBuilderProvider({
 			searchParams?: Record<string, unknown>,
 			redirectingUrl?: (typeof ROUTES)[keyof typeof ROUTES],
 			shouldNotStringify?: boolean,
-			navigateOptions?: { newTab?: boolean; replace?: boolean },
+			newTab?: boolean,
 		) => {
 			const queryType =
 				!query.queryType || !Object.values(EQueryType).includes(query.queryType)
@@ -1001,18 +1001,14 @@ export function QueryBuilderProvider({
 			);
 
 			if (searchParams) {
-				Object.keys(searchParams).forEach((param) => {
-					if (searchParams[param] === null) {
-						urlQuery.delete(param);
-						return;
-					}
+				Object.keys(searchParams).forEach((param) =>
 					urlQuery.set(
 						param,
 						shouldNotStringify
 							? (searchParams[param] as string)
 							: JSON.stringify(searchParams[param]),
-					);
-				});
+					),
+				);
 			}
 			// Remove Hidden Filters from URL query parameters on query change
 			urlQuery.delete(QueryParams.activeLogId);
@@ -1021,7 +1017,7 @@ export function QueryBuilderProvider({
 				? `${redirectingUrl}?${urlQuery}`
 				: `${location.pathname}?${urlQuery}`;
 
-			safeNavigate(generatedUrl, navigateOptions);
+			safeNavigate(generatedUrl, { newTab });
 		},
 		[location.pathname, safeNavigate],
 	);

frontend/src/providers/__tests__/QueryBuilder.redirectSearchParams.test.tsx

@@ -1,53 +0,0 @@
-import { safeNavigateMock } from '__tests__/safeNavigateMock';
-import { QueryParams } from 'constants/query';
-import { initialQueriesMap } from 'constants/queryBuilder';
-import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
-import { act, AllTheProviders, renderHook } from 'tests/test-utils';
-
-describe('redirectWithQueryBuilderData search params', () => {
-	it('removes params passed as null and JSON-encodes the rest', () => {
-		const { result } = renderHook(() => useQueryBuilder(), {
-			wrapper: ({ children }): JSX.Element => (
-				<AllTheProviders
-					initialRoute={`/unconstrained?${QueryParams.relativeTime}=6h&${QueryParams.viewName}=kept`}
-				>
-					{children}
-				</AllTheProviders>
-			),
-		});
-
-		act(() => {
-			result.current.redirectWithQueryBuilderData(initialQueriesMap.logs, {
-				[QueryParams.relativeTime]: null,
-				[QueryParams.startTime]: 1000,
-			});
-		});
-
-		const [url] = safeNavigateMock.mock.lastCall ?? [];
-		expect(url).toContain(`${QueryParams.startTime}=1000`);
-		expect(url).not.toContain(`${QueryParams.relativeTime}=`);
-		expect(url).toContain(`${QueryParams.viewName}=kept`);
-	});
-
-	it('passes navigate options to safeNavigate', () => {
-		const { result } = renderHook(() => useQueryBuilder(), {
-			wrapper: ({ children }): JSX.Element => (
-				<AllTheProviders initialRoute="/unconstrained">{children}</AllTheProviders>
-			),
-		});
-
-		act(() => {
-			result.current.redirectWithQueryBuilderData(
-				initialQueriesMap.logs,
-				{ [QueryParams.startTime]: 1000 },
-				undefined,
-				undefined,
-				{ replace: false },
-			);
-		});
-
-		expect(safeNavigateMock).toHaveBeenLastCalledWith(expect.any(String), {
-			replace: false,
-		});
-	});
-});

frontend/src/types/common/queryBuilder.ts

@@ -312,7 +312,7 @@ export type QueryBuilderContextType = {
 		searchParams?: Record<string, unknown>,
 		redirectToUrl?: (typeof ROUTES)[keyof typeof ROUTES],
 		shallStringify?: boolean,
-		navigateOptions?: { newTab?: boolean; replace?: boolean },
+		newTab?: boolean,
 	) => void;
 	handleRunQuery: () => void;
 	resetQuery: (newCurrentQuery?: QueryState) => void;

go.mod

@@ -5,9 +5,9 @@ go 1.25.7
 require (
 	dario.cat/mergo v1.0.2
 	github.com/AfterShip/clickhouse-sql-parser v0.5.6
-	github.com/ClickHouse/clickhouse-go/v2 v2.47.0
+	github.com/ClickHouse/clickhouse-go/v2 v2.44.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
-	github.com/SigNoz/clickhouse-go-mock v0.15.1
+	github.com/SigNoz/clickhouse-go-mock v0.14.0
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
 	github.com/SigNoz/signoz-otel-collector v0.144.6
 	github.com/SigNoz/signoz-semantic-conventions/gen/go v0.0.2
@@ -75,13 +75,13 @@ require (
 	go.opentelemetry.io/collector/pdata v1.54.0
 	go.opentelemetry.io/contrib/config v0.10.0
 	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0
 	go.opentelemetry.io/otel v1.45.0
 	go.opentelemetry.io/otel/metric v1.45.0
 	go.opentelemetry.io/otel/sdk v1.45.0
 	go.opentelemetry.io/otel/trace v1.45.0
 	go.uber.org/multierr v1.11.0
-	go.uber.org/zap v1.28.0
+	go.uber.org/zap v1.27.1
 	golang.org/x/crypto v0.55.0
 	golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90
 	golang.org/x/net v0.58.0
@@ -173,11 +173,11 @@ require (
 	cloud.google.com/go/auth v0.18.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
-	github.com/ClickHouse/ch-go v0.73.0
+	github.com/ClickHouse/ch-go v0.71.0
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Yiling-J/theine-go v0.6.2 // indirect
 	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
-	github.com/andybalholm/brotli v1.2.1 // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beevik/etree v1.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -190,7 +190,7 @@ require (
 	github.com/dennwc/varint v1.0.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/ebitengine/purego v0.10.1 // indirect
+	github.com/ebitengine/purego v0.9.1 // indirect
 	github.com/edsrzf/mmap-go v1.2.0 // indirect
 	github.com/elastic/lunes v0.2.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
@@ -233,7 +233,7 @@ require (
 	github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.7 // indirect
-	github.com/hashicorp/go-version v1.9.0 // indirect
+	github.com/hashicorp/go-version v1.8.0 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/hashicorp/memberlist v0.5.4 // indirect
@@ -254,7 +254,7 @@ require (
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/leodido/go-syslog/v4 v4.3.0 // indirect
 	github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
-	github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
+	github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
 	github.com/magefile/mage v1.15.0 // indirect
 	github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@@ -275,9 +275,9 @@ require (
 	github.com/open-feature/go-sdk v1.17.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.144.0 // indirect
 	github.com/openfga/openfga v1.14.1
-	github.com/paulmach/orb v0.13.0 // indirect
+	github.com/paulmach/orb v0.12.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
-	github.com/pierrec/lz4/v4 v4.1.27 // indirect
+	github.com/pierrec/lz4/v4 v4.1.26 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/pressly/goose/v3 v3.27.0 // indirect
@@ -293,7 +293,7 @@ require (
 	github.com/segmentio/asm v1.2.1 // indirect
 	github.com/segmentio/backo-go v1.0.1 // indirect
 	github.com/sethvargo/go-retry v0.3.0 // indirect
-	github.com/shirou/gopsutil/v4 v4.26.5 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.12 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 // indirect
@@ -308,8 +308,8 @@ require (
 	github.com/swaggest/openapi-go v0.2.60
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
-	github.com/tklauser/go-sysconf v0.4.0 // indirect
-	github.com/tklauser/numcpus v0.12.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.16 // indirect
+	github.com/tklauser/numcpus v0.11.0 // indirect
 	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
 	github.com/valyala/fastjson v1.6.7 // indirect
 	github.com/vjeantet/grok v1.0.1 // indirect

go.sum

@@ -79,10 +79,10 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgv
 github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/ClickHouse/ch-go v0.73.0 h1:jsHiGRbQ3sz+gekvDFJF29LWDo5dzbJm5s1h8TWVP2M=
-github.com/ClickHouse/ch-go v0.73.0/go.mod h1:wkFIxrqlXeRJ9cn3r5Fz5Qen9jl5aTMPuGZeuJpANNY=
-github.com/ClickHouse/clickhouse-go/v2 v2.47.0 h1:ZDAzrnKSOPTIsm4tdUNfrii2yc8dk4SVRLC77BR7Z5Q=
-github.com/ClickHouse/clickhouse-go/v2 v2.47.0/go.mod h1:sPj7C7UYQ2MWHcfX+4eGN6nwnCqwUKfgO6PcwKpd6K8=
+github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM=
+github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw=
+github.com/ClickHouse/clickhouse-go/v2 v2.44.0 h1:9pxs5pRwIvhni5BDRPn/n5A8DeUod5TnBaeulFBX8EQ=
+github.com/ClickHouse/clickhouse-go/v2 v2.44.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c=
 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
 github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
@@ -94,8 +94,8 @@ github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/SigNoz/clickhouse-go-mock v0.15.1 h1:ifoh/3Jx6+Zt4ptuSeUnZrOW5tVy/WSgzGpAh+w4NaY=
-github.com/SigNoz/clickhouse-go-mock v0.15.1/go.mod h1:r1qiBP8tA1KG7bmuCg0wrt0YxBMWylL7FPTBbx9wTTs=
+github.com/SigNoz/clickhouse-go-mock v0.14.0 h1:yYoGMJk0UDVvLCk12dover/zoRIuznW3BfoqUhMoJSY=
+github.com/SigNoz/clickhouse-go-mock v0.14.0/go.mod h1:tXmSXVLWz7N/yVLCianszmNIN4cHozXvwEXCOLzzqzI=
 github.com/SigNoz/expr v1.17.7-beta h1:FyZkleM5dTQ0O6muQfwGpoH5A2ohmN/XTasRCO72gAA=
 github.com/SigNoz/expr v1.17.7-beta/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkbj57eGXx8H3ZJ4zhmQXBnrW523ktj8=
@@ -116,8 +116,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
 github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
-github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
-github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/antchfx/xmlquery v1.5.0 h1:uAi+mO40ZWfyU6mlUBxRVvL6uBNZ6LMU4M3+mQIBV4c=
 github.com/antchfx/xmlquery v1.5.0/go.mod h1:lJfWRXzYMK1ss32zm1GQV3gMIW/HFey3xDZmkP1SuNc=
 github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI=
@@ -255,15 +255,15 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c=
-github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q=
+github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
+github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
-github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
+github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
+github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84=
 github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U=
@@ -596,8 +596,8 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA=
-github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
+github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
@@ -676,6 +676,7 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
 github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
@@ -711,8 +712,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b h1:11UHH39z1RhZ5dc4y4r/4koJo6IYFgTRMe/LlwRTEw0=
 github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg=
-github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak=
-github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
+github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
+github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
 github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
 github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
@@ -767,10 +768,10 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
-github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc=
-github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
-github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc=
-github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s=
+github.com/moby/moby/api v1.53.0 h1:PihqG1ncw4W+8mZs69jlwGXdaYBeb5brF6BL7mPIS/w=
+github.com/moby/moby/api v1.53.0/go.mod h1:8mb+ReTlisw4pS6BRzCMts5M49W5M7bKt1cJy/YbAqc=
+github.com/moby/moby/client v0.2.2 h1:Pt4hRMCAIlyjL3cr8M5TrXCwKzguebPAc2do2ur7dEM=
+github.com/moby/moby/client v0.2.2/go.mod h1:2EkIPVNCqR05CMIzL1mfA07t0HvVUUOl85pasRz/GmQ=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -779,6 +780,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -840,8 +842,9 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw=
-github.com/paulmach/orb v0.13.0/go.mod h1:6scRWINywA2Jf05dcjOfLfxrUIMECvTSG2MVbRLxu/k=
+github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s=
+github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
 github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
@@ -852,8 +855,8 @@ github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8
 github.com/perses/spec v0.1.2 h1:yGoygcR3ZusuGDCmRMwsVXCMvMwi1qZndKV6NYNreEw=
 github.com/perses/spec v0.1.2/go.mod h1:NoGI5jmGwRdkdPgyYSZJTBL4/Py+dqIPKS2QV8NOvGE=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=
-github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
+github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -962,8 +965,8 @@ github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc
 github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE=
 github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
 github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
-github.com/shirou/gopsutil/v4 v4.26.5 h1:RPcBXkpz7kOj9PqGFQOlBPZHsyaPvPVQc098y9RmCNM=
-github.com/shirou/gopsutil/v4 v4.26.5/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
+github.com/shirou/gopsutil/v4 v4.25.12 h1:e7PvW/0RmJ8p8vPGJH4jvNkOyLmbkXgXW4m6ZPic6CY=
+github.com/shirou/gopsutil/v4 v4.25.12/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU=
 github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
 github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c h1:aqg5Vm5dwtvL+YgDpBcK1ITf3o96N/K7/wsRXQnUTEs=
@@ -1041,13 +1044,14 @@ github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
 github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
-github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
-github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
-github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
+github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
+github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
+github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
 github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
 github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -1089,8 +1093,12 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA=
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M=
@@ -1112,6 +1120,7 @@ go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dY
 go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU=
 go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -1250,8 +1259,8 @@ go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63
 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0/go.mod h1:34csimR1lUhdT5HH4Rii9aKPrvBcnFRwxLwcevsU+Kk=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.67.0 h1:c9r/G1CSw4dPI1jaNNG9RnQP+q4SvZnHciDQJVIvchU=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.67.0/go.mod h1:gO9smoZe9KnZcJCqcB0lMmQ4Z5VEifYmjMTpnwtTSuQ=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo=
 go.opentelemetry.io/contrib/otelconf v0.18.0 h1:ciF2Gf00BWs0DnexKFZXcxg9kJ8r3SUW1LOzW3CsKA8=
 go.opentelemetry.io/contrib/otelconf v0.18.0/go.mod h1:FcP7k+JLwBLdOxS6qY6VQ/4b5VBntI6L6o80IMwhAeI=
 go.opentelemetry.io/contrib/propagators/b3 v1.39.0 h1:PI7pt9pkSnimWcp5sQhUA9OzLbc3Ba4sL+VEUTNsxrk=
@@ -1323,8 +1332,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
-go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
-go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
+go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
+go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
@@ -1341,6 +1350,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
 golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

pkg/alertmanager/alertmanagernotify/email/email.go

@@ -59,7 +59,7 @@ type Email struct {
 
 // layoutData is the value passed to the email.signoz.html layout
 // template. It embeds NotificationTemplateData so templates can reference
-// `.Alert.Status`, `.Alert.TotalFiring`, `.Alert.TotalResolved`,
+// `.Alert.Status`, `.Alert.TotalFiring`, `.Alert.TotalRecovering`, `.Alert.TotalResolved`,
 // `.NotificationTemplateData.ExternalURL`, etc. alongside the rendered
 // Title and per-alert Bodies.
 type layoutData struct {

pkg/alertmanager/alertmanagernotify/slack/slack_test.go

@@ -389,7 +389,7 @@ func TestPrepareContent(t *testing.T) {
 | Metric | Value |
 |--------|-------|
 | **Current** | *$value* |
-| **Threshold** | *$threshold.value* |
+| **Threshold** | *$threshold* |
 
 **Status:** $alert.status | **Severity:** $labels.severity`
 		titleTemplate := "[$alert.status] $rule.name — $labels.service"
@@ -401,10 +401,10 @@ func TestPrepareContent(t *testing.T) {
 				StartsAt: time.Now(),
 				EndsAt:   time.Now().Add(time.Hour),
 				Annotations: model.LabelSet{
-					ruletypes.AnnotationTitleTemplate: model.LabelValue(titleTemplate),
-					ruletypes.AnnotationBodyTemplate:  model.LabelValue(bodyTemplate),
-					"value":                           "100",
-					"threshold.value":                 "200",
+					ruletypes.AnnotationTitleTemplate:  model.LabelValue(titleTemplate),
+					ruletypes.AnnotationBodyTemplate:   model.LabelValue(bodyTemplate),
+					ruletypes.AnnotationValue:          "100",
+					ruletypes.AnnotationThresholdValue: "200",
 				},
 			},
 		}
@@ -414,10 +414,10 @@ func TestPrepareContent(t *testing.T) {
 				StartsAt: time.Now().Add(-2 * time.Hour),
 				EndsAt:   time.Now().Add(-time.Hour),
 				Annotations: model.LabelSet{
-					ruletypes.AnnotationTitleTemplate: model.LabelValue(titleTemplate),
-					ruletypes.AnnotationBodyTemplate:  model.LabelValue(bodyTemplate),
-					"value":                           "50",
-					"threshold.value":                 "200",
+					ruletypes.AnnotationTitleTemplate:  model.LabelValue(titleTemplate),
+					ruletypes.AnnotationBodyTemplate:   model.LabelValue(bodyTemplate),
+					ruletypes.AnnotationValue:          "50",
+					ruletypes.AnnotationThresholdValue: "200",
 				},
 			},
 		}

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate.go

@@ -205,16 +205,29 @@ func (at *templater) buildNotificationTemplateData(
 		alertDataSlice = append(alertDataSlice, ad)
 	}
 
-	// count the number of firing and resolved alerts
-	var firing, resolved int
+	// count the number of firing, recovering and resolved alerts
+	var firing, recovering, resolved int
+	var isTest bool
 	for _, ad := range alertDataSlice {
-		if ad.Alert.IsFiring {
+		isTest = isTest || ad.Alert.IsTest
+		switch {
+		case ad.Alert.IsFiring:
 			firing++
-		} else if ad.Alert.IsResolved {
+		case ad.Alert.IsRecovering:
+			recovering++
+		case ad.Alert.IsResolved:
 			resolved++
 		}
 	}
 
+	// firing wins over recovering, which wins over resolved
+	status := string(model.AlertResolved)
+	if firing > 0 {
+		status = string(model.AlertFiring)
+	} else if recovering > 0 {
+		status = ruletypes.StateRecovering.StringValue()
+	}
+
 	// build the group labels
 	gl := make(template.KV, len(groupLabels))
 	for k, v := range groupLabels {
@@ -224,10 +237,12 @@ func (at *templater) buildNotificationTemplateData(
 	// build the notification template data
 	return &alertmanagertypes.NotificationTemplateData{
 		Alert: alertmanagertypes.NotificationAlert{
-			Receiver:      receiver,
-			Status:        string(types.Alerts(alerts...).Status()),
-			TotalFiring:   firing,
-			TotalResolved: resolved,
+			Receiver:        receiver,
+			Status:          status,
+			TotalFiring:     firing,
+			TotalRecovering: recovering,
+			TotalResolved:   resolved,
+			IsTest:          isTest,
 		},
 		Rule:              buildRuleInfo(commonLabels, rawCommonAnnotations),
 		GroupLabels:       gl,
@@ -252,27 +267,39 @@ func buildAlertData(a *types.Alert, receiver string) alertmanagertypes.AlertData
 		annotations[string(k)] = string(v)
 	}
 
+	// The alertmanager only knows firing and resolved; the rule manager flags a
+	// firing alert whose value is between the threshold and the recovery
+	// threshold.
+	status := string(a.Status())
+	if a.Status() == model.AlertFiring && annotations[ruletypes.AnnotationIsRecovering] == "true" {
+		status = ruletypes.StateRecovering.StringValue()
+	}
+
 	return alertmanagertypes.AlertData{
 		Alert: alertmanagertypes.AlertInfo{
-			Status:        string(a.Status()),
+			Status:        status,
 			Receiver:      receiver,
 			Value:         annotations[ruletypes.AnnotationValue],
 			StartsAt:      a.StartsAt,
 			EndsAt:        a.EndsAt,
 			GeneratorURL:  a.GeneratorURL,
 			Fingerprint:   a.Fingerprint().String(),
-			IsFiring:      a.Status() == model.AlertFiring,
-			IsResolved:    a.Status() == model.AlertResolved,
+			IsFiring:      status == string(model.AlertFiring),
+			IsResolved:    status == string(model.AlertResolved),
 			IsMissingData: labels[ruletypes.LabelNoData] == "true",
-			IsRecovering:  labels[ruletypes.LabelIsRecovering] == "true",
+			IsRecovering:  status == ruletypes.StateRecovering.StringValue(),
+			IsTest:        labels[ruletypes.LabelTestAlert] == "true",
 		},
-		Rule:   buildRuleInfo(labels, annotations),
-		Log:    alertmanagertypes.LinkInfo{URL: annotations[ruletypes.AnnotationRelatedLogs]},
-		Trace:  alertmanagertypes.LinkInfo{URL: annotations[ruletypes.AnnotationRelatedTraces]},
-		Labels: labels,
+		Rule:      buildRuleInfo(labels, annotations),
+		Log:       alertmanagertypes.LinkInfo{URL: annotations[ruletypes.AnnotationRelatedLogs]},
+		Trace:     alertmanagertypes.LinkInfo{URL: annotations[ruletypes.AnnotationRelatedTraces]},
+		Exception: alertmanagertypes.LinkInfo{URL: annotations[ruletypes.AnnotationRelatedExceptions]},
+		Labels:    labels,
 		// Strip private annotations once the structured fields above have
 		// been populated from the raw map.
 		Annotations: alertmanagertypes.FilterPublicAnnotations(annotations),
+		Value:       annotations[ruletypes.AnnotationValue],
+		Threshold:   annotations[ruletypes.AnnotationThresholdValue],
 	}
 }
 
@@ -284,10 +311,12 @@ func buildRuleInfo(labels, annotations template.KV) alertmanagertypes.RuleInfo {
 		ID:        labels[ruletypes.LabelRuleID],
 		URL:       labels[ruletypes.LabelRuleSource],
 		Severity:  labels[ruletypes.LabelSeverityName],
+		Type:      annotations[ruletypes.AnnotationRuleType],
 		MatchType: annotations[ruletypes.AnnotationMatchType],
 		Threshold: alertmanagertypes.Threshold{
-			Value: annotations[ruletypes.AnnotationThresholdValue],
-			Op:    annotations[ruletypes.AnnotationCompareOp],
+			Value:         annotations[ruletypes.AnnotationThresholdValue],
+			Op:            annotations[ruletypes.AnnotationCompareOp],
+			RecoveryValue: annotations[ruletypes.AnnotationRecoveryThreshold],
 		},
 	}
 }

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/alertmanager/notify"
@@ -247,6 +248,178 @@ Description: Request rate exceeded 10k/s`},
 			wantBody:          []string{"Runbook: https://runbook.example.com"},
 			wantIsDefaultBody: true,
 		},
+		{
+			name: "value annotation feeds alert.value and is hidden from annotations",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors"},
+					map[string]string{"summary": "errors are high", ruletypes.AnnotationValue: "523"},
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$rule.name",
+				BodyTemplate:  "Value: $alert.value, annotations: {{ len $annotations }}",
+			},
+			wantTitle: "HighErrors",
+			wantBody:  []string{"Value: 523, annotations: 1"},
+		},
+		{
+			name: "value shortcut resolves bare and in braces and wins over a label named value",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors", "value": "from-label"},
+					map[string]string{ruletypes.AnnotationValue: "523"},
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$rule.name",
+				BodyTemplate:  "$value | {{ $value }} | {{ if $value }}set{{ end }} | $labels.value",
+			},
+			wantTitle: "HighErrors",
+			wantBody:  []string{"523 | 523 | set | from-label"},
+		},
+		{
+			name: "threshold shortcut and rule.threshold.value read the threshold annotation and win over a label named threshold",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors", "threshold": "from-label"},
+					map[string]string{ruletypes.AnnotationThresholdValue: "500"},
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$rule.name above $rule.threshold.value",
+				BodyTemplate:  "$threshold | {{ $threshold }} | $rule.threshold.value | $labels.threshold | annotations: {{ len $annotations }}",
+			},
+			wantTitle: "HighErrors above 500",
+			wantBody:  []string{"500 | 500 | 500 | from-label | annotations: 0"},
+		},
+		{
+			name: "rule threshold op and recovery value read their private annotations",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors"},
+					map[string]string{ruletypes.AnnotationThresholdValue: "500", ruletypes.AnnotationCompareOp: "above", ruletypes.AnnotationRecoveryThreshold: "400"},
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$rule.name $rule.threshold.op $rule.threshold.value",
+				BodyTemplate:  "$rule.threshold.op $rule.threshold.value (recovers at $rule.threshold.recovery_value), annotations: {{ len $annotations }}",
+			},
+			wantTitle: "HighErrors above 500",
+			wantBody:  []string{"above 500 (recovers at 400), annotations: 0"},
+		},
+		{
+			name: "rule type and match type read their private annotations",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors", "type": "from-label"},
+					map[string]string{ruletypes.AnnotationRuleType: "logs", ruletypes.AnnotationMatchType: "on_average"},
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$rule.type: $rule.name",
+				BodyTemplate:  "$rule.type $rule.match_type | $type | annotations: {{ len $annotations }}",
+			},
+			wantTitle: "logs: HighErrors",
+			wantBody:  []string{"logs on_average | from-label | annotations: 0"},
+		},
+		{
+			name: "test label sets is_test on the title and each alert",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors", ruletypes.LabelTestAlert: "true"},
+					nil,
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "{{ if $alert.is_test }}[TEST] {{ end }}$rule.name",
+				BodyTemplate:  "is_test=$alert.is_test",
+			},
+			wantTitle: "[TEST] HighErrors",
+			wantBody:  []string{"is_test=true"},
+		},
+		{
+			name: "alert without test label is not a test",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighErrors"}, nil, true),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "{{ if $alert.is_test }}[TEST] {{ end }}$rule.name",
+				BodyTemplate:  "is_test=$alert.is_test",
+			},
+			wantTitle: "HighErrors",
+			wantBody:  []string{"is_test=false"},
+		},
+		{
+			name: "exception url reads its private annotation",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "CheckoutExceptions"},
+					map[string]string{ruletypes.AnnotationRelatedExceptions: "https://signoz.example/exceptions?serviceName=checkout"},
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$rule.name",
+				BodyTemplate:  "[View exceptions]($exception.url), annotations: {{ len $annotations }}",
+			},
+			wantTitle: "CheckoutExceptions",
+			wantBody:  []string{"[View exceptions](https://signoz.example/exceptions?serviceName=checkout), annotations: 0"},
+		},
+		{
+			name: "recovering annotation sets status, flags and counts with firing winning the title",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors", "team": "core"},
+					map[string]string{ruletypes.AnnotationIsRecovering: "true"},
+					true,
+				),
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors", "team": "edge"},
+					nil,
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$alert.status: $alert.total_firing firing, $alert.total_recovering recovering, $alert.total_resolved resolved",
+				BodyTemplate:  "$labels.team: $alert.status firing=$alert.is_firing recovering=$alert.is_recovering resolved=$alert.is_resolved, annotations: {{ len $annotations }}",
+			},
+			wantTitle: "firing: 1 firing, 1 recovering, 0 resolved",
+			wantBody: []string{
+				"core: recovering firing=false recovering=true resolved=false, annotations: 0",
+				"edge: firing firing=true recovering=false resolved=false, annotations: 0",
+			},
+		},
+		{
+			name: "recovering wins the title over resolved and a resolved alert ignores a stale recovering flag",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors", "team": "core"},
+					map[string]string{ruletypes.AnnotationIsRecovering: "true"},
+					true,
+				),
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "HighErrors", "team": "edge"},
+					map[string]string{ruletypes.AnnotationIsRecovering: "true"},
+					false,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$alert.status: $alert.total_firing firing, $alert.total_recovering recovering, $alert.total_resolved resolved",
+				BodyTemplate:  "$labels.team: $alert.status recovering=$alert.is_recovering",
+			},
+			wantTitle: "recovering: 0 firing, 1 recovering, 1 resolved",
+			wantBody: []string{
+				"core: recovering recovering=true",
+				"edge: resolved recovering=false",
+			},
+		},
 		{
 			name: "using non-existing function in template",
 			alerts: []*types.Alert{
@@ -285,3 +458,61 @@ Description: Request rate exceeded 10k/s`},
 		})
 	}
 }
+
+func TestExpandTemplateFunctions(t *testing.T) {
+	at, ctx := testSetup(t)
+	startsAt := time.Date(2026, time.November, 12, 5, 30, 0, 0, time.UTC)
+	alert := &types.Alert{Alert: model.Alert{
+		Labels:      model.LabelSet{ruletypes.LabelAlertName: "HighLatency", ruletypes.LabelSeverityName: "critical", "env": "prod-eu", "host": "web-1.example.com"},
+		Annotations: model.LabelSet{"summary": "  p99 is high  "},
+		StartsAt:    startsAt,
+		EndsAt:      startsAt.Add(time.Hour),
+	}}
+
+	testCases := []struct {
+		name     string
+		body     string
+		wantBody string
+	}{
+		{name: "ToUpper", body: "{{ $env | toUpper }}", wantBody: "PROD-EU"},
+		{name: "ToLower", body: "{{ $env | toLower }}", wantBody: "prod-eu"},
+		{name: "Title", body: "{{ $rule.severity | title }}", wantBody: "Critical"},
+		{name: "TrimSpace", body: "[{{ $annotations.summary | trimSpace }}]", wantBody: "[p99 is high]"},
+		{name: "Match", body: `{{ if $env | match "^prod" }}production{{ end }}`, wantBody: "production"},
+		{name: "ReReplaceAll", body: `{{ $host | reReplaceAll "^([^.]+)\\..*" "$1" }}`, wantBody: "web-1"},
+		{name: "StringSlice_Join", body: `{{ stringSlice $env $host | join ", " }}`, wantBody: "prod-eu, web-1.example.com"},
+		{name: "Urlescape", body: `q={{ "a b&c" | urlescape }}`, wantBody: "q=a+b%26c"},
+		{name: "Date_AlertStartsAt", body: `{{ $alert.starts_at | date "Jan 2, 15:04 MST" }}`, wantBody: "Nov 12, 05:30 UTC"},
+		{name: "Tz_Date_AlertStartsAt", body: `{{ $alert.starts_at | tz "Asia/Kolkata" | date "15:04 MST" }}`, wantBody: "11:00 IST"},
+		{name: "Date_AlertEndsAt", body: `{{ $alert.ends_at | date "15:04" }}`, wantBody: "06:30"},
+		{name: "HumanizeDuration_Seconds", body: "{{ humanizeDuration 3725 }}", wantBody: "1h 2m 5s"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			result, err := at.Expand(ctx, alertmanagertypes.ExpandRequest{TitleTemplate: "$rule.name", BodyTemplate: testCase.body}, []*types.Alert{alert})
+			require.NoError(t, err)
+			require.Len(t, result.Body, 1)
+			assert.Equal(t, testCase.wantBody, result.Body[0])
+		})
+	}
+
+	t.Run("Now_Tz_Date", func(t *testing.T) {
+		result, err := at.Expand(ctx, alertmanagertypes.ExpandRequest{TitleTemplate: "$rule.name", BodyTemplate: `{{ now | tz "Asia/Kolkata" | date "2006 MST" }}`}, []*types.Alert{alert})
+		require.NoError(t, err)
+		require.Len(t, result.Body, 1)
+		assert.Equal(t, time.Now().In(time.FixedZone("IST", 5*60*60+30*60)).Format("2006 MST"), result.Body[0])
+	})
+
+	t.Run("Since_HumanizeDuration_AlertStartsAt", func(t *testing.T) {
+		recent := &types.Alert{Alert: model.Alert{
+			Labels:   model.LabelSet{ruletypes.LabelAlertName: "HighLatency"},
+			StartsAt: time.Now().Add(-time.Hour),
+			EndsAt:   time.Now().Add(time.Hour),
+		}}
+		result, err := at.Expand(ctx, alertmanagertypes.ExpandRequest{TitleTemplate: "$rule.name", BodyTemplate: "{{ $alert.starts_at | since | humanizeDuration }}"}, []*types.Alert{recent})
+		require.NoError(t, err)
+		require.Len(t, result.Body, 1)
+		assert.Regexp(t, `^1h 0m \d+s$`, result.Body[0])
+	})
+}

pkg/alertmanager/alertmanagertemplate/preprocessor.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"time"
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
-	"github.com/go-viper/mapstructure/v2"
 )
 
 // fieldPath is a dotted mapstructure path into the templating data map,
@@ -131,16 +131,51 @@ func structRootSet(data any) map[string]bool {
 	return roots
 }
 
+// toTemplateMap converts a tagged struct into nested maps keyed by its
+// mapstructure tag names, the names templates reference. Unlike
+// mapstructure.Decode it keeps time.Time values as is, so date, tz and since
+// work on them. Fields that are untagged or tagged "-" are skipped.
+func toTemplateMap(val reflect.Value) map[string]any {
+	typ := val.Type()
+	result := make(map[string]any, typ.NumField())
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		if !field.IsExported() {
+			continue
+		}
+		name := strings.Split(field.Tag.Get("mapstructure"), ",")[0]
+		if name == "" || name == "-" {
+			continue
+		}
+
+		fv := val.Field(i)
+		if fv.Kind() == reflect.Pointer {
+			if fv.IsNil() {
+				result[name] = nil
+				continue
+			}
+			fv = fv.Elem()
+		}
+		if fv.Kind() == reflect.Struct && fv.Type() != reflect.TypeOf(time.Time{}) {
+			result[name] = toTemplateMap(fv)
+			continue
+		}
+		result[name] = fv.Interface()
+	}
+	return result
+}
+
 // buildDataMap converts the typed data struct to the map[string]any that the
 // template engine indexes into. Each label and annotation is additionally
 // exposed at the root under its raw key, so $service.name resolves a flat
 // OTel-style label as a single-key index on the root. Struct-path keys
 // already present at the root take precedence on collisions.
 func buildDataMap(data any) (map[string]any, error) {
-	var result map[string]any
-	if err := mapstructure.Decode(data, &result); err != nil {
-		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to build template data map")
+	val := reflect.Indirect(reflect.ValueOf(data))
+	if val.Kind() != reflect.Struct {
+		return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to build template data map: expected a struct, got %T", data)
 	}
+	result := toTemplateMap(val)
 
 	flatten := func(labels, annotations map[string]string) {
 		for k, v := range labels {

pkg/alertmanager/alertmanagertemplate/preprocessor_test.go

@@ -229,10 +229,12 @@ func TestPreProcessTemplateAndData(t *testing.T) {
 			},
 			expectedData: map[string]any{
 				"alert": map[string]any{
-					"receiver":       "pagerduty",
-					"status":         "firing",
-					"total_firing":   3,
-					"total_resolved": 0,
+					"receiver":         "pagerduty",
+					"status":           "firing",
+					"total_firing":     3,
+					"total_recovering": 0,
+					"total_resolved":   0,
+					"is_test":          false,
 				},
 				"severity":            "critical",
 				"http.request.method": "GET",

pkg/apiserver/signozapiserver/session.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/http/handler"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/types/coretypes"
 	"github.com/gorilla/mux"
 )
 
@@ -23,12 +22,7 @@ func (provider *provider) addSessionRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusNotFound},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceSession,
-		Verb:     coretypes.VerbCreate,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodPost).GetError(); err != nil {
+	})).Methods(http.MethodPost).GetError(); err != nil {
 		return err
 	}
 
@@ -45,12 +39,7 @@ func (provider *provider) addSessionRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceSession,
-		Verb:     coretypes.VerbRead,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodGet).GetError(); err != nil {
+	})).Methods(http.MethodGet).GetError(); err != nil {
 		return err
 	}
 
@@ -67,12 +56,7 @@ func (provider *provider) addSessionRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceSession,
-		Verb:     coretypes.VerbUpdate,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodPost).GetError(); err != nil {
+	})).Methods(http.MethodPost).GetError(); err != nil {
 		return err
 	}
 
@@ -89,12 +73,7 @@ func (provider *provider) addSessionRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{{Name: authtypes.IdentNProviderTokenizer.StringValue()}},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceSession,
-		Verb:     coretypes.VerbDelete,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodDelete).GetError(); err != nil {
+	})).Methods(http.MethodDelete).GetError(); err != nil {
 		return err
 	}
 
@@ -111,12 +90,7 @@ func (provider *provider) addSessionRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusNotFound},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceSession,
-		Verb:     coretypes.VerbCreate,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodGet).GetError(); err != nil {
+	})).Methods(http.MethodGet).GetError(); err != nil {
 		return err
 	}
 
@@ -136,12 +110,7 @@ func (provider *provider) addSessionRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusNotFound, http.StatusUnavailableForLegalReasons},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceSession,
-		Verb:     coretypes.VerbCreate,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodPost).GetError(); err != nil {
+	})).Methods(http.MethodPost).GetError(); err != nil {
 		return err
 	}
 
@@ -158,12 +127,7 @@ func (provider *provider) addSessionRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusNotFound, http.StatusUnavailableForLegalReasons},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceSession,
-		Verb:     coretypes.VerbCreate,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodGet).GetError(); err != nil {
+	})).Methods(http.MethodGet).GetError(); err != nil {
 		return err
 	}
 

pkg/apiserver/signozapiserver/user.go

@@ -68,13 +68,7 @@ func (provider *provider) addUserRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{{Name: authtypes.IdentNProviderTokenizer.StringValue()}},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceUser,
-		Verb:     coretypes.VerbRead,
-		Category: coretypes.ActionCategoryAccessControl,
-		ID:       provider.userIDExtractor(),
-		Selector: coretypes.IDSelector,
-	}))).Methods(http.MethodGet).GetError(); err != nil {
+	})).Methods(http.MethodGet).GetError(); err != nil {
 		return err
 	}
 
@@ -134,13 +128,7 @@ func (provider *provider) addUserRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{{Name: authtypes.IdentNProviderTokenizer.StringValue()}},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceUser,
-		Verb:     coretypes.VerbUpdate,
-		Category: coretypes.ActionCategoryAccessControl,
-		ID:       provider.userIDExtractor(),
-		Selector: coretypes.IDSelector,
-	}))).Methods(http.MethodPut).GetError(); err != nil {
+	})).Methods(http.MethodPut).GetError(); err != nil {
 		return err
 	}
 
@@ -306,12 +294,7 @@ func (provider *provider) addUserRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusNotFound},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceFactorPassword,
-		Verb:     coretypes.VerbRead,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodPost).GetError(); err != nil {
+	})).Methods(http.MethodPost).GetError(); err != nil {
 		return err
 	}
 
@@ -328,12 +311,7 @@ func (provider *provider) addUserRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusNotFound},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{{Name: authtypes.IdentNProviderTokenizer.StringValue()}},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceFactorPassword,
-		Verb:     coretypes.VerbUpdate,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodPut).GetError(); err != nil {
+	})).Methods(http.MethodPut).GetError(); err != nil {
 		return err
 	}
 
@@ -350,12 +328,7 @@ func (provider *provider) addUserRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusUnprocessableEntity},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceFactorPassword,
-		Verb:     coretypes.VerbCreate,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodPost).GetError(); err != nil {
+	})).Methods(http.MethodPost).GetError(); err != nil {
 		return err
 	}
 
@@ -372,12 +345,7 @@ func (provider *provider) addUserRoutes(router *mux.Router) error {
 		ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusNotFound},
 		Deprecated:          false,
 		SecuritySchemes:     []handler.OpenAPISecurityScheme{},
-	}, handler.WithResourceDefs(handler.BasicResourceDef{
-		Resource: coretypes.ResourceMetaResourceFactorPassword,
-		Verb:     coretypes.VerbUpdate,
-		Category: coretypes.ActionCategoryAccessControl,
-		Selector: coretypes.WildcardSelector,
-	}))).Methods(http.MethodPost).GetError(); err != nil {
+	})).Methods(http.MethodPost).GetError(); err != nil {
 		return err
 	}
 
@@ -525,21 +493,6 @@ func (provider *provider) addUserRoutes(router *mux.Router) error {
 	return nil
 }
 
-func (provider *provider) userIDExtractor() coretypes.ResourceIDExtractor {
-	return coretypes.NewResourceIDExtractor(coretypes.PhaseRequest, func(ec coretypes.ExtractorContext) (string, error) {
-		if ec.Request == nil {
-			return "", nil
-		}
-
-		claims, err := authtypes.ClaimsFromContext(ec.Request.Context())
-		if err != nil {
-			return "", err
-		}
-
-		return claims.UserID, nil
-	})
-}
-
 func (provider *provider) userRoleUserIDExtractor() coretypes.ResourceIDExtractor {
 	return coretypes.NewResourceIDExtractor(coretypes.PhaseRequest, func(ec coretypes.ExtractorContext) (string, error) {
 		if ec.Request == nil {

pkg/contextlinks/links.go

@@ -1,6 +1,7 @@
 package contextlinks
 
 import (
+	"cmp"
 	"encoding/json"
 	"net/url"
 	"slices"
@@ -26,6 +27,25 @@ func PrepareParamsForLogsV5(start, end time.Time, whereClause string) url.Values
 	return prepareExplorerParams("logs", qbtypes.QueryTypeBuilder, start.UnixMilli(), end.UnixMilli(), whereClause)
 }
 
+// PrepareParamsForExceptions returns the exceptions page query params for the
+// given range, filtered to the service and exception type found in labels
+// (serviceName and exceptionType from SQL, or their builder keys). The page
+// writes its time params in milliseconds.
+// It doesn't reuse prepareExplorerParams: the exceptions page reads only the
+// older filters.items from compositeQuery, not filter.expression.
+func PrepareParamsForExceptions(start, end time.Time, labels map[string]string) url.Values {
+	params := url.Values{}
+	params.Set("startTime", strconv.FormatInt(start.UnixMilli(), 10))
+	params.Set("endTime", strconv.FormatInt(end.UnixMilli(), 10))
+	if serviceName := cmp.Or(labels["serviceName"], labels["service.name"]); serviceName != "" {
+		params.Set("serviceName", serviceName)
+	}
+	if exceptionType := cmp.Or(labels["exceptionType"], labels["exception.type"]); exceptionType != "" {
+		params.Set("exceptionType", exceptionType)
+	}
+	return params
+}
+
 // The end link is double encoded because otherwise a filter expression with `%` somewhere in it breaks.
 func prepareExplorerParams(dataSource string, queryType qbtypes.QueryType, start, end int64, whereClause string) url.Values {
 	// builder_query is the explorer default, so it is left out to keep existing links unchanged

pkg/contextlinks/links_test.go

@@ -2,6 +2,7 @@ package contextlinks
 
 import (
 	"testing"
+	"time"
 
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/ruletypes"
@@ -222,3 +223,30 @@ func TestPrepareRuleFilterExpression(t *testing.T) {
 		})
 	}
 }
+
+func TestPrepareParamsForExceptions(t *testing.T) {
+	start := time.UnixMilli(1705468620000)
+	end := time.UnixMilli(1705468920000)
+
+	testCases := []struct {
+		name          string
+		labels        map[string]string
+		serviceName   string
+		exceptionType string
+	}{
+		{name: "SQLColumns_SetsServiceAndExceptionType", labels: map[string]string{"serviceName": "checkout", "exceptionType": "TimeoutError"}, serviceName: "checkout", exceptionType: "TimeoutError"},
+		{name: "BuilderKeys_SetsServiceAndExceptionType", labels: map[string]string{"service.name": "payments", "exception.type": "ValueError"}, serviceName: "payments", exceptionType: "ValueError"},
+		{name: "NoMatchingLabels_SetsTimeOnly", labels: map[string]string{"host": "web-1"}},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			params := PrepareParamsForExceptions(start, end, testCase.labels)
+
+			assert.Equal(t, "1705468620000", params.Get("startTime"))
+			assert.Equal(t, "1705468920000", params.Get("endTime"))
+			assert.Equal(t, testCase.serviceName, params.Get("serviceName"))
+			assert.Equal(t, testCase.exceptionType, params.Get("exceptionType"))
+		})
+	}
+}

pkg/querier/builder_query.go

@@ -457,13 +457,6 @@ func emptyResultFor(kind qbtypes.RequestType, queryName string) *qbtypes.Result
 
 // executeWithContext executes the query with query window and step context for partial value detection.
 func (q *builderQuery[T]) executeWithContext(ctx context.Context, query string, args []any) (*qbtypes.Result, error) {
-	result, _, err := q.executeSkippingRaw(ctx, query, args, 0)
-	return result, err
-}
-
-// executeSkippingRaw is executeWithContext for a raw query that drops its first
-// skip rows undecoded. It returns how many rows were skipped.
-func (q *builderQuery[T]) executeSkippingRaw(ctx context.Context, query string, args []any, skip int) (*qbtypes.Result, int, error) {
 	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
 		instrumentationtypes.TelemetrySignal: q.spec.Signal.StringValue(),
 		instrumentationtypes.QueryDuration:   instrumentationtypes.DurationBucket(q.fromMS, q.toMS),
@@ -482,19 +475,19 @@ func (q *builderQuery[T]) executeSkippingRaw(ctx context.Context, query string,
 	rows, err := q.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
 	if err != nil {
 		if errors.Is(err, context.DeadlineExceeded) {
-			return nil, 0, errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "Query timed out").
+			return nil, errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "Query timed out").
 				WithAdditional("Try refining your search by adding relevant resource attributes filtering")
 		}
 
 		if !errors.Is(err, context.Canceled) {
-			return nil, 0, errors.Newf(
+			return nil, errors.Newf(
 				errors.TypeInternal,
 				errors.CodeInternal,
 				"Something went wrong on our end. It's not you, it's us. Our team is notified about it. Reach out to support if issue persists.",
 			)
 		}
 
-		return nil, 0, err
+		return nil, err
 	}
 	defer rows.Close()
 
@@ -508,15 +501,9 @@ func (q *builderQuery[T]) executeSkippingRaw(ctx context.Context, query string,
 		kind = qbtypes.RequestTypeTimeSeries
 	}
 
-	var payload any
-	skipped := 0
-	if skip > 0 {
-		payload, skipped, err = readAsRawSkipping(rows, q.spec.Name, skip)
-	} else {
-		payload, err = consume(rows, kind, queryWindow, q.spec.StepInterval, q.spec.Name)
-	}
+	payload, err := consume(rows, kind, queryWindow, q.spec.StepInterval, q.spec.Name)
 	if err != nil {
-		return nil, 0, err
+		return nil, err
 	}
 
 	// TODO: This should move to readAsRaw function in consume.go but for now we are keeping it here since it's only relevant for traces
@@ -536,7 +523,7 @@ func (q *builderQuery[T]) executeSkippingRaw(ctx context.Context, query string,
 			BytesScanned: totalBytes,
 			DurationMS:   uint64(elapsed.Milliseconds()),
 		},
-	}, skipped, nil
+	}, nil
 }
 
 func (q *builderQuery[T]) executeWindowList(ctx context.Context) (*qbtypes.Result, error) {
@@ -636,7 +623,7 @@ func (q *builderQuery[T]) executeWindowList(ctx context.Context) (*qbtypes.Resul
 			}
 		}
 		// Execute with proper context for partial value detection
-		res, skipped, err := q.executeSkippingRaw(ctx, stmt.Query, stmt.Args, offsetLeft)
+		res, err := q.executeWithContext(ctx, stmt.Query, stmt.Args)
 		if err != nil {
 			return nil, err
 		}
@@ -644,10 +631,13 @@ func (q *builderQuery[T]) executeWindowList(ctx context.Context) (*qbtypes.Resul
 		totalBytes += res.Stats.BytesScanned
 
 		rawRows := res.Value.(*qbtypes.RawData).Rows
-		need -= skipped + len(rawRows)
-		offsetLeft -= skipped
+		need -= len(rawRows)
 
 		for _, rr := range rawRows {
+			if offsetLeft > 0 { // client-requested initial offset
+				offsetLeft--
+				continue
+			}
 			rows = append(rows, rr)
 			if len(rows) >= reqLimit { // page filled
 				break

pkg/querier/consume.go

@@ -521,26 +521,13 @@ func derefValue(v any) any {
 }
 
 func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) {
-	data, _, err := readAsRawSkipping(rows, queryName, 0)
-	return data, err
-}
-
-// readAsRawSkipping advances past the first skip rows without scanning them, so
-// rows dropped for an offset are never decoded. It returns how many were skipped.
-func readAsRawSkipping(rows driver.Rows, queryName string, skip int) (*qbtypes.RawData, int, error) {
 	colNames := rows.Columns()
 	colTypes := rows.ColumnTypes()
 	colCnt := len(colNames)
 
 	var outRows []*qbtypes.RawRow
-	skipped := 0
 
 	for rows.Next() {
-		if skipped < skip {
-			skipped++
-			continue
-		}
-
 		// fresh copy of the scan slice (otherwise the driver reuses pointers)
 		scan := make([]any, colCnt)
 		for i := range colTypes {
@@ -548,7 +535,7 @@ func readAsRawSkipping(rows driver.Rows, queryName string, skip int) (*qbtypes.R
 		}
 
 		if err := rows.Scan(scan...); err != nil {
-			return nil, 0, err
+			return nil, err
 		}
 
 		rr := qbtypes.RawRow{
@@ -580,13 +567,13 @@ func readAsRawSkipping(rows driver.Rows, queryName string, skip int) (*qbtypes.R
 		outRows = append(outRows, &rr)
 	}
 	if err := rows.Err(); err != nil {
-		return nil, 0, err
+		return nil, err
 	}
 
 	return &qbtypes.RawData{
 		QueryName: queryName,
 		Rows:      outRows,
-	}, skipped, nil
+	}, nil
 }
 
 // mergeSpanAttributeColumns merges (attributes_string, attributes_number, attributes_bool, resources_string) into

pkg/querier/consume_test.go

@@ -5,11 +5,9 @@ import (
 	"testing"
 
 	"github.com/ClickHouse/clickhouse-go/v2/lib/chcol"
-	cmock "github.com/SigNoz/clickhouse-go-mock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/types/spantypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrystoretypes"
 )
@@ -256,41 +254,3 @@ func TestMergeSpanAttributeColumns_JSONColumn(t *testing.T) {
 		})
 	}
 }
-
-func TestReadAsRawSkipping(t *testing.T) {
-	testCases := []struct {
-		name        string
-		values      []string
-		skip        int
-		wantSkipped int
-		wantIDs     []string
-	}{
-		{name: "NoSkip", values: []string{"a", "b", "c"}, skip: 0, wantSkipped: 0, wantIDs: []string{"a", "b", "c"}},
-		{name: "SkipInsideResult", values: []string{"d", "e", "f", "g"}, skip: 2, wantSkipped: 2, wantIDs: []string{"f", "g"}},
-		{name: "SkipWholeResult", values: []string{"h", "i"}, skip: 2, wantSkipped: 2, wantIDs: nil},
-		{name: "SkipPastResult", values: []string{"j"}, skip: 5, wantSkipped: 1, wantIDs: nil},
-	}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			values := make([][]any, len(testCase.values))
-			for i, value := range testCase.values {
-				values[i] = []any{value}
-			}
-			rows := cmock.NewRows([]cmock.ColumnType{{Name: "id", Type: "String"}}, values)
-			for i := 0; i < testCase.skip && i < len(values); i++ {
-				rows.RowError(i, errors.New(errors.TypeInternal, errors.CodeInternal, "skipped row was scanned"))
-			}
-
-			data, skipped, err := readAsRawSkipping(rows, "A", testCase.skip)
-			require.NoError(t, err)
-
-			var ids []string
-			for _, row := range data.Rows {
-				ids = append(ids, row.Data["id"].(string))
-			}
-			assert.Equal(t, testCase.wantSkipped, skipped)
-			assert.Equal(t, testCase.wantIDs, ids)
-		})
-	}
-}

pkg/query-service/rules/base_rule.go

@@ -17,6 +17,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/types/rulestatehistorytypes"
 	"github.com/SigNoz/signoz/pkg/types/ruletypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/units"
 	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
@@ -257,6 +258,37 @@ func (r *BaseRule) ExpandAnnotations(expand func(string) string) ruletypes.Label
 	return annotations
 }
 
+// AlertAnnotations returns the expanded rule annotations plus the private ones
+// the notifier reads into template fields. value and threshold are smpl.V and
+// smpl.Target formatted for display.
+func (r *BaseRule) AlertAnnotations(expand func(string) string, smpl ruletypes.Sample, value, threshold string) ruletypes.Labels {
+	annotations := r.ExpandAnnotations(expand)
+	if !smpl.IsMissing {
+		annotations = append(annotations,
+			ruletypes.Label{Name: ruletypes.AnnotationValue, Value: value},
+			ruletypes.Label{Name: ruletypes.AnnotationThresholdValue, Value: threshold},
+		)
+		if op := smpl.CompareOperator.Literal(); op != "" {
+			annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationCompareOp, Value: op})
+		}
+		if matchType := smpl.MatchType.Literal(); matchType != "" {
+			annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationMatchType, Value: matchType})
+		}
+		if smpl.RecoveryTarget != nil {
+			recovery := units.FormatterFromUnit(r.Unit()).Format(*smpl.RecoveryTarget, smpl.TargetUnit)
+			annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationRecoveryThreshold, Value: recovery})
+		}
+	}
+	if smpl.IsRecovering {
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationIsRecovering, Value: "true"})
+	}
+	if signal := r.typ.Signal(); signal != "" {
+		annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationRuleType, Value: signal})
+	}
+
+	return annotations
+}
+
 func (r *BaseRule) ExternalURL(path string, params url.Values) string {
 	u := r.externalURL.JoinPath(path)
 	if len(params) > 0 {

pkg/query-service/rules/base_rule_test.go

@@ -885,3 +885,50 @@ func TestExpandAnnotations(t *testing.T) {
 		})
 	}
 }
+
+func TestAlertAnnotations(t *testing.T) {
+	recoveryTarget := 400.0
+	testCases := []struct {
+		name      string
+		alertType ruletypes.AlertType
+		smpl      ruletypes.Sample
+		want      map[string]string
+	}{
+		{
+			name: "Firing_AddsValueAndThreshold",
+			smpl: ruletypes.Sample{},
+			want: map[string]string{"summary": "high errors", ruletypes.AnnotationValue: "523 ms", ruletypes.AnnotationThresholdValue: "500 ms"},
+		},
+		{
+			name: "Recovering_AddsValueThresholdAndRecoveringFlag",
+			smpl: ruletypes.Sample{IsRecovering: true},
+			want: map[string]string{"summary": "high errors", ruletypes.AnnotationValue: "523 ms", ruletypes.AnnotationThresholdValue: "500 ms", ruletypes.AnnotationIsRecovering: "true"},
+		},
+		{
+			name: "OperatorAndRecoveryTarget_AddsLiteralOpAndRecoveryThreshold",
+			smpl: ruletypes.Sample{CompareOperator: ruletypes.ValueIsAboveSymbol, RecoveryTarget: &recoveryTarget, TargetUnit: "ms"},
+			want: map[string]string{"summary": "high errors", ruletypes.AnnotationValue: "523 ms", ruletypes.AnnotationThresholdValue: "500 ms", ruletypes.AnnotationCompareOp: "above", ruletypes.AnnotationRecoveryThreshold: "400 ms"},
+		},
+		{
+			name:      "MatchTypeAndAlertType_AddsLiteralMatchTypeAndSignal",
+			alertType: ruletypes.AlertTypeLogs,
+			smpl:      ruletypes.Sample{MatchType: ruletypes.OnAverageShort},
+			want:      map[string]string{"summary": "high errors", ruletypes.AnnotationValue: "523 ms", ruletypes.AnnotationThresholdValue: "500 ms", ruletypes.AnnotationMatchType: "on_average", ruletypes.AnnotationRuleType: "logs"},
+		},
+		{
+			name:      "MissingData_OmitsValueAndThreshold_KeepsRuleType",
+			alertType: ruletypes.AlertTypeMetric,
+			smpl:      ruletypes.Sample{IsMissing: true, CompareOperator: ruletypes.ValueIsAbove, MatchType: ruletypes.AtleastOnce, RecoveryTarget: &recoveryTarget},
+			want:      map[string]string{"summary": "high errors", ruletypes.AnnotationRuleType: "metrics"},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			rule := &BaseRule{typ: testCase.alertType, annotations: ruletypes.FromMap(map[string]string{"summary": "high errors"}), ruleCondition: &ruletypes.RuleCondition{CompositeQuery: &ruletypes.AlertCompositeQuery{Unit: "ms"}}}
+			expand := func(text string) string { return text }
+
+			assert.Equal(t, testCase.want, rule.AlertAnnotations(expand, testCase.smpl, "523 ms", "500 ms").Map())
+		})
+	}
+}

pkg/query-service/rules/prom_rule.go

@@ -175,9 +175,10 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 		}
 		r.logger.DebugContext(ctx, "alerting for series", slog.Any("series", result))
 
+		value := valueFormatter.Format(result.V, r.Unit())
 		threshold := valueFormatter.Format(result.Target, result.TargetUnit)
 
-		tmplData := ruletypes.AlertTemplateData(l, valueFormatter.Format(result.V, r.Unit()), threshold)
+		tmplData := ruletypes.AlertTemplateData(l, value, threshold)
 		// Inject some convenience variables that are easier to remember for users
 		// who are not used to Go's templating system.
 		defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}"
@@ -209,7 +210,7 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 		lb.Set(ruletypes.AlertRuleIDLabel, r.ID())
 		lb.Set(ruletypes.RuleSourceLabel, r.GeneratorURL())
 
-		annotations := r.ExpandAnnotations(expand)
+		annotations := r.AlertAnnotations(expand, result, value, threshold)
 		if result.IsMissing {
 			lb.Set(ruletypes.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(ruletypes.NoDataLabel, "true")

pkg/query-service/rules/prom_rule_test.go

@@ -938,6 +938,12 @@ func TestPromRuleUnitCombinations(t *testing.T) {
 		if c.expectAlerts != 0 {
 			foundCount := 0
 			for _, item := range rule.Active {
+				value := item.Annotations.Get(ruletypes.AnnotationValue)
+				assert.NotEmpty(t, value, "case %d", idx)
+				assert.Contains(t, item.Annotations.Get("summary"), value, "case %d", idx)
+				threshold := item.Annotations.Get(ruletypes.AnnotationThresholdValue)
+				assert.NotEmpty(t, threshold, "case %d", idx)
+				assert.Contains(t, item.Annotations.Get("summary"), threshold, "case %d", idx)
 				for _, summary := range c.summaryAny {
 					if strings.Contains(item.Annotations.Get("summary"), summary) {
 						foundCount++
@@ -1042,6 +1048,8 @@ func TestPromRuleNoData(t *testing.T) {
 		for _, item := range rule.Active {
 			if c.expectNoData {
 				assert.True(t, strings.Contains(item.Labels.Get(ruletypes.AlertNameLabel), "[No data]"), "case %d", idx)
+				assert.Empty(t, item.Annotations.Get(ruletypes.AnnotationValue), "case %d", idx)
+				assert.Empty(t, item.Annotations.Get(ruletypes.AnnotationThresholdValue), "case %d", idx)
 			} else {
 				assert.False(t, strings.Contains(item.Labels.Get(ruletypes.AlertNameLabel), "[No data]"), "case %d", idx)
 			}

pkg/query-service/rules/threshold_rule.go

@@ -125,6 +125,17 @@ func (r *ThresholdRule) prepareParamsForTraces(ctx context.Context, ts time.Time
 	return contextlinks.PrepareParamsForTracesV5(start, end, whereClause, r.typ.BuilderQueryType())
 }
 
+func (r *ThresholdRule) prepareParamsForExceptions(ctx context.Context, ts time.Time, resultLabels ruletypes.Labels) url.Values {
+	qr, err := r.prepareQueryRange(ctx, ts)
+	if err != nil {
+		return nil
+	}
+	start := time.UnixMilli(int64(qr.Start))
+	end := time.UnixMilli(int64(qr.End))
+
+	return contextlinks.PrepareParamsForExceptions(start, end, resultLabels.Map())
+}
+
 func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, ts time.Time) (ruletypes.Vector, error) {
 	params, err := r.prepareQueryRange(ctx, ts)
 	if err != nil {
@@ -285,7 +296,7 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 		lb.Set(ruletypes.AlertRuleIDLabel, r.ID())
 		lb.Set(ruletypes.RuleSourceLabel, r.GeneratorURL())
 
-		annotations := r.ExpandAnnotations(expand)
+		annotations := r.AlertAnnotations(expand, smpl, value, threshold)
 		if smpl.IsMissing {
 			lb.Set(ruletypes.AlertNameLabel, "[No data] "+r.Name())
 			lb.Set(ruletypes.NoDataLabel, "true")
@@ -313,6 +324,13 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
 				r.logger.InfoContext(ctx, "adding logs link to annotations", slog.String("annotation.link", link))
 				annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationRelatedLogs, Value: link})
 			}
+		case ruletypes.AlertTypeExceptions:
+			params := r.prepareParamsForExceptions(ctx, ts, smpl.Metric)
+			if len(params) > 0 {
+				link := r.ExternalURL("exceptions", params)
+				r.logger.InfoContext(ctx, "adding exceptions link to annotations", slog.String("annotation.link", link))
+				annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationRelatedExceptions, Value: link})
+			}
 		}
 
 		lbs := lb.Labels()

pkg/query-service/rules/threshold_rule_test.go

@@ -207,6 +207,60 @@ func TestPrepareParamsForLogs(t *testing.T) {
 	assert.JSONEq(t, `{"queryType":"builder","builder":{"queryData":[{"dataSource":"logs","filter":{}}],"queryFormulas":[]}}`, compositeQuery)
 }
 
+func TestPrepareParamsForExceptions(t *testing.T) {
+	postableRule := ruletypes.PostableRule{
+		AlertName: valuer.NewDisplayName("Exceptions In Checkout"),
+		AlertType: ruletypes.AlertTypeExceptions,
+		RuleType:  ruletypes.RuleTypeThreshold,
+		Evaluation: &ruletypes.EvaluationEnvelope{Kind: ruletypes.RollingEvaluation, Spec: ruletypes.RollingWindow{
+			EvalWindow: valuer.MustParseTextDuration("5m"),
+			Frequency:  valuer.MustParseTextDuration("1m"),
+		}},
+		RuleCondition: &ruletypes.RuleCondition{
+			CompositeQuery: &ruletypes.AlertCompositeQuery{
+				QueryType: ruletypes.QueryTypeClickHouseSQL,
+				Queries: []qbtypes.QueryEnvelope{
+					{
+						Type: qbtypes.QueryTypeClickHouseSQL,
+						Spec: qbtypes.ClickHouseQuery{
+							Name:  "A",
+							Query: "SELECT count() AS value, toStartOfInterval(timestamp, toIntervalMinute(1)) AS interval, serviceName FROM signoz_traces.distributed_signoz_error_index_v2 WHERE timestamp BETWEEN {{.start_datetime}} AND {{.end_datetime}} GROUP BY serviceName, interval",
+						},
+					},
+				},
+			},
+			CompareOperator: ruletypes.ValueIsAbove,
+			MatchType:       ruletypes.AtleastOnce,
+			Target:          &[]float64{10.0}[0],
+			SelectedQuery:   "A",
+		},
+	}
+
+	logger := instrumentationtest.New().Logger()
+	postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
+		Kind: ruletypes.BasicThresholdKind,
+		Spec: ruletypes.BasicRuleThresholds{
+			{
+				TargetValue:     postableRule.RuleCondition.Target,
+				MatchType:       postableRule.RuleCondition.MatchType,
+				CompareOperator: postableRule.RuleCondition.CompareOperator,
+			},
+		},
+	}
+
+	externalURL := mustParseURL(t, "http://localhost:8080")
+	rule, err := NewThresholdRule("70", valuer.GenerateUUID(), &postableRule, nil, logger, externalURL, WithEvalDelay(valuer.MustParseTextDuration("2m")))
+	require.NoError(t, err)
+
+	ts := time.UnixMilli(1705469040000)
+
+	params := rule.prepareParamsForExceptions(context.Background(), ts, ruletypes.FromMap(map[string]string{"serviceName": "checkout"}))
+	assert.Equal(t, "1705468620000", params.Get("startTime"))
+	assert.Equal(t, "1705468920000", params.Get("endTime"))
+	assert.Equal(t, "checkout", params.Get("serviceName"))
+	assert.False(t, params.Has("exceptionType"))
+}
+
 func TestPrepareParamsForLogsFilterExpression(t *testing.T) {
 	postableRule := ruletypes.PostableRule{
 		AlertName: valuer.NewDisplayName("Tricky Condition Tests"),
@@ -678,6 +732,14 @@ func TestThresholdRuleUnitCombinations(t *testing.T) {
 		if c.expectAlerts != 0 {
 			foundCount := 0
 			for _, item := range rule.Active {
+				value := item.Annotations.Get(ruletypes.AnnotationValue)
+				assert.NotEmpty(t, value, "case %d", idx)
+				assert.Contains(t, item.Annotations.Get("summary"), value, "case %d", idx)
+				threshold := item.Annotations.Get(ruletypes.AnnotationThresholdValue)
+				assert.NotEmpty(t, threshold, "case %d", idx)
+				assert.Contains(t, item.Annotations.Get("summary"), threshold, "case %d", idx)
+				assert.Equal(t, c.compareOperator.Literal(), item.Annotations.Get(ruletypes.AnnotationCompareOp), "case %d", idx)
+				assert.Equal(t, c.matchType.Literal(), item.Annotations.Get(ruletypes.AnnotationMatchType), "case %d", idx)
 				for _, summary := range c.summaryAny {
 					if strings.Contains(item.Annotations.Get("summary"), summary) {
 						foundCount++
@@ -787,6 +849,8 @@ func TestThresholdRuleNoData(t *testing.T) {
 		for _, item := range rule.Active {
 			if c.expectNoData {
 				assert.True(t, strings.Contains(item.Labels.Get(ruletypes.AlertNameLabel), "[No data]"), "case %d", idx)
+				assert.Empty(t, item.Annotations.Get(ruletypes.AnnotationValue), "case %d", idx)
+				assert.Empty(t, item.Annotations.Get(ruletypes.AnnotationThresholdValue), "case %d", idx)
 			} else {
 				assert.False(t, strings.Contains(item.Labels.Get(ruletypes.AlertNameLabel), "[No data]"), "case %d", idx)
 			}

pkg/ruler/authz.go

@@ -103,7 +103,7 @@ func NewCreateRuleChannelAttachExtractor(alertmanagerService alertmanager.Alertm
 			return nil, err
 		}
 
-		return resolveKnownChannelIDs(ctx, alertmanagerService, orgID, names)
+		return alertmanagerService.GetChannelIDsByNames(ctx, orgID.StringValue(), names)
 	}}
 }
 
@@ -124,7 +124,7 @@ func NewCloneRuleChannelAttachExtractor(alertmanagerService alertmanager.Alertma
 			return nil, err
 		}
 
-		return resolveKnownChannelIDs(ctx, alertmanagerService, orgID, stored.Channels())
+		return alertmanagerService.GetChannelIDsByNames(ctx, orgID.StringValue(), stored.Channels())
 	}}
 }
 
@@ -168,10 +168,10 @@ func newRuleChannelDiffExtractor(alertmanagerService alertmanager.Alertmanager,
 
 		attached, detached := diffLinks(stored.Channels(), incomingNames)
 		if side == detachedSide {
-			return resolveKnownChannelIDs(ctx, alertmanagerService, orgID, detached)
+			return resolveStoredChannelIDs(ctx, alertmanagerService, orgID, detached)
 		}
 
-		return resolveKnownChannelIDs(ctx, alertmanagerService, orgID, attached)
+		return alertmanagerService.GetChannelIDsByNames(ctx, orgID.StringValue(), attached)
 	}}
 }
 
@@ -199,7 +199,7 @@ func applyRulePatch(stored *ruletypes.GettableRule, body []byte) (*ruletypes.Pos
 
 func parseRuleChannels(body []byte) ([]string, error) {
 	rule := new(ruletypes.PostableRule)
-	if err := rule.DecodeStrict(string(body)); err != nil {
+	if err := json.Unmarshal(body, rule); err != nil {
 		return nil, err
 	}
 
@@ -278,8 +278,7 @@ func newRoutePolicyDiffExtractor(alertmanagerService alertmanager.Alertmanager,
 	}}
 }
 
-// resolveKnownChannelIDs skips unknown names, the handler rejects them with a clearer error.
-func resolveKnownChannelIDs(ctx context.Context, alertmanagerService alertmanager.Alertmanager, orgID valuer.UUID, names []string) ([]string, error) {
+func resolveStoredChannelIDs(ctx context.Context, alertmanagerService alertmanager.Alertmanager, orgID valuer.UUID, names []string) ([]string, error) {
 	if len(names) == 0 {
 		return nil, nil
 	}

pkg/ruler/authz_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertest"
-	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/types"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes/alertmanagertypestest"
@@ -373,20 +372,10 @@ func TestParseRuleChannels(t *testing.T) {
 			wantChannels: nil,
 		},
 		{
-			name:    "NotJSON_InvalidInput",
+			name:    "NotJSON_Errors",
 			body:    "not json",
 			wantErr: true,
 		},
-		{
-			name:    "TruncatedJSON_InvalidInput",
-			body:    `{"alert": `,
-			wantErr: true,
-		},
-		{
-			name:    "UnknownField_InvalidInput",
-			body:    `{"preferredChannels": ["pager"]}`,
-			wantErr: true,
-		},
 	}
 
 	for _, testCase := range testCases {
@@ -394,8 +383,7 @@ func TestParseRuleChannels(t *testing.T) {
 			channels, err := parseRuleChannels([]byte(testCase.body))
 
 			if testCase.wantErr {
-				require.Error(t, err)
-				assert.True(t, errors.Ast(err, errors.TypeInvalidInput), err.Error())
+				assert.Error(t, err)
 				return
 			}
 
@@ -586,7 +574,7 @@ func TestOptionalRuleBodyTelemetryResources(t *testing.T) {
 	}
 }
 
-func TestResolveKnownChannelIDs(t *testing.T) {
+func TestResolveStoredChannelIDs(t *testing.T) {
 	channels := []*alertmanagertypes.Channel{
 		{Identifiable: types.Identifiable{ID: valuer.GenerateUUID()}, Name: "slack", DisplayName: "slack"},
 	}
@@ -619,7 +607,7 @@ func TestResolveKnownChannelIDs(t *testing.T) {
 				alertmanagerService.EXPECT().ListChannels(mock.Anything, mock.Anything).Return(channels, nil)
 			}
 
-			ids, err := resolveKnownChannelIDs(context.Background(), alertmanagerService, valuer.GenerateUUID(), testCase.names)
+			ids, err := resolveStoredChannelIDs(context.Background(), alertmanagerService, valuer.GenerateUUID(), testCase.names)
 
 			require.NoError(t, err)
 			assert.Equal(t, testCase.wantIDs, ids)

pkg/types/alertmanagertypes/template.go

@@ -3,6 +3,7 @@ package alertmanagertypes
 import (
 	"bytes"
 	"net/url"
+	"time"
 
 	tmplhtml "html/template"
 	tmpltext "text/template"
@@ -19,6 +20,7 @@ func AdditionalFuncMap() tmpltext.FuncMap {
 		"urlescape": func(value string) tmplhtml.HTML {
 			return tmplhtml.HTML(url.QueryEscape(value))
 		},
+		"now": time.Now,
 	}
 }
 

pkg/types/alertmanagertypes/templatedata.go

@@ -76,8 +76,14 @@ type AlertData struct {
 	Rule        RuleInfo    `json:"rule"        mapstructure:"rule"`
 	Log         LinkInfo    `json:"log"         mapstructure:"log"`
 	Trace       LinkInfo    `json:"trace"       mapstructure:"trace"`
+	Exception   LinkInfo    `json:"exception"   mapstructure:"exception"`
 	Labels      template.KV `json:"labels"      mapstructure:"labels"`
 	Annotations template.KV `json:"annotations" mapstructure:"annotations"`
+
+	// Value and Threshold back the $value and $threshold shortcuts for
+	// $alert.value and $rule.threshold.value.
+	Value     string `json:"value"     mapstructure:"value"`
+	Threshold string `json:"threshold" mapstructure:"threshold"`
 }
 
 // AlertInfo holds the per-alert state and timing data.
@@ -93,6 +99,7 @@ type AlertInfo struct {
 	IsResolved    bool      `json:"isResolved"    mapstructure:"is_resolved"`
 	IsMissingData bool      `json:"isMissingData" mapstructure:"is_missing_data"`
 	IsRecovering  bool      `json:"isRecovering"  mapstructure:"is_recovering"`
+	IsTest        bool      `json:"isTest"        mapstructure:"is_test"`
 }
 
 // RuleInfo holds the rule metadata extracted from well-known labels and
@@ -102,18 +109,20 @@ type RuleInfo struct {
 	ID        string    `json:"id"        mapstructure:"id"`
 	URL       string    `json:"url"       mapstructure:"url"`
 	Severity  string    `json:"severity"  mapstructure:"severity"`
+	Type      string    `json:"type"      mapstructure:"type"`
 	MatchType string    `json:"matchType" mapstructure:"match_type"`
 	Threshold Threshold `json:"threshold" mapstructure:"threshold"`
 }
 
 // Threshold describes the breach condition.
 type Threshold struct {
-	Value string `json:"value" mapstructure:"value"`
-	Op    string `json:"op"    mapstructure:"op"`
+	Value         string `json:"value"         mapstructure:"value"`
+	Op            string `json:"op"            mapstructure:"op"`
+	RecoveryValue string `json:"recoveryValue" mapstructure:"recovery_value"`
 }
 
-// LinkInfo groups a single URL so templates can reference $log.url and
-// $trace.url uniformly.
+// LinkInfo groups a single URL so templates can reference $log.url,
+// $trace.url and $exception.url uniformly.
 type LinkInfo struct {
 	URL string `json:"url" mapstructure:"url"`
 }
@@ -137,8 +146,10 @@ type NotificationTemplateData struct {
 // NotificationAlert holds the aggregate alert fields available to title
 // templates (counts, overall status, receiver).
 type NotificationAlert struct {
-	Receiver      string `json:"receiver"      mapstructure:"receiver"`
-	Status        string `json:"status"        mapstructure:"status"`
-	TotalFiring   int    `json:"totalFiring"   mapstructure:"total_firing"`
-	TotalResolved int    `json:"totalResolved" mapstructure:"total_resolved"`
+	Receiver        string `json:"receiver"        mapstructure:"receiver"`
+	Status          string `json:"status"          mapstructure:"status"`
+	TotalFiring     int    `json:"totalFiring"     mapstructure:"total_firing"`
+	TotalRecovering int    `json:"totalRecovering" mapstructure:"total_recovering"`
+	TotalResolved   int    `json:"totalResolved"   mapstructure:"total_resolved"`
+	IsTest          bool   `json:"isTest"          mapstructure:"is_test"`
 }

pkg/types/ruletypes/api_params.go

@@ -41,6 +41,25 @@ func (AlertType) Enum() []any {
 	}
 }
 
+// Signal returns the signal the alert type evaluates (metrics, logs, traces,
+// exceptions or ai_traces), or an empty string for an unknown type.
+func (t AlertType) Signal() string {
+	switch t {
+	case AlertTypeMetric:
+		return "metrics"
+	case AlertTypeLogs:
+		return "logs"
+	case AlertTypeTraces:
+		return "traces"
+	case AlertTypeExceptions:
+		return "exceptions"
+	case AlertTypeAITraces:
+		return "ai_traces"
+	default:
+		return ""
+	}
+}
+
 // BuilderQueryType returns the query type the alert type's builder queries
 // carry; only AI trace alerts use builder_ai_query.
 func (t AlertType) BuilderQueryType() qbtypes.QueryType {

pkg/types/ruletypes/api_params_test.go

@@ -608,3 +608,24 @@ func TestParseIntoRuleMultipleThresholds(t *testing.T) {
 
 	assert.Equal(t, 1, len(vector))
 }
+
+func TestAlertTypeSignal(t *testing.T) {
+	testCases := []struct {
+		name      string
+		alertType AlertType
+		want      string
+	}{
+		{name: "Metric", alertType: AlertTypeMetric, want: "metrics"},
+		{name: "Logs", alertType: AlertTypeLogs, want: "logs"},
+		{name: "Traces", alertType: AlertTypeTraces, want: "traces"},
+		{name: "Exceptions", alertType: AlertTypeExceptions, want: "exceptions"},
+		{name: "AITraces", alertType: AlertTypeAITraces, want: "ai_traces"},
+		{name: "Unknown_Empty", alertType: "", want: ""},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			assert.Equal(t, testCase.want, testCase.alertType.Signal())
+		})
+	}
+}

pkg/types/ruletypes/constants.go

@@ -13,29 +13,34 @@ const (
 	LabelNoData           = "nodata"
 	LabelTestAlert        = "testalert"
 	LabelAlertName        = "alertname"
-	LabelIsRecovering     = "is_recovering"
 )
 
 // Annotations set by the rule manager and consumed by the alertmanager
 // templating layer.
 //
-// Those prefixed with "_" are private: they're stripped from
-// notification-visible surfaces by alertmanagertypes.FilterPublicAnnotations
-// before rendering. Only the raw template strings are private — echoing
-// them into a notification is circular and never useful.
+// Those prefixed with "_" are private: the alertmanager reads them into
+// structured template fields, then strips them from notification-visible
+// surfaces by alertmanagertypes.FilterPublicAnnotations before rendering.
+// The raw template strings, the breached value ($alert.value), the configured
+// threshold ($rule.threshold.value), its comparator ($rule.threshold.op), the
+// recovery threshold ($rule.threshold.recovery_value), the match type
+// ($rule.match_type), the rule's signal ($rule.type), the exceptions page link
+// ($exception.url) and the recovering flag ($alert.is_recovering) are private.
 //
-// The rest are public: they describe the firing alert (the breached value,
-// the configured threshold, the comparator, the match type, and deep links
-// to relevant logs/traces) and users may reference them directly as
-// {{ .Annotations.value }}, {{ .Annotations.threshold.value }}, etc. in
+// The rest are public: they are deep links to relevant logs/traces and users
+// may reference them directly as {{ .Annotations.related_logs }}, etc. in
 // their channel templates.
 const (
-	AnnotationTitleTemplate  = "_title_template"
-	AnnotationBodyTemplate   = "_body_template"
-	AnnotationRelatedLogs    = "related_logs"
-	AnnotationRelatedTraces  = "related_traces"
-	AnnotationValue          = "value"
-	AnnotationThresholdValue = "threshold.value"
-	AnnotationCompareOp      = "compare_op"
-	AnnotationMatchType      = "match_type"
+	AnnotationTitleTemplate     = "_title_template"
+	AnnotationBodyTemplate      = "_body_template"
+	AnnotationRelatedLogs       = "related_logs"
+	AnnotationRelatedTraces     = "related_traces"
+	AnnotationRelatedExceptions = "_related_exceptions"
+	AnnotationValue             = "_value"
+	AnnotationIsRecovering      = "_is_recovering"
+	AnnotationThresholdValue    = "_threshold"
+	AnnotationRecoveryThreshold = "_recovery_threshold"
+	AnnotationCompareOp         = "_compare_op"
+	AnnotationMatchType         = "_match_type"
+	AnnotationRuleType          = "_rule_type"
 )

templates/alertmanager/email.gotmpl

@@ -62,6 +62,7 @@
               <p style="margin:0;font-size:14px;color:#666">
                 Status: <strong>{{.Alert.Status}}</strong>
                 {{if .Alert.TotalFiring}} | Firing: <strong style="color:#e53e3e">{{.Alert.TotalFiring}}</strong>{{end}}
+                {{if .Alert.TotalRecovering}} | Recovering: <strong style="color:#dd6b20">{{.Alert.TotalRecovering}}</strong>{{end}}
                 {{if .Alert.TotalResolved}} | Resolved: <strong style="color:#38a169">{{.Alert.TotalResolved}}</strong>{{end}}
               </p>
             </td>

tests/conftest.py

@@ -114,8 +114,8 @@ def pytest_addoption(parser: pytest.Parser):
         help="clickhouse version",
     )
     parser.addoption(
-        "--signoz-otel-collector-version",
+        "--schema-migrator-version",
         action="store",
-        default="v0.144.16-rc.1",
-        help="signoz-otel-collector version, used to run clickhouse migrations",
+        default="v0.144.9",  # todo(nikhil): change to 0.144.10
+        help="schema migrator version",
     )

tests/fixtures/metrics.py

@@ -990,7 +990,7 @@ def insert_reduced_metrics_to_clickhouse(
 ) -> None:
     """Insert reduced series into distributed_time_series_v4_reduced and 60s
     buckets into the reduced samples tables. These tables exist only when
-    the signoz-otel-collector version includes the metrics cardinality-control
+    the schema migrator version includes the metrics cardinality-control
     migration."""
     if time_series:
         conn.insert(

tests/fixtures/migrator.py

@@ -19,28 +19,37 @@ def create_migrator(  # pylint: disable=too-many-arguments,too-many-positional-a
     """
     Factory function for running schema migrations. Accepts optional
     env_overrides to customize the migrator environment; the release comes
-    from the --signoz-otel-collector-version option.
+    from the --schema-migrator-version option.
     """
 
     def create() -> None:
-        collector_version = request.config.getoption("--signoz-otel-collector-version")
+        migrator_version = request.config.getoption("--schema-migrator-version")
         client = docker.from_env()
 
-        environment = {
-            "SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN": clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN"],
-            "SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_CLUSTER": "cluster",
-            "SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_REPLICATION": "true",
-            "SIGNOZ_OTEL_COLLECTOR_TIMEOUT": "10m",
-            **(env_overrides or {}),
-        }
+        environment = dict(env_overrides) if env_overrides else {}
 
         container = client.containers.run(
-            image=f"signoz/signoz-otel-collector:{collector_version}",
-            entrypoint="/bin/sh",
-            command=[
-                "-c",
-                "/signoz-otel-collector migrate bootstrap && /signoz-otel-collector migrate sync up && /signoz-otel-collector migrate async up",
-            ],
+            image=f"signoz/signoz-schema-migrator:{migrator_version}",
+            command=f"sync --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN']}",
+            detach=True,
+            auto_remove=False,
+            network=network.id,
+            environment=environment,
+        )
+
+        result = container.wait()
+
+        if result["StatusCode"] != 0:
+            logs = container.logs().decode(encoding="utf-8")
+            container.remove()
+            print(logs)
+            raise RuntimeError("failed to run migrations on clickhouse")
+
+        container.remove()
+
+        container = client.containers.run(
+            image=f"signoz/signoz-schema-migrator:{migrator_version}",
+            command=f"async --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN']}",
             detach=True,
             auto_remove=False,
             network=network.id,

tests/fixtures/querier.py

@@ -1018,38 +1018,20 @@ def assert_distribution_sums_heatmap(
 # we already create the evolution for resource during schema migration
 # since we have to create test data around it, we need to get the evolution time
 def get_resource_evolution_time(signoz: types.SigNoz, signal: str) -> datetime:
-    return get_column_evolution_time(signoz, signal, "resource", "resource")
-
-
-# the migrator also seeds the traces attributes JSON rollout at install
-def get_attribute_evolution_time(signoz: types.SigNoz, signal: str) -> datetime:
-    return get_column_evolution_time(signoz, signal, "attributes", "attribute")
-
-
-def get_attribute_backend_now(signoz: types.SigNoz, attribute_backend: str) -> datetime:
-    """The "now" a test anchors its spans and query window to. "map" sits before the traces
-    attributes JSON rollout, where the querier reads only the legacy maps, the same SQL as with
-    use_trace_attributes_json off; "json" is the wall clock, after the rollout."""
-    if attribute_backend == "map":
-        return get_attribute_evolution_time(signoz, "traces") - timedelta(hours=1)
-    return datetime.now(tz=UTC)
-
-
-def get_column_evolution_time(signoz: types.SigNoz, signal: str, column_name: str, field_context: str) -> datetime:
     result = signoz.telemetrystore.conn.query(
         """
-        SELECT min(release_time)
+        SELECT release_time
         FROM signoz_metadata.distributed_column_evolution_metadata
         WHERE signal = %(signal)s
-          AND field_context = %(field_context)s
+          AND field_context = 'resource'
           AND field_name = '__all__'
-          AND column_name = %(column_name)s
-        HAVING count() > 0
+          AND column_name = 'resource'
+        LIMIT 1
         """,
-        parameters={"signal": signal, "column_name": column_name, "field_context": field_context},
+        parameters={"signal": signal},
     ).result_rows
 
-    assert result, f"Expected {signal} {column_name} evolution metadata to exist"
+    assert result, f"Expected {signal} resource evolution metadata to exist"
 
     release_time_ns = int(result[0][0])
     return datetime.fromtimestamp(release_time_ns / 1e9, tz=UTC)
@@ -1150,7 +1132,7 @@ def generate_logs_with_corrupt_metadata() -> list[Logs]:
     ]
 
 
-def generate_traces_with_corrupt_metadata(now: datetime | None = None) -> list[Traces]:
+def generate_traces_with_corrupt_metadata() -> list[Traces]:
     """
     Specifically, entries with 'id', 'timestamp', 'trace_id' and 'duration_nano' fields in metadata
     """
@@ -1161,7 +1143,7 @@ def generate_traces_with_corrupt_metadata(now: datetime | None = None) -> list[T
     topic_service_trace_id = TraceIdGenerator.trace_id()
     topic_service_span_id = TraceIdGenerator.span_id()
 
-    now = (now or datetime.now(tz=UTC)).replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
 
     return [
         Traces(

tests/fixtures/signoz.py

@@ -92,7 +92,7 @@ def create_signoz(
                 "SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__WAIT": "1s",
                 "SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__INTERVAL": "5s",
                 "SIGNOZ_CLOUDINTEGRATION_AGENT_VERSION": "v0.0.8",
-                # no schema metadata caching in tests
+                # tests reseed the attributes rollout row between cases
                 "SIGNOZ_QUERIER_SCHEMAMETADATASTORE_CACHE__TTL": "1ns",
             }
             | sqlstore.env

tests/fixtures/traces.py

@@ -986,6 +986,60 @@ def insert_traces(
     )
 
 
+def insert_attribute_evolution_to_clickhouse(conn, signal: str, release_time: datetime.datetime) -> None:
+    """Seed the `attributes` JSON column-evolution row for a signal at release_time. Unlike the
+    resource row (seeded by the migrator at install), the attribute JSON rollout is install-specific
+    and not migrator-seeded, so tests insert it to gate map-vs-JSON resolution across a window."""
+    # insert_deduplicate=0: successive tests seed a byte-identical row (release_time is
+    # minute-aligned), and ReplicatedMergeTree would drop the re-insert as a duplicate even
+    # after the prior test's teardown deleted it, leaving the querier to fall back to the Map.
+    conn.command(
+        """
+        INSERT INTO signoz_metadata.distributed_column_evolution_metadata
+            (signal, column_name, column_type, field_context, field_name, version, release_time)
+        SETTINGS insert_deduplicate = 0
+        VALUES (%(signal)s, 'attributes', 'JSON()', 'attribute', '__all__', 1, %(release_time_ns)s)
+        """,
+        parameters={"signal": signal, "release_time_ns": int(release_time.timestamp() * 1e9)},
+    )
+
+
+@pytest.fixture(name="seed_attribute_evolution", scope="function")
+def seed_attribute_evolution(
+    clickhouse: types.TestContainerClickhouse,
+) -> Generator[Callable[[str, datetime.datetime], None], Any]:
+    def _seed(signal: str, release_time: datetime.datetime) -> None:
+        insert_attribute_evolution_to_clickhouse(clickhouse.conn, signal, release_time)
+
+    yield _seed
+
+    cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
+    clickhouse.conn.query(f"ALTER TABLE signoz_metadata.column_evolution_metadata ON CLUSTER '{cluster}' DELETE WHERE column_name = 'attributes' AND field_context = 'attribute' AND field_name = '__all__' SETTINGS mutations_sync = 1")
+
+
+# A rollout far before any test's query window, so seeding it makes the querier read span
+# attributes entirely from the native JSON column rather than straddling into the legacy Map.
+ATTRIBUTE_JSON_ROLLOUT_TIME = datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)
+
+
+@pytest.fixture(name="use_attribute_backend")
+def use_attribute_backend(
+    seed_attribute_evolution: Callable[[str, datetime.datetime], None],
+) -> Callable[[str], None]:
+    """Applies the physical attribute layout a test runs under. "map" leaves the querier on the
+    legacy attributes_{string,number,bool} maps; "json" seeds the column-evolution row so it reads
+    the native `attributes` JSON column instead. Pair with
+    @pytest.mark.parametrize("attribute_backend", ["map", "json"]) and call it once in the body;
+    insert_traces dual-writes both layouts, so a test that passes under both has strict Map/JSON
+    parity."""
+
+    def _apply(backend: str) -> None:
+        if backend == "json":
+            seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
+
+    return _apply
+
+
 @pytest.fixture(name="insert_top_level_operations", scope="function")
 def insert_top_level_operations(
     clickhouse: types.TestContainerClickhouse,

tests/integration/tests/querierlogs/01_list.py

@@ -9,12 +9,7 @@ from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.logs import Logs
 from fixtures.querier import (
-    BuilderQuery,
-    OrderBy,
-    RequestType,
-    TelemetryFieldKey,
     assert_identical_query_response,
-    get_rows,
     make_query_request,
 )
 
@@ -503,64 +498,3 @@ def test_logs_list_with_order_by(
     ids = [row["data"]["attributes_string"].get("id", "") for row in rows]
 
     assert ids == expected_order
-
-
-@pytest.mark.parametrize(
-    "limit,offset",
-    [
-        pytest.param(2, 0, id="first_chunk"),
-        pytest.param(2, 1, id="first_into_second_chunk"),
-        pytest.param(2, 2, id="second_chunk"),
-        pytest.param(2, 3, id="second_into_third_chunk"),
-        pytest.param(2, 4, id="third_chunk"),
-        pytest.param(4, 5, id="last_partial_page"),
-        pytest.param(2, 6, id="offset_past_end"),
-    ],
-)
-def test_logs_list_offset_across_chunks(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_logs: Callable[[list[Logs]], None],
-    limit: int,
-    offset: int,
-) -> None:
-    """
-    A 4h list runs as 1h, 2h and 1h chunks, newest first. Two logs sit in each
-    chunk, so the offset lands in, or crosses into, every chunk.
-    """
-    now = datetime.now(tz=UTC).replace(microsecond=0)
-    minutes_ago = [10, 20, 90, 150, 200, 220]
-    insert_logs(
-        [
-            Logs(
-                timestamp=now - timedelta(minutes=minutes),
-                resources={"service.name": "chunked-offset-service"},
-                body=f"chunked-log-{minutes}",
-            )
-            for minutes in minutes_ago
-        ]
-    )
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    response = make_query_request(
-        signoz,
-        token,
-        start_ms=int((now - timedelta(hours=4)).timestamp() * 1000),
-        end_ms=int((now + timedelta(seconds=1)).timestamp() * 1000),
-        request_type=RequestType.RAW,
-        queries=[
-            BuilderQuery(
-                signal="logs",
-                name="A",
-                limit=limit,
-                offset=offset,
-                filter_expression="service.name = 'chunked-offset-service'",
-                order=[OrderBy(TelemetryFieldKey("timestamp"), "desc"), OrderBy(TelemetryFieldKey("id"), "desc")],
-            ).to_dict()
-        ],
-    )
-
-    assert response.status_code == HTTPStatus.OK, response.text
-    expected_bodies = [f"chunked-log-{minutes}" for minutes in minutes_ago][offset : offset + limit]
-    assert [row["data"]["body"] for row in get_rows(response)] == expected_bodies

tests/integration/tests/queriertraces/01_list.py

@@ -917,69 +917,6 @@ def test_traces_list_pagination(
     assert [row["data"]["span_id"] for row in rows] == expected_ids
 
 
-@pytest.mark.parametrize(
-    "limit,offset",
-    [
-        pytest.param(2, 0, id="first_chunk"),
-        pytest.param(2, 1, id="first_into_second_chunk"),
-        pytest.param(2, 2, id="second_chunk"),
-        pytest.param(2, 3, id="second_into_third_chunk"),
-        pytest.param(2, 4, id="third_chunk"),
-        pytest.param(4, 5, id="last_partial_page"),
-        pytest.param(2, 6, id="offset_past_end"),
-    ],
-)
-def test_traces_list_offset_across_chunks(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-    insert_traces: Callable[[list[Traces]], None],
-    limit: int,
-    offset: int,
-) -> None:
-    """
-    A 4h list runs as 1h, 2h and 1h chunks, newest first. Two spans sit in each
-    chunk, so the offset lands in, or crosses into, every chunk.
-    """
-    now = datetime.now(tz=UTC).replace(microsecond=0)
-    minutes_ago = [10, 20, 90, 150, 200, 220]
-    insert_traces(
-        [
-            Traces(
-                timestamp=now - timedelta(minutes=minutes),
-                trace_id=TraceIdGenerator.trace_id(),
-                span_id=TraceIdGenerator.span_id(),
-                name=f"chunked-span-{minutes}",
-                resources={"service.name": "chunked-offset-service"},
-            )
-            for minutes in minutes_ago
-        ]
-    )
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    response = make_query_request(
-        signoz,
-        token,
-        start_ms=int((now - timedelta(hours=4)).timestamp() * 1000),
-        end_ms=int((now + timedelta(seconds=1)).timestamp() * 1000),
-        request_type=RequestType.RAW,
-        queries=[
-            BuilderQuery(
-                signal="traces",
-                name="A",
-                limit=limit,
-                offset=offset,
-                filter_expression="service.name = 'chunked-offset-service'",
-                order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")],
-            ).to_dict()
-        ],
-    )
-
-    assert response.status_code == HTTPStatus.OK, response.text
-    expected_names = [f"chunked-span-{minutes}" for minutes in minutes_ago][offset : offset + limit]
-    assert [row["data"]["name"] for row in get_rows(response)] == expected_names
-
-
 # ============================================================================
 # Time window
 # ============================================================================

tests/integration/tests/queriertraces/02_aggregation.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
 from typing import Any
 
@@ -20,7 +20,6 @@ from fixtures.querier import (
     build_order_by,
     build_traces_scalar_query,
     get_all_series,
-    get_attribute_backend_now,
     get_scalar_table_data,
     index_series_by_label,
     make_query_request,
@@ -71,6 +70,7 @@ def test_traces_aggregate_order_by_count(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     order_by: dict[str, str],
     aggregation_alias: str,
@@ -86,9 +86,9 @@ def test_traces_aggregate_order_by_count(
     fieldContext. Only the combinations that double-specify the context are a bad
     request; the rest resolve and return the span count.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     trace_id = TraceIdGenerator.trace_id()
     spans = [
         Traces(
@@ -109,11 +109,9 @@ def test_traces_aggregate_order_by_count(
     response = make_query_request(
         signoz,
         token,
-        start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
-        end_ms=int(now.timestamp() * 1000),
+        start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
+        end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
         request_type=RequestType.TIME_SERIES,
-        # cases share the query, and a "map" window is old enough to be served from the cache
-        no_cache=True,
         queries=[
             BuilderQuery(
                 signal="traces",
@@ -147,6 +145,7 @@ def test_traces_aggregate_functions(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     noise: str,
 ) -> None:
@@ -164,10 +163,10 @@ def test_traces_aggregate_functions(
     the corrupt variant the same-named colliding attributes must not change any of
     these.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
     extra_attrs, extra_resources = trace_noise(noise)
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     # A scalar rate divides by the whole query window, so both sides agree on it.
     # Traces derives this separately from logs (telemetrytraces/statement_builder.go).
     lookback_minutes = 5
@@ -254,6 +253,7 @@ def test_traces_aggregate_calculated_range_countif(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
 ) -> None:
     """
@@ -264,9 +264,9 @@ def test_traces_aggregate_calculated_range_countif(
     countIf(response_status_code >= 400 AND response_status_code < 500) counts the
     single 4XX span (404).
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     spans = [
         Traces(
             timestamp=now - timedelta(seconds=i + 1),
@@ -307,6 +307,7 @@ def test_traces_aggregate_having(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     noise: str,
 ) -> None:
@@ -318,10 +319,10 @@ def test_traces_aggregate_having(
     A grouped scalar query with `having count() > 2` returns only the groups
     whose count qualifies (svc-a), derived from the inserted spans.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
     extra_attrs, extra_resources = trace_noise(noise)
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     counts = {"svc-a": 3, "svc-b": 1}
     spans = [
         Traces(
@@ -378,6 +379,7 @@ def test_traces_aggregate_time_series_limit(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     noise: str,
     limit: int,
@@ -393,10 +395,10 @@ def test_traces_aggregate_time_series_limit(
     summing to that service's span count. Under the corrupt variant the grouping
     (resource.service.name) and count are unaffected by the colliding attributes.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
     extra_attrs, extra_resources = trace_noise(noise)
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     counts = {"svc-a": 5, "svc-b": 3, "svc-c": 7, "svc-d": 1}
     spans = [
         Traces(
@@ -461,6 +463,7 @@ def test_traces_aggregate_unknown_key(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     expression: str,
     expected_value: Any,
@@ -473,9 +476,9 @@ def test_traces_aggregate_unknown_key(
     Aggregating a bare unknown key synthesizes a null column: count() of it is 0
     (no non-null values) and sum()/avg() are null — the query never errors.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     spans = [
         Traces(
             timestamp=now - timedelta(seconds=i + 1),

tests/integration/tests/queriertraces/06_trace_operator.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
 from typing import Any
 
@@ -16,7 +16,6 @@ from fixtures.querier import (
     find_named_result,
     format_timestamp,
     generate_traces_with_corrupt_metadata,
-    get_attribute_backend_now,
     get_rows,
     make_query_request,
 )
@@ -238,10 +237,11 @@ def test_trace_operator(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     case: dict,
 ) -> None:
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
     t1_trace_id = TraceIdGenerator.trace_id()
     t1_checkout_span_id = TraceIdGenerator.span_id()  # POST /checkout — structural root of T1
@@ -251,7 +251,7 @@ def test_trace_operator(
     t2_root_span_id = TraceIdGenerator.span_id()
     t2_child_span_id = TraceIdGenerator.span_id()
 
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
 
     insert_traces(
         [
@@ -580,6 +580,7 @@ def test_trace_operator_with_adjusted_keys(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     payload_factory: Callable[[list[Traces]], list[dict[str, Any]]],
     request_type: str,
@@ -591,9 +592,9 @@ def test_trace_operator_with_adjusted_keys(
     queries, otherwise deprecated keys and context-prefixed aliases don't
     resolve.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
-    traces = generate_traces_with_corrupt_metadata(now)
+    traces = generate_traces_with_corrupt_metadata()
     insert_traces(traces)
     payload = payload_factory(traces)
 
@@ -602,8 +603,8 @@ def test_trace_operator_with_adjusted_keys(
     response = make_query_request(
         signoz,
         token,
-        start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
-        end_ms=int(now.timestamp() * 1000),
+        start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
+        end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
         request_type=request_type,
         queries=payload,
     )
@@ -641,6 +642,7 @@ def test_trace_operator_select_fields(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     select_fields: list[dict[str, Any]],
     expansion: str,
@@ -665,9 +667,9 @@ def test_trace_operator_select_fields(
     - pkg/telemetrytraces/trace_operator_cte_builder.go::buildListQuery for
       the per-row SELECT.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
 
     trace_id = TraceIdGenerator.trace_id()
     parent_span_id = TraceIdGenerator.span_id()
@@ -752,8 +754,8 @@ def test_trace_operator_select_fields(
     response = make_query_request(
         signoz,
         token,
-        start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
-        end_ms=int(now.timestamp() * 1000),
+        start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
+        end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
         request_type=RequestType.RAW,
         queries=queries,
     )
@@ -790,6 +792,7 @@ def test_trace_operator_scalar_limit(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
 ) -> None:
     """
@@ -800,9 +803,9 @@ def test_trace_operator_scalar_limit(
     1. A scalar trace operator grouped by service.name with limit=2 returns
        only the top two groups by count.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     span_counts = {"limit-a": 4, "limit-b": 3, "limit-c": 2, "limit-d": 1}
 
     insert_traces(

tests/integration/tests/queriertraces/09_unknown_keys.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
 
 import pytest
@@ -12,7 +12,6 @@ from fixtures.querier import (
     build_group_by_field,
     build_raw_query,
     build_traces_scalar_query,
-    get_attribute_backend_now,
     get_rows,
     get_scalar_table_data,
     make_query_request,
@@ -29,12 +28,14 @@ def test_traces_filter_unknown_key_synthesizes(
     signoz: types.SigNoz,
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
 ) -> None:
     """An unknown attribute key runs against synthesized columns and returns 200 with a
     "not found" warning so typos still surface."""
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
+    now = datetime.now(tz=UTC)
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
     response = make_query_request(
         signoz,
@@ -62,13 +63,15 @@ def test_traces_filter_unknown_resource_key_filters(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
 ) -> None:
     """A resource-qualified unknown key must FILTER (0 rows), not be dropped: the
     resource-fingerprint sub-query skips keys absent from metadata, so the synthesized
     condition stays in the main query. A regression here returns every span."""
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
+    now = datetime.now(tz=UTC)
     insert_traces(
         [
             Traces(
@@ -114,12 +117,14 @@ def test_traces_group_by_unknown_key_null_bucket(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
 ) -> None:
     """Grouping by an unknown key runs against synthesized columns: every span lands in
     the NULL bucket instead of the query failing."""
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
+    now = datetime.now(tz=UTC)
     insert_traces(
         [
             Traces(

tests/integration/tests/queriertraces/10_filter_operators.py

@@ -1,12 +1,12 @@
 from collections.abc import Callable
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
 
 import pytest
 
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.querier import BuilderQuery, OrderBy, RequestType, TelemetryFieldKey, get_attribute_backend_now, get_rows, make_query_request
+from fixtures.querier import BuilderQuery, OrderBy, RequestType, TelemetryFieldKey, get_rows, make_query_request
 from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
 
 # Filter-operator coverage for traces, modelled on the shapes that dominate real
@@ -70,6 +70,7 @@ def test_traces_filter_operators(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     expression: str,
     expected_names: set[str],
@@ -82,9 +83,9 @@ def test_traces_filter_operators(
     Tests:
     Each production-shaped filter operator selects exactly the expected spans.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     # (name, service, deployment.environment, http.request.method, http.response.status_code, latency_ms, has_error_type)
     specs = [
         ("checkout-handler", "checkout", "prod", "GET", "200", 100, False),

tests/integration/tests/queriertraces/11_aggregation_depth.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
 
 import numpy as np
@@ -12,7 +12,6 @@ from fixtures.querier import (
     build_group_by_field,
     build_order_by,
     build_traces_scalar_query,
-    get_attribute_backend_now,
     get_scalar_table_data,
     make_scalar_query_request,
 )
@@ -34,6 +33,7 @@ def test_traces_aggregate_percentiles(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     noise: str,
 ) -> None:
@@ -49,10 +49,10 @@ def test_traces_aggregate_percentiles(
     intrinsic column rather than error with ClickHouse NO_COMMON_TYPE (386) — the
     regression behind the span_percentile 500.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
     extra_attrs, extra_resources = trace_noise(noise)
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     durations_s = [1, 2, 3, 4, 5]
     spans = [
         Traces(
@@ -101,6 +101,7 @@ def test_traces_aggregate_group_by_non_resource(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     noise: str,
     group_key: str,
@@ -115,10 +116,10 @@ def test_traces_aggregate_group_by_non_resource(
     not just a resource key — buckets by the real value; under the corrupt variant a
     same-named colliding attribute must not divert the grouping.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
     extra_attrs, extra_resources = trace_noise(noise)
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     specs = [("op-a", "/a"), ("op-a", "/a"), ("op-b", "/b")]
     spans = [
         Traces(
@@ -156,6 +157,7 @@ def test_traces_aggregate_multi_group_by(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     noise: str,
 ) -> None:
@@ -168,10 +170,10 @@ def test_traces_aggregate_multi_group_by(
     Grouping by two keys (a resource key and an intrinsic column) returns one row per
     present pair with the correct per-pair count.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
     extra_attrs, extra_resources = trace_noise(noise)
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     pair_counts = {("svc-1", "op-a"): 2, ("svc-1", "op-b"): 1, ("svc-2", "op-a"): 1}
     spans = [
         Traces(
@@ -222,6 +224,7 @@ def test_traces_aggregate_having_breadth(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     noise: str,
     having_expression: str,
@@ -236,10 +239,10 @@ def test_traces_aggregate_having_breadth(
     each keep only the qualifying groups — extending the count()-only HAVING in
     02_aggregation.py.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
     extra_attrs, extra_resources = trace_noise(noise)
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     specs = [("svc-a", 1), ("svc-a", 1), ("svc-a", 1), ("svc-b", 5)]
     spans = [
         Traces(
@@ -285,6 +288,7 @@ def test_traces_duration_nano_qol_filter_with_collision(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
     duration_filter: str,
 ) -> None:
@@ -299,9 +303,9 @@ def test_traces_duration_nano_qol_filter_with_collision(
     ClickHouse NO_COMMON_TYPE (386). The string attribute is cast; the intrinsic column
     stays a bare comparison.
     """
-    now = get_attribute_backend_now(signoz, attribute_backend)
+    use_attribute_backend(attribute_backend)
 
-    now = now.replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     durations_s = [1, 2, 3, 4, 5]
     spans = [
         Traces(

tests/integration/tests/queriertraces/13_attributes_json_evolution.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
 
 from fixtures import types
@@ -14,7 +14,6 @@ from fixtures.querier import (
     build_group_by_field,
     build_traces_scalar_query,
     get_all_warnings,
-    get_attribute_evolution_time,
     get_preview_sql,
     get_rows,
     index_series_by_label,
@@ -29,6 +28,7 @@ def test_traces_attributes_json_evolution(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
     """`http.route` is a dotted key, so the `attributes` JSON column nests it under the path
     http.route while the legacy attributes_string map keys it verbatim. Spans before the attribute
@@ -37,7 +37,8 @@ def test_traces_attributes_json_evolution(
     map+JSON multiIf (straddling), and must return identical rows across the boundary."""
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    evolution_time = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
+    seed_attribute_evolution("traces", evolution_time)
 
     before_2 = evolution_time - timedelta(minutes=10)
     before_1 = evolution_time - timedelta(minutes=5)
@@ -161,13 +162,15 @@ def test_traces_attributes_json_typed_filters(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
     """In the JSON-only window each dotted attribute reads through the nested path with its native
     cast: string (::String), Int64 (toFloat64(...::Nullable(Float64))), Bool (::Nullable(Bool)),
     and existence via the raw path. Filters must select the same rows the Map path would."""
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    evolution_time = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
+    seed_attribute_evolution("traces", evolution_time)
 
     hit = evolution_time + timedelta(minutes=5)
     miss = evolution_time + timedelta(minutes=6)
@@ -227,6 +230,7 @@ def test_traces_attributes_json_collision_and_map_parity(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
     """A name stored under two types (`app.status` as 200 and 'teapot') is one JSON path with
     per-row types. The Map-era behavior must hold on it: group-by reads each row as its stored
@@ -235,7 +239,8 @@ def test_traces_attributes_json_collision_and_map_parity(
     across every stored type."""
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    evolution_time = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
+    seed_attribute_evolution("traces", evolution_time)
 
     hit = evolution_time + timedelta(minutes=5)
     insert_traces(
@@ -338,12 +343,14 @@ def test_traces_attributes_json_number_reads(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
     """A JSON number reads like the attributes_number map: a numeric string as its number, an
     integer above 2^53 rounded, and a non-numeric string as no number."""
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    evolution_time = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
+    seed_attribute_evolution("traces", evolution_time)
 
     hit = evolution_time + timedelta(minutes=5)
     insert_traces(
@@ -393,12 +400,14 @@ def test_traces_attributes_json_list_view(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
     """One straddling list query surfaces each row's attributes bag from its own home
     (legacy maps / dual-written / JSON-only), flattened to dotted keys with native types."""
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    evolution_time = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
+    seed_attribute_evolution("traces", evolution_time)
 
     service = "list-homes-service"
     spans = [
@@ -468,12 +477,14 @@ def test_traces_attributes_json_unreported_key(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
     """An attribute metadata does not report follows the rollout like a reported one: the legacy
     map before it, the attributes JSON column after it."""
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    evolution_time = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
+    seed_attribute_evolution("traces", evolution_time)
 
     before = evolution_time - timedelta(minutes=5)
     after = evolution_time + timedelta(minutes=5)

tests/integration/tests/queriertraces/14_scalar_object_attribute.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable
-from datetime import timedelta
+from datetime import UTC, datetime, timedelta
 from http import HTTPStatus
 
 from fixtures import types
@@ -9,7 +9,6 @@ from fixtures.querier import (
     OrderBy,
     RequestType,
     TelemetryFieldKey,
-    get_attribute_evolution_time,
     get_rows,
     make_query_request,
 )
@@ -21,13 +20,15 @@ def test_traces_scalar_and_object_key_both_survive(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
     """A span attribute stored as both a scalar and an object prefix (db.function and
     db.function.arg) keeps both dotted keys through the native JSON read. json_only writes only the
     JSON column, so no legacy attribute map backfills the keys."""
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    evolution_time = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    evolution_time = datetime.now(tz=UTC).replace(second=0, microsecond=0) - timedelta(minutes=30)
+    seed_attribute_evolution("traces", evolution_time)
 
     service = "scalar-object-service"
     span_time = evolution_time + timedelta(minutes=5)

tests/integration/tests/queriertraces/14_skip_indexes.py

@@ -8,7 +8,7 @@ import requests
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.querier import RequestType, build_aggregation, build_traces_scalar_query, get_preview_statements, make_preview_query_request
-from fixtures.traces import Traces
+from fixtures.traces import ATTRIBUTE_JSON_ROLLOUT_TIME, Traces
 
 PROMOTED_PATHS = "/api/v1/promoted_paths"
 
@@ -18,8 +18,10 @@ def test_traces_filters_use_skip_indexes(
     create_user_admin: types.Operation,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
     request: pytest.FixtureRequest,
 ) -> None:
+    seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
     resp = requests.post(

tests/integration/tests/ruler/08_rule_validation.py

@@ -422,8 +422,8 @@ def test_update_rejects_malformed_id(
         "status": "error",
         "error": {
             "type": "invalid-input",
-            "code": "invalid_selector",
-            "message": "invalid selector for type metaresource: not-a-uuid",
+            "code": "invalid_input",
+            "message": "id is not a valid uuid",
             "errors": [],
             "suggestions": [],
         },

tests/integration/tests/tracedetail/01_summary.py

@@ -7,7 +7,6 @@ import requests
 
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.querier import get_attribute_backend_now, get_attribute_evolution_time
 from fixtures.querierai import root_span
 from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
 
@@ -28,13 +27,15 @@ def test_summary_ai_trace(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    use_attribute_backend: Callable[[str], None],
     attribute_backend: str,
 ) -> None:
     """The summary carries the waterfall's trace-level fields and, for a trace with gen_ai
     spans, token totals over every LLM span and the cost summed over the spans that carry it.
     Spans are written to one layout only, so a read from the wrong column sums to zero."""
-    now = get_attribute_backend_now(signoz, attribute_backend).replace(second=0, microsecond=0)
+    use_attribute_backend(attribute_backend)
     write_mode = "json_only" if attribute_backend == "json" else "legacy_only"
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     service = f"td-summary-{attribute_backend}"
     resources = {"service.name": service}
     trace_id = TraceIdGenerator.trace_id()
@@ -128,11 +129,14 @@ def test_summary_ai_trace_across_json_rollout(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
     """A trace that straddles the attribute JSON rollout has LLM spans written only to the legacy
     maps before it and to the JSON column after it. The summary window covers both, so the gen_ai
     reads must fall back across columns and sum every span."""
-    rollout = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    rollout = now - timedelta(minutes=30)
+    seed_attribute_evolution("traces", rollout)
 
     service = "td-summary-rollout"
     resources = {"service.name": service}

tests/integration/tests/tracedetail/02_thread.py

@@ -8,8 +8,7 @@ import requests
 
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.querier import get_attribute_evolution_time
-from fixtures.traces import TraceIdGenerator, Traces, TracesKind
+from fixtures.traces import ATTRIBUTE_JSON_ROLLOUT_TIME, TraceIdGenerator, Traces, TracesKind
 
 
 def test_thread_returns_gen_ai_spans_in_order(
@@ -17,7 +16,9 @@ def test_thread_returns_gen_ai_spans_in_order(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
+    seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     now = datetime.now(tz=UTC).replace(microsecond=0)
     trace_id = TraceIdGenerator.trace_id()
     root_id, first_llm_id, tool_id, agent_id, second_llm_id, legacy_llm_id, third_llm_id, flattened_llm_id, responses_llm_id, langchain_llm_id, openrouter_llm_id, vercel_llm_id, anthropic_llm_id, gemini_llm_id = (TraceIdGenerator.span_id() for _ in range(14))
@@ -29,7 +30,7 @@ def test_thread_returns_gen_ai_spans_in_order(
 
     insert_traces(
         [
-            Traces(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=9), trace_id=trace_id, span_id=root_id, name="POST /chat", kind=TracesKind.SPAN_KIND_SERVER, resources=resources),
+            Traces(timestamp=now - timedelta(seconds=10), duration=timedelta(seconds=9), trace_id=trace_id, span_id=root_id, name="POST /chat", kind=TracesKind.SPAN_KIND_SERVER, resources=resources, attribute_write_mode="json_only"),
             Traces(
                 timestamp=now - timedelta(seconds=8),
                 trace_id=trace_id,
@@ -38,6 +39,7 @@ def test_thread_returns_gen_ai_spans_in_order(
                 name="chat gpt-4o",
                 resources=resources,
                 attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.system_instructions": first_instructions, "gen_ai.input.messages": first_input, "gen_ai.output.messages": first_output, "gen_ai.response.finish_reasons": json.dumps(["tool_calls"])},
+                attribute_write_mode="json_only",
             ),
             Traces(
                 timestamp=now - timedelta(seconds=6),
@@ -47,9 +49,10 @@ def test_thread_returns_gen_ai_spans_in_order(
                 name="execute_tool get_weather",
                 resources=resources,
                 attributes={"gen_ai.tool.name": "get_weather", "gen_ai.tool.call.id": "call_1", "gen_ai.tool.call.arguments": json.dumps({"city": "Bangalore"}), "gen_ai.tool.call.result": "sunny"},
+                attribute_write_mode="json_only",
             ),
-            Traces(timestamp=now - timedelta(seconds=5), trace_id=trace_id, span_id=agent_id, parent_span_id=root_id, name="invoke_agent planner", resources=resources, attributes={"gen_ai.agent.name": "planner"}),
-            Traces(timestamp=now - timedelta(seconds=4), trace_id=trace_id, span_id=second_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": second_input}),
+            Traces(timestamp=now - timedelta(seconds=5), trace_id=trace_id, span_id=agent_id, parent_span_id=root_id, name="invoke_agent planner", resources=resources, attributes={"gen_ai.agent.name": "planner"}, attribute_write_mode="json_only"),
+            Traces(timestamp=now - timedelta(seconds=4), trace_id=trace_id, span_id=second_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": second_input}, attribute_write_mode="json_only"),
             Traces(
                 timestamp=now - timedelta(seconds=3),
                 trace_id=trace_id,
@@ -58,8 +61,9 @@ def test_thread_returns_gen_ai_spans_in_order(
                 name="openai.chat",
                 resources=resources,
                 attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.prompt.0.role": "user", "gen_ai.prompt.0.content": "and tomorrow?", "gen_ai.completion.0.role": "assistant", "gen_ai.completion.0.content": "Also sunny.", "gen_ai.completion.0.finish_reason": "stop"},
+                attribute_write_mode="json_only",
             ),
-            Traces(timestamp=now - timedelta(seconds=2), trace_id=trace_id, span_id=third_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.output.messages": "It is sunny in Bangalore."}),
+            Traces(timestamp=now - timedelta(seconds=2), trace_id=trace_id, span_id=third_llm_id, parent_span_id=root_id, name="chat gpt-4o", resources=resources, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.output.messages": "It is sunny in Bangalore."}, attribute_write_mode="json_only"),
             Traces(
                 timestamp=now - timedelta(seconds=1),
                 trace_id=trace_id,
@@ -75,6 +79,7 @@ def test_thread_returns_gen_ai_spans_in_order(
                     "llm.output_messages.0.message.content": "You're welcome.",
                     "llm.finish_reason": "stop",
                 },
+                attribute_write_mode="json_only",
             ),
             Traces(
                 timestamp=now,
@@ -88,6 +93,7 @@ def test_thread_returns_gen_ai_spans_in_order(
                     "gen_ai.input.messages": json.dumps({"model": "gpt-4o", "instructions": "Be brief.", "input": [{"role": "user", "content": "bye"}]}),
                     "gen_ai.output.messages": json.dumps({"object": "response", "status": "completed", "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Bye!"}]}]}),
                 },
+                attribute_write_mode="json_only",
             ),
             Traces(
                 timestamp=now + timedelta(seconds=1),
@@ -103,6 +109,7 @@ def test_thread_returns_gen_ai_spans_in_order(
                         {"generations": [[{"text": "See you!", "generation_info": {"finish_reason": "stop"}, "type": "ChatGeneration", "message": {"lc": 1, "type": "constructor", "id": ["langchain", "schema", "messages", "AIMessage"], "kwargs": {"content": "See you!", "type": "ai"}}}]]}
                     ),
                 },
+                attribute_write_mode="json_only",
             ),
             Traces(
                 timestamp=now + timedelta(seconds=2),
@@ -116,6 +123,7 @@ def test_thread_returns_gen_ai_spans_in_order(
                     "gen_ai.input.messages": json.dumps({"messages": [{"role": "user", "content": "one word"}]}),
                     "gen_ai.output.messages": json.dumps({"completion": "Yes.", "reasoning": None, "rawRequest": {"model": "openai/gpt-4o-mini"}}),
                 },
+                attribute_write_mode="json_only",
             ),
             Traces(
                 timestamp=now + timedelta(seconds=3),
@@ -131,6 +139,7 @@ def test_thread_returns_gen_ai_spans_in_order(
                     "gen_ai.input.messages": json.dumps([{"role": "user", "content": [{"type": "text", "text": "last one"}]}]),
                     "gen_ai.output.messages": "Done.",
                 },
+                attribute_write_mode="json_only",
             ),
             Traces(
                 timestamp=now + timedelta(seconds=4),
@@ -144,6 +153,7 @@ def test_thread_returns_gen_ai_spans_in_order(
                     "gen_ai.input.messages": json.dumps({"system": "Be terse.", "messages": [{"role": "user", "content": "ping"}]}),
                     "gen_ai.output.messages": json.dumps({"type": "message", "role": "assistant", "content": [{"type": "text", "text": "pong"}], "stop_reason": "end_turn"}),
                 },
+                attribute_write_mode="json_only",
             ),
             Traces(
                 timestamp=now + timedelta(seconds=5),
@@ -157,6 +167,7 @@ def test_thread_returns_gen_ai_spans_in_order(
                     "gen_ai.input.messages": json.dumps({"contents": [{"role": "user", "parts": [{"text": "ping"}]}]}),
                     "gen_ai.output.messages": json.dumps({"candidates": [{"content": {"role": "model", "parts": [{"text": "pong"}]}, "finishReason": "STOP"}]}),
                 },
+                attribute_write_mode="json_only",
             ),
         ]
     )
@@ -220,7 +231,9 @@ def test_thread_paginates_with_cursors(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
+    seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     now = datetime.now(tz=UTC).replace(microsecond=0)
     trace_id = TraceIdGenerator.trace_id()
     span_ids = [TraceIdGenerator.span_id() for _ in range(3)]
@@ -228,7 +241,7 @@ def test_thread_paginates_with_cursors(
     timestamps = [now - timedelta(seconds=6), now - timedelta(seconds=3), now - timedelta(seconds=3)]
     insert_traces(
         [
-            Traces(timestamp=timestamp, trace_id=trace_id, span_id=span_id, name="chat gpt-4o", resources={"service.name": "tracedetail-thread-pages"}, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])})
+            Traces(timestamp=timestamp, trace_id=trace_id, span_id=span_id, name="chat gpt-4o", resources={"service.name": "tracedetail-thread-pages"}, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])}, attribute_write_mode="json_only")
             for span_id, timestamp in zip(span_ids, timestamps, strict=True)
         ]
     )
@@ -273,7 +286,9 @@ def test_thread_paginates_across_buckets(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
+    seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     now = datetime.now(tz=UTC).replace(microsecond=0)
     bucket = now.replace(minute=0 if now.minute < 30 else 30, second=0)
     trace_id = TraceIdGenerator.trace_id()
@@ -288,7 +303,7 @@ def test_thread_paginates_across_buckets(
     span_ids = [TraceIdGenerator.span_id() for _ in timestamps]
     insert_traces(
         [
-            Traces(timestamp=timestamp, trace_id=trace_id, span_id=span_id, name="chat gpt-4o", resources={"service.name": "tracedetail-thread-buckets"}, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])})
+            Traces(timestamp=timestamp, trace_id=trace_id, span_id=span_id, name="chat gpt-4o", resources={"service.name": "tracedetail-thread-buckets"}, attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])}, attribute_write_mode="json_only")
             for span_id, timestamp in zip(span_ids, timestamps, strict=True)
         ]
     )
@@ -325,7 +340,9 @@ def test_thread_opens_around_span(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
+    seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     now = datetime.now(tz=UTC).replace(microsecond=0)
     trace_id = TraceIdGenerator.trace_id()
     resources = {"service.name": "tracedetail-thread-anchor"}
@@ -335,7 +352,7 @@ def test_thread_opens_around_span(
     # http span, not a thread row, sits between the third and fourth llm spans
     insert_traces(
         [
-            Traces(timestamp=now - timedelta(seconds=20), duration=timedelta(seconds=19), trace_id=trace_id, span_id=root_id, name="POST /chat", kind=TracesKind.SPAN_KIND_SERVER, resources=resources),
+            Traces(timestamp=now - timedelta(seconds=20), duration=timedelta(seconds=19), trace_id=trace_id, span_id=root_id, name="POST /chat", kind=TracesKind.SPAN_KIND_SERVER, resources=resources, attribute_write_mode="json_only"),
             *(
                 Traces(
                     timestamp=now - timedelta(seconds=18 - 3 * i),
@@ -345,10 +362,11 @@ def test_thread_opens_around_span(
                     name="chat gpt-4o",
                     resources=resources,
                     attributes={"gen_ai.request.model": "gpt-4o", "gen_ai.input.messages": json.dumps([{"role": "user", "content": span_id}])},
+                    attribute_write_mode="json_only",
                 )
                 for i, span_id in enumerate(llm_ids)
             ),
-            Traces(timestamp=now - timedelta(seconds=11), trace_id=trace_id, span_id=http_id, parent_span_id=root_id, name="GET /weather", resources=resources, attributes={"http.request.method": "GET"}),
+            Traces(timestamp=now - timedelta(seconds=11), trace_id=trace_id, span_id=http_id, parent_span_id=root_id, name="GET /weather", resources=resources, attributes={"http.request.method": "GET"}, attribute_write_mode="json_only"),
         ]
     )
 
@@ -402,8 +420,11 @@ def test_thread_reads_spans_across_json_rollout(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
-    rollout = get_attribute_evolution_time(signoz, "traces").replace(second=0, microsecond=0)
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    rollout = now - timedelta(minutes=30)
+    seed_attribute_evolution("traces", rollout)
     resources = {"service.name": "tracedetail-thread-rollout"}
 
     # trace entirely before the rollout: messages live only in the legacy maps
@@ -451,14 +472,16 @@ def test_thread_without_gen_ai_spans_is_empty(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
+    seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     trace_id = TraceIdGenerator.trace_id()
     resources = {"service.name": "tracedetail-thread-empty"}
     # messages without a gate key, as a failed call or an unmapped SDK writes them, are not a row either
     insert_traces(
         [
-            Traces(timestamp=datetime.now(tz=UTC) - timedelta(seconds=5), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="GET /health", resources=resources),
-            Traces(timestamp=datetime.now(tz=UTC) - timedelta(seconds=4), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="chat", resources=resources, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": "hi"}])}),
+            Traces(timestamp=datetime.now(tz=UTC) - timedelta(seconds=5), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="GET /health", resources=resources, attribute_write_mode="json_only"),
+            Traces(timestamp=datetime.now(tz=UTC) - timedelta(seconds=4), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="chat", resources=resources, attributes={"gen_ai.input.messages": json.dumps([{"role": "user", "content": "hi"}])}, attribute_write_mode="json_only"),
         ]
     )
 

tests/integration/tests/tracedetail/03_waterfall.py

@@ -7,7 +7,7 @@ import requests
 
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.traces import TraceIdGenerator, Traces, TracesKind
+from fixtures.traces import ATTRIBUTE_JSON_ROLLOUT_TIME, TraceIdGenerator, Traces, TracesKind
 
 
 def test_waterfall_formats_messages_on_gen_ai_spans(
@@ -15,7 +15,9 @@ def test_waterfall_formats_messages_on_gen_ai_spans(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
+    seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     now = datetime.now(tz=UTC).replace(microsecond=0)
     trace_id = TraceIdGenerator.trace_id()
     root_id, llm_id, tool_id, output_only_id = (TraceIdGenerator.span_id() for _ in range(4))

tests/integration/tests/tracedetail/04_attribute_homes.py

@@ -6,7 +6,7 @@ import requests
 
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
-from fixtures.traces import TraceIdGenerator, Traces, TracesKind
+from fixtures.traces import ATTRIBUTE_JSON_ROLLOUT_TIME, TraceIdGenerator, Traces, TracesKind
 
 
 def test_waterfall_and_flamegraph_read_each_attribute_home(
@@ -14,7 +14,9 @@ def test_waterfall_and_flamegraph_read_each_attribute_home(
     create_user_admin: None,  # pylint: disable=unused-argument
     get_token: Callable[[str, str], str],
     insert_traces: Callable[[list[Traces]], None],
+    seed_attribute_evolution: Callable[[str, datetime], None],
 ) -> None:
+    seed_attribute_evolution("traces", ATTRIBUTE_JSON_ROLLOUT_TIME)
     now = datetime.now(tz=UTC).replace(microsecond=0)
     trace_id = TraceIdGenerator.trace_id()
     legacy_id, json_id, dual_id = (TraceIdGenerator.span_id() for _ in range(3))