test(promql): replay the conformance corpus on both providers

Every corpus case now runs twice — default provider, and pinned to
clickhousev2 via the flag-gated X-SigNoz-PromQL-Provider header — each leg
asserted against the same frozen expectations, each leg with its own
known-divergences ledger enforced in both directions. The new
known_divergences_v2.json is the rollout scorecard: the provider swap is
measured by burning it to empty.

The legs are never asserted against each other: both can sit within one
rounding quantum of the expected value yet differ by up to two quanta at a
rounding boundary, so a leg-vs-leg check would reintroduce the boundary
noise the tolerance absorbs. Anchoring both to the same oracle over the
same ingested bytes already localizes any disagreement to a provider.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/integrationci.yaml

@@ -54,6 +54,7 @@ jobs:
           - querierscalar
           - queriercommon
           - rawexportdata
+          - promqlconformance
           - querierauthz
           - role
           - rootuser

frontend/src/api/ErrorResponseHandlerForGeneratedAPIs.ts

@@ -2,6 +2,29 @@ import { RenderErrorResponseDTO } from 'api/generated/services/sigNoz.schemas';
 import { AxiosError } from 'axios';
 import APIError from 'types/api/error';
 
+// The wire shape these handlers can actually rely on. The generated
+// RenderErrorResponseDTO marks code/message/url/errors as required, but the
+// server omits any of them even on valid errors (e.g. a 400 with just a
+// message), so a present `error` object is all the guard can guarantee.
+type ErrorEnvelope = {
+	error: {
+		code?: string;
+		message?: string;
+		url?: string;
+		errors?: { message?: string }[];
+	};
+};
+
+function isErrorEnvelope(data: unknown): data is ErrorEnvelope {
+	return (
+		typeof data === 'object' &&
+		data !== null &&
+		'error' in data &&
+		typeof (data as ErrorEnvelope).error === 'object' &&
+		(data as ErrorEnvelope).error !== null
+	);
+}
+
 // @deprecated Use convertToApiError instead
 export function ErrorResponseHandlerForGeneratedAPIs(
 	error: AxiosError<RenderErrorResponseDTO>,
@@ -10,15 +33,29 @@ export function ErrorResponseHandlerForGeneratedAPIs(
 	// The request was made and the server responded with a status code
 	// that falls out of the range of 2xx
 	if (response) {
+		// The body isn't guaranteed to be an error envelope — e.g. a gateway 5xx
+		// with an HTML/empty body during a deploy. Verify the shape before reading
+		// it; otherwise synthesize a consistent error from the status.
+		const data: unknown = response.data;
+		if (isErrorEnvelope(data)) {
+			const { code, message, url, errors } = data.error;
+			throw new APIError({
+				httpStatusCode: response.status || 500,
+				error: {
+					code: code ?? '',
+					message: message ?? '',
+					url: url ?? '',
+					errors: (errors ?? []).map((e) => ({ message: e.message ?? '' })),
+				},
+			});
+		}
 		throw new APIError({
 			httpStatusCode: response.status || 500,
 			error: {
-				code: response.data.error.code,
-				message: response.data.error.message,
-				url: response.data.error.url ?? '',
-				errors: (response.data.error.errors ?? []).map((e) => ({
-					message: e.message ?? '',
-				})),
+				code: 'UPSTREAM_UNAVAILABLE',
+				message: error.message || 'Something went wrong',
+				url: '',
+				errors: [],
 			},
 		});
 	}
@@ -62,9 +99,7 @@ export function convertToApiError(
 	return new APIError({
 		httpStatusCode: response?.status || error.status || 500,
 		error: {
-			code:
-				errorData?.code ||
-				String(response?.status || error.code || 'unknown_error'),
+			code: errorData?.code || 'UPSTREAM_UNAVAILABLE',
 			message:
 				errorData?.message ||
 				response?.statusText ||

frontend/src/api/ErrorResponseHandlerV2.ts

@@ -2,19 +2,38 @@ import { AxiosError } from 'axios';
 import { ErrorV2Resp } from 'types/api';
 import APIError from 'types/api/error';
 
+function isErrorV2Resp(data: unknown): data is ErrorV2Resp {
+	return (
+		typeof data === 'object' &&
+		data !== null &&
+		'error' in data &&
+		typeof (data as ErrorV2Resp).error?.code === 'string'
+	);
+}
+
 // reference - https://axios-http.com/docs/handling_errors
 export function ErrorResponseHandlerV2(error: AxiosError<ErrorV2Resp>): never {
 	const { response, request } = error;
 	// The request was made and the server responded with a status code
 	// that falls out of the range of 2xx
 	if (response) {
+		// response.data isn't guaranteed to be a V2 envelope (e.g. a gateway 5xx
+		// with an HTML/empty body during a deploy), so verify the shape first.
+		const data: unknown = response.data;
+		if (isErrorV2Resp(data)) {
+			const { code, message, url, errors } = data.error;
+			throw new APIError({
+				httpStatusCode: response.status || 500,
+				error: { code, message, url, errors },
+			});
+		}
 		throw new APIError({
 			httpStatusCode: response.status || 500,
 			error: {
-				code: response.data.error.code,
-				message: response.data.error.message,
-				url: response.data.error.url,
-				errors: response.data.error.errors,
+				code: 'UPSTREAM_UNAVAILABLE',
+				message: error.message || 'Something went wrong',
+				url: '',
+				errors: [],
 			},
 		});
 	}

frontend/src/api/__tests__/ErrorResponseHandlerV2.test.ts

@@ -0,0 +1,126 @@
+import { ErrorResponseHandlerV2 } from 'api/ErrorResponseHandlerV2';
+import { AxiosError } from 'axios';
+import { ErrorV2Resp } from 'types/api';
+import APIError from 'types/api/error';
+
+function asAxiosError(partial: Partial<AxiosError>): AxiosError<ErrorV2Resp> {
+	return partial as AxiosError<ErrorV2Resp>;
+}
+
+// ErrorResponseHandlerV2 always throws — capture the APIError so assertions stay
+// unconditional.
+function runHandler(error: AxiosError<ErrorV2Resp>): APIError {
+	try {
+		ErrorResponseHandlerV2(error);
+	} catch (thrown) {
+		return thrown as APIError;
+	}
+	throw new Error('expected ErrorResponseHandlerV2 to throw');
+}
+
+type ExpectedError = {
+	httpStatusCode: number;
+	code: string;
+	message: string;
+	errors: { message: string }[];
+};
+
+// One row per response shape the handler must normalize. New shapes (with
+// different bodies) can be added here without a new test block.
+const cases: {
+	name: string;
+	error: AxiosError<ErrorV2Resp>;
+	expected: ExpectedError;
+}[] = [
+	{
+		name: 'well-formed V2 error envelope',
+		error: asAxiosError({
+			message: 'Request failed with status code 400',
+			response: {
+				status: 400,
+				data: {
+					error: {
+						code: 'bad_request',
+						message: 'Invalid dashboard payload',
+						url: 'https://signoz.io/docs',
+						errors: [{ message: 'name is required' }, { message: 'name too long' }],
+					},
+				},
+			} as AxiosError['response'],
+		}),
+		expected: {
+			httpStatusCode: 400,
+			code: 'bad_request',
+			message: 'Invalid dashboard payload',
+			errors: [{ message: 'name is required' }, { message: 'name too long' }],
+		},
+	},
+	{
+		// Regression: during a deployment the gateway returns a 5xx with a
+		// non-envelope body. Reading response.data.error.code used to throw a
+		// TypeError from inside the handler itself. See engineering-pod#5760.
+		name: '5xx with a non-envelope HTML body',
+		error: asAxiosError({
+			message: 'Request failed with status code 503',
+			response: {
+				status: 503,
+				data: '<html><body>503 Service Temporarily Unavailable</body></html>',
+			} as AxiosError['response'],
+		}),
+		expected: {
+			httpStatusCode: 503,
+			code: 'UPSTREAM_UNAVAILABLE',
+			message: 'Request failed with status code 503',
+			errors: [],
+		},
+	},
+	{
+		name: '5xx with an empty body',
+		error: asAxiosError({
+			message: 'Request failed with status code 502',
+			response: {
+				status: 502,
+				data: undefined,
+			} as AxiosError['response'],
+		}),
+		expected: {
+			httpStatusCode: 502,
+			code: 'UPSTREAM_UNAVAILABLE',
+			message: 'Request failed with status code 502',
+			errors: [],
+		},
+	},
+	{
+		name: 'no response received (network error)',
+		error: asAxiosError({
+			message: 'Network Error',
+			code: 'ERR_NETWORK',
+			name: 'AxiosError',
+			request: {},
+		}),
+		expected: {
+			httpStatusCode: 500,
+			code: 'ERR_NETWORK',
+			message: 'Network Error',
+			errors: [],
+		},
+	},
+];
+
+describe('ErrorResponseHandlerV2', () => {
+	it.each(cases)(
+		'normalizes $name into a consistent APIError',
+		({ error, expected }) => {
+			const apiError = runHandler(error);
+
+			expect(apiError).toBeInstanceOf(APIError);
+			expect(apiError.getHttpStatusCode()).toBe(expected.httpStatusCode);
+			expect(apiError.getErrorCode()).toBe(expected.code);
+			expect(apiError.getErrorMessage()).toBe(expected.message);
+			// The sub-error messages feed several parts of the UI, so assert them.
+			expect(apiError.getErrorDetails().error.errors).toStrictEqual(
+				expected.errors,
+			);
+		},
+	);
+});

frontend/src/components/FieldsSelector/OtherFields.tsx

@@ -8,7 +8,6 @@ import { buildCompositeKey } from 'container/OptionsMenu/utils';
 import { useGetQueryKeySuggestions } from 'hooks/querySuggestions/useGetQueryKeySuggestions';
 import {
 	FieldContext,
-	FieldDataType,
 	SignalType,
 	TelemetryFieldKey,
 } from 'types/api/v5/queryRange';
@@ -55,7 +54,7 @@ function OtherFields({
 				key: buildCompositeKey(attr.name, attr.fieldContext as string),
 				signal: attr.signal as SignalType,
 				fieldContext: attr.fieldContext as FieldContext,
-				fieldDataType: attr.fieldDataType as FieldDataType,
+				fieldDataType: attr.fieldDataType,
 			}),
 		);
 		const addedIds = new Set(

frontend/src/components/QueryBuilderV2/QueryV2/QueryAggregation/QueryAggregationSelect.tsx

@@ -25,12 +25,12 @@ import CodeMirror, {
 } from '@uiw/react-codemirror';
 import { Button, Popover, Tooltip } from 'antd';
 import { getKeySuggestions } from 'api/querySuggestions/getKeySuggestions';
-import { QUERY_BUILDER_KEY_TYPES } from 'constants/antlrQueryConstants';
 import { QueryBuilderKeys } from 'constants/queryBuilder';
 import { tracesAggregateOperatorOptions } from 'constants/queryBuilderOperators';
 import { useIsDarkMode } from 'hooks/useDarkMode';
 import { Info, TriangleAlert } from '@signozhq/icons';
 import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
+import { FieldDataType } from 'types/api/v5/queryRange';
 import { TracesAggregatorOperator } from 'types/common/queryBuilder';
 
 import { useQueryBuilderV2Context } from '../../QueryBuilderV2Context';
@@ -323,16 +323,16 @@ function QueryAggregationSelect({
 				TracesAggregatorOperator.RATE,
 			];
 
-			const fieldDataType =
+			const fieldDataType: FieldDataType | undefined =
 				functionContextForFetch &&
 				operatorsWithoutDataType.includes(functionContextForFetch)
 					? undefined
-					: QUERY_BUILDER_KEY_TYPES.NUMBER;
+					: 'number';
 
 			return getKeySuggestions({
 				signal: queryData.dataSource,
 				searchText: '',
-				fieldDataType: fieldDataType as QUERY_BUILDER_KEY_TYPES,
+				fieldDataType,
 			});
 		},
 		{

frontend/src/components/QuickFilters/FilterRenderers/Checkbox/Checkbox.styles.scss

@@ -92,52 +92,76 @@
 						color: var(--l3-foreground);
 					}
 
-					.only-btn {
-						cursor: not-allowed;
-						color: var(--l3-foreground);
-					}
-
+					.only-btn,
 					.toggle-btn {
 						cursor: not-allowed;
 						color: var(--l3-foreground);
 					}
 				}
 
-				.only-btn {
-					display: none;
+				// Stack "Only"/"All" and "Toggle" in a single cell so revealing
+				// one swaps in place instead of shifting the row layout.
+				.value-actions {
+					display: grid;
+					align-items: center;
+					justify-items: end;
+
+					> * {
+						grid-area: 1 / 1;
+					}
 				}
+
+				.only-btn,
 				.toggle-btn {
 					display: none;
-				}
-
-				.toggle-btn:hover {
-					background-color: unset;
-				}
-
-				.only-btn:hover {
-					background-color: unset;
-				}
-			}
-
-			.checkbox-value-section:hover {
-				.toggle-btn {
-					display: none;
-				}
-				.only-btn {
-					display: flex;
 					align-items: center;
 					justify-content: center;
 					height: 21px;
+					opacity: 0;
+					transform: translateX(4px);
+					// `display` is animated as a discrete property so the button
+					// stays mounted through its fade-out on exit.
+					transition:
+						opacity 0.16s ease,
+						transform 0.16s ease,
+						display 0.16s allow-discrete;
+
+					&:hover {
+						background-color: unset;
+					}
+				}
+			}
+
+			// Hovering the label/value area reveals the "Only"/"All" action.
+			.checkbox-value-section:hover .only-btn {
+				display: flex;
+				opacity: 1;
+				transform: translateX(0);
+
+				// Entry animation start state (fires on the display switch).
+				@starting-style {
+					opacity: 0;
+					transform: translateX(4px);
+				}
+			}
+
+			// Hovering the checkbox reveals the "Toggle" action.
+			.check-box:hover ~ .checkbox-value-section .toggle-btn {
+				display: flex;
+				opacity: 1;
+				transform: translateX(0);
+
+				@starting-style {
+					opacity: 0;
+					transform: translateX(4px);
 				}
 			}
 		}
 
-		.value:hover {
+		@media (prefers-reduced-motion: reduce) {
+			.only-btn,
 			.toggle-btn {
-				display: flex;
-				align-items: center;
-				justify-content: center;
-				height: 21px;
+				transition: none;
 			}
 		}
 	}

frontend/src/components/QuickFilters/FilterRenderers/Checkbox/CheckboxValueRow.tsx

@@ -53,12 +53,14 @@ function CheckboxValueRow({
 						</Typography.Text>
 					</TooltipSimple>
 				)}
-				<Button type="text" className="only-btn">
-					{onlyButtonLabel}
-				</Button>
-				<Button type="text" className="toggle-btn">
-					Toggle
-				</Button>
+				<div className="value-actions">
+					<Button type="text" className="only-btn">
+						{onlyButtonLabel}
+					</Button>
+					<Button type="text" className="toggle-btn">
+						Toggle
+					</Button>
+				</div>
 			</div>
 		</div>
 	);

frontend/src/components/WarningPopover/WarningPopover.styles.scss

@@ -1,6 +1,11 @@
+.warning-popover-overlay {
+	--antd-arrow-background-color: var(--l2-background);
+}
+
 .warning-content {
 	display: flex;
 	flex-direction: column;
+	background-color: var(--l2-background);
 
 	// === SECTION: Summary (Top)
 	&__summary-section {
@@ -10,16 +15,25 @@
 
 	&__summary {
 		display: flex;
+		align-items: flex-start;
 		justify-content: space-between;
+		gap: var(--spacing-8);
 		padding: 16px;
 	}
 
 	&__summary-left {
 		display: flex;
-		align-items: baseline;
+		align-items: flex-start;
 		gap: 8px;
 	}
 
+	&__icon-wrapper {
+		display: flex;
+		align-items: center;
+		flex-shrink: 0;
+		height: var(--spacing-12);
+	}
+
 	&__summary-text {
 		display: flex;
 		flex-direction: column;

frontend/src/components/WarningPopover/WarningPopover.tsx

@@ -2,6 +2,7 @@ import { ReactNode, useMemo } from 'react';
 import { Color } from '@signozhq/design-tokens';
 import { Button, Popover, PopoverProps } from 'antd';
 import ErrorIcon from 'assets/Error';
+import cx from 'classnames';
 import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
 import { BookOpenText, ChevronsDown, TriangleAlert } from '@signozhq/icons';
 import KeyValueLabel from 'periscope/components/KeyValueLabel';
@@ -30,16 +31,18 @@ export function WarningContent({ warning }: WarningContentProps): JSX.Element {
 			{/* Summary Header */}
 			<section className="warning-content__summary-section">
 				<header className="warning-content__summary">
-					<div className="warning-content__summary-left">
-						<div className="warning-content__icon-wrapper">
-							<ErrorIcon />
-						</div>
+					{(warningCode || warningMessage) && (
+						<div className="warning-content__summary-left">
+							<div className="warning-content__icon-wrapper">
+								<ErrorIcon />
+							</div>
 
-						<div className="warning-content__summary-text">
-							<h2 className="warning-content__warning-code">{warningCode}</h2>
-							<p className="warning-content__warning-message">{warningMessage}</p>
+							<div className="warning-content__summary-text">
+								<h2 className="warning-content__warning-code">{warningCode}</h2>
+								<p className="warning-content__warning-message">{warningMessage}</p>
+							</div>
 						</div>
-					</div>
+					)}
 
 					{warningUrl && (
 						<div className="warning-content__summary-right">
@@ -154,6 +157,10 @@ function WarningPopover({
 			overlayInnerStyle={{ padding: 0 }}
 			autoAdjustOverflow
 			{...popoverProps}
+			overlayClassName={cx(
+				'warning-popover-overlay',
+				popoverProps.overlayClassName,
+			)}
 		>
 			{children || (
 				<TriangleAlert

frontend/src/container/MetricsExplorer/MetricDetails/DashboardsAndAlertsPopover.tsx

@@ -6,7 +6,7 @@ import { Skeleton } from 'antd';
 import { Typography } from '@signozhq/ui/typography';
 import {
 	useGetMetricAlerts,
-	useGetMetricDashboards,
+	useGetMetricDashboardsV2,
 } from 'api/generated/services/metrics';
 import { QueryParams } from 'constants/query';
 import ROUTES from 'constants/routes';
@@ -38,7 +38,7 @@ function DashboardsAndAlertsPopover({
 		data: dashboardsData,
 		isLoading: isLoadingDashboards,
 		isError: isErrorDashboards,
-	} = useGetMetricDashboards(
+	} = useGetMetricDashboardsV2(
 		{
 			metricName,
 		},
@@ -55,7 +55,8 @@ function DashboardsAndAlertsPopover({
 
 	const dashboards = useMemo(() => {
 		const currentDashboards = dashboardsData?.data.dashboards ?? [];
-		// Remove duplicate dashboards
+		// The API returns one entry per referencing panel, so a dashboard repeats
+		// once per panel that uses the metric.
 		return currentDashboards.filter(
 			(dashboard, index, self) =>
 				index === self.findIndex((t) => t.dashboardId === dashboard.dashboardId),

frontend/src/container/MetricsExplorer/MetricDetails/__tests__/DashboardsAndAlertsPopover.test.tsx

@@ -23,7 +23,7 @@ const useGetMetricAlertsMock = jest.spyOn(
 );
 const useGetMetricDashboardsMock = jest.spyOn(
 	metricsExplorerHooks,
-	'useGetMetricDashboards',
+	'useGetMetricDashboardsV2',
 );
 
 describe('DashboardsAndAlertsPopover', () => {
@@ -153,11 +153,15 @@ describe('DashboardsAndAlertsPopover', () => {
 		);
 	});
 
-	it('renders unique dashboards even when there are duplicates', async () => {
+	it('collapses multiple panels of the same dashboard into one entry', async () => {
 		useGetMetricDashboardsMock.mockReturnValue(
 			getMockDashboardsData({
 				data: {
-					dashboards: [MOCK_DASHBOARD_1, MOCK_DASHBOARD_2, MOCK_DASHBOARD_1],
+					dashboards: [
+						MOCK_DASHBOARD_1,
+						MOCK_DASHBOARD_2,
+						{ ...MOCK_DASHBOARD_1, panelId: '3', panelName: 'Panel 3' },
+					],
 				},
 			}),
 		);

frontend/src/container/MetricsExplorer/MetricDetails/__tests__/testUtlls.ts

@@ -2,7 +2,7 @@ import * as metricsExplorerHooks from 'api/generated/services/metrics';
 import {
 	GetMetricAlerts200,
 	GetMetricAttributes200,
-	GetMetricDashboards200,
+	GetMetricDashboardsV2200,
 	GetMetricHighlights200,
 	GetMetricMetadata200,
 	MetrictypesTemporalityDTO,
@@ -40,14 +40,14 @@ export function getMockMetricHighlightsData(
 export const MOCK_DASHBOARD_1 = {
 	dashboardName: 'Dashboard 1',
 	dashboardId: '1',
-	widgetId: '1',
-	widgetName: 'Widget 1',
+	panelId: '1',
+	panelName: 'Panel 1',
 };
 export const MOCK_DASHBOARD_2 = {
 	dashboardName: 'Dashboard 2',
 	dashboardId: '2',
-	widgetId: '2',
-	widgetName: 'Widget 2',
+	panelId: '2',
+	panelName: 'Panel 2',
 };
 export const MOCK_ALERT_1 = {
 	alertName: 'Alert 1',
@@ -59,7 +59,7 @@ export const MOCK_ALERT_2 = {
 };
 
 export function getMockDashboardsData(
-	overrides?: Partial<GetMetricDashboards200>,
+	overrides?: Partial<GetMetricDashboardsV2200>,
 	{
 		isLoading = false,
 		isError = false,
@@ -67,7 +67,7 @@ export function getMockDashboardsData(
 		isLoading?: boolean;
 		isError?: boolean;
 	} = {},
-): ReturnType<typeof metricsExplorerHooks.useGetMetricDashboards> {
+): ReturnType<typeof metricsExplorerHooks.useGetMetricDashboardsV2> {
 	return {
 		data: {
 			data: {
@@ -79,7 +79,7 @@ export function getMockDashboardsData(
 
 		isLoading,
 		isError,
-	} as ReturnType<typeof metricsExplorerHooks.useGetMetricDashboards>;
+	} as ReturnType<typeof metricsExplorerHooks.useGetMetricDashboardsV2>;
 }
 
 export function getMockAlertsData(

frontend/src/container/OptionsMenu/useOptionsMenu.ts

@@ -113,7 +113,7 @@ const useOptionsMenu = ({
 					(suggestion) => ({
 						name: suggestion.name,
 						signal: suggestion.signal as SignalType,
-						fieldDataType: suggestion.fieldDataType as FieldDataType,
+						fieldDataType: suggestion.fieldDataType,
 						fieldContext: suggestion.fieldContext as FieldContext,
 					}),
 				);
@@ -192,7 +192,7 @@ const useOptionsMenu = ({
 				name: e.name,
 				signal: e.signal as SignalType,
 				fieldContext: e.fieldContext as FieldContext,
-				fieldDataType: e.fieldDataType as FieldDataType,
+				fieldDataType: e.fieldDataType,
 			}));
 		}
 		if (dataSource === DataSource.TRACES) {

frontend/src/container/QueryTable/Drilldown/BreakoutOptions.tsx

@@ -4,11 +4,11 @@ import { Input } from '@signozhq/ui/input';
 import { Skeleton } from 'antd';
 import { getKeySuggestions } from 'api/querySuggestions/getKeySuggestions';
 import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
-import { QUERY_BUILDER_KEY_TYPES } from 'constants/antlrQueryConstants';
 import useDebounce from 'hooks/useDebounce';
 import { ContextMenu } from 'periscope/components/ContextMenu';
 import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
 import { MetricAggregation } from 'types/api/v5/queryRange';
+import { fieldDataTypeToDataType } from 'utils/fieldDataType';
 
 import { BreakoutOptionsProps } from './contextConfig';
 import { BreakoutAttributeType } from './types';
@@ -80,7 +80,7 @@ function BreakoutOptions({
 			keyArray.forEach((keyData) => {
 				transformedOptions.push({
 					key: keyData.name,
-					dataType: keyData.fieldDataType as QUERY_BUILDER_KEY_TYPES,
+					dataType: fieldDataTypeToDataType(keyData.fieldDataType),
 					type: '',
 				});
 			});

frontend/src/container/QueryTable/Drilldown/__tests__/Breakout.test.tsx

@@ -238,6 +238,10 @@ describe('TableDrilldown Breakout Functionality', () => {
 		expect(aggregateQueryData.groupBy).toHaveLength(1);
 		expect(aggregateQueryData.groupBy[0].key).toBe('deployment.environment');
 
+		// The picked field's type travels with it — dropping it leaves the breakout query
+		// untyped, so a drilldown on its result can't tell a number from a string.
+		expect(aggregateQueryData.groupBy[0].dataType).toBe('string');
+
 		// Verify that orderBy has been cleared (as per getBreakoutQuery logic)
 		expect(aggregateQueryData.orderBy).toStrictEqual([]);
 

frontend/src/container/QueryTable/Drilldown/tableDrilldownUtils.tsx

@@ -1,6 +1,5 @@
 import { OPERATORS, PANEL_TYPES } from 'constants/queryBuilder';
 import cloneDeep from 'lodash-es/cloneDeep';
-import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
 import { IBuilderQuery, Query } from 'types/api/queryBuilder/queryBuilderData';
 
 import { addFilterToSelectedQuery, FilterData } from './drilldownUtils';
@@ -89,7 +88,11 @@ export const getBreakoutQuery = (
 		.filter((item: IBuilderQuery) => item.queryName === aggregateData.queryName)
 		.map((item: IBuilderQuery) => ({
 			...item,
-			groupBy: [{ key: groupBy.key, type: groupBy.type } as BaseAutocompleteData],
+			// The picked field's type travels with it: dropped, the breakout query goes out
+			// untyped and a drilldown on its result can't tell a number from a string.
+			groupBy: [
+				{ key: groupBy.key, dataType: groupBy.dataType, type: groupBy.type },
+			],
 			orderBy: [],
 			legend: item.legend && groupBy.key ? `{{${groupBy.key}}}` : '',
 		}));

frontend/src/container/QueryTable/Drilldown/types.ts

@@ -1,6 +1,8 @@
 import { ReactNode } from 'react';
-import { QUERY_BUILDER_KEY_TYPES } from 'constants/antlrQueryConstants';
-import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
+import {
+	BaseAutocompleteData,
+	DataTypes,
+} from 'types/api/queryBuilder/queryAutocompleteResponse';
 import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
 
 export type ContextMenuItem = ReactNode;
@@ -31,7 +33,8 @@ export interface AggregateContextMenuConfig {
 
 export interface BreakoutAttributeType {
 	key: string;
-	dataType: QUERY_BUILDER_KEY_TYPES;
+	/** The picked field's type, in the vocabulary the group-by it becomes is read with. */
+	dataType: DataTypes;
 	type: string;
 }
 

frontend/src/pages/DashboardPageV2/DashboardContainer/DashboardPageToolbar/DashboardActions/DashboardActions.module.scss

@@ -11,3 +11,9 @@
 	border: 1px solid var(--l3-border) !important;
 	box-shadow: none !important;
 }
+
+/* Highlights the dashboard name inside the delete-confirm title. */
+.deleteName {
+	color: var(--danger-background);
+	font-weight: var(--font-weight-medium);
+}

frontend/src/pages/DashboardPageV2/DashboardContainer/DashboardPageToolbar/DashboardActions/DashboardActions.tsx

@@ -27,16 +27,13 @@ import logEvent from 'api/common/logEvent';
 import { cloneDashboardV2 } from 'api/generated/services/dashboard';
 import type { DashboardtypesGettableDashboardV2DTO } from 'api/generated/services/sigNoz.schemas';
 import ROUTES from 'constants/routes';
-import { useDeleteDashboard } from 'hooks/dashboard/useDeleteDashboard';
 import { useSafeNavigate } from 'hooks/useSafeNavigate';
-import history from 'lib/history';
 import { DashboardDetailEvents } from 'pages/DashboardPageV2/constants/events';
 import { useAppContext } from 'providers/App/App';
 import { useErrorModal } from 'providers/ErrorModalProvider';
 import APIError from 'types/api/error';
 import { USER_ROLES } from 'types/roles';
 
-import ConfirmDeleteDialog from '../../components/ConfirmDeleteDialog/ConfirmDeleteDialog';
 import DisabledControlTooltip from '../../components/DisabledControlTooltip/DisabledControlTooltip';
 import DisabledMenuItemLabel from '../../components/DisabledMenuItemLabel/DisabledMenuItemLabel';
 import DashboardSettings from '../../DashboardSettings';
@@ -45,6 +42,7 @@ import SectionTitleModal from '../../PanelsAndSectionsLayout/Section/SectionTitl
 import JsonEditorDrawer from '../JsonEditorDrawer/JsonEditorDrawer';
 import SettingsDrawer from '../SettingsDrawer';
 import styles from './DashboardActions.module.scss';
+import { useDeleteDashboardAction } from './useDeleteDashboardAction';
 import { DASHBOARD_LOCKED_REASON } from '../../hooks/useDashboardEditGuard';
 import { useDashboardStore } from '../../store/useDashboardStore';
 
@@ -84,8 +82,12 @@ function DashboardActions({
 	const [isCloning, setIsCloning] = useState<boolean>(false);
 	const [isNewSectionOpen, setIsNewSectionOpen] = useState<boolean>(false);
 
-	const [isDeleteOpen, setIsDeleteOpen] = useState<boolean>(false);
-	const deleteDashboardMutation = useDeleteDashboard(dashboard.id);
+	const { contextHolder: deleteConfirmHolder, confirmDeleteDashboard } =
+		useDeleteDashboardAction({
+			dashboardId: dashboard.id,
+			dashboardName: title,
+			panelCount: Object.keys(dashboard.spec.panels).length,
+		});
 
 	// Open the settings drawer when something in the tree requests it (e.g. the
 	// variables bar's "Add variable" button).
@@ -132,19 +134,6 @@ function DashboardActions({
 		}
 	}, [dashboard.id, title, safeNavigate, showErrorModal]);
 
-	const handleConfirmDelete = useCallback((): void => {
-		deleteDashboardMutation.mutate(undefined, {
-			onSuccess: () => {
-				void logEvent(DashboardDetailEvents.Deleted, {
-					dashboardId: dashboard.id,
-					panelCount: Object.keys(dashboard.spec.panels).length,
-				});
-				setIsDeleteOpen(false);
-				history.replace(ROUTES.ALL_DASHBOARD);
-			},
-		});
-	}, [deleteDashboardMutation, dashboard.id, dashboard.spec.panels]);
-
 	const handleOpenSettings = useCallback((): void => {
 		void logEvent(DashboardDetailEvents.SettingsOpened, {
 			dashboardId: dashboard.id,
@@ -248,7 +237,7 @@ function DashboardActions({
 					icon: <Trash2 size={14} />,
 					danger: true,
 					disabled: isLocked,
-					onClick: (): void => setIsDeleteOpen(true),
+					onClick: confirmDeleteDashboard,
 				},
 			);
 		}
@@ -266,6 +255,7 @@ function DashboardActions({
 		handleClone,
 		onLockToggle,
 		handleEnterFullScreen,
+		confirmDeleteDashboard,
 	]);
 
 	return (
@@ -348,14 +338,7 @@ function DashboardActions({
 				isOpen={isJsonEditorOpen}
 				onClose={(): void => setIsJsonEditorOpen(false)}
 			/>
-			<ConfirmDeleteDialog
-				open={isDeleteOpen}
-				title={`Delete dashboard"?`}
-				description={`Are you sure you want to delete this dashboard - "${title}"? This action cannot be undone.`}
-				isLoading={deleteDashboardMutation.isLoading}
-				onConfirm={handleConfirmDelete}
-				onClose={(): void => setIsDeleteOpen(false)}
-			/>
+			{deleteConfirmHolder}
 			<SectionTitleModal
 				open={isNewSectionOpen}
 				heading="New section"

frontend/src/pages/DashboardPageV2/DashboardContainer/DashboardPageToolbar/DashboardActions/useDeleteDashboardAction.tsx

@@ -0,0 +1,89 @@
+import { type ReactNode, useCallback } from 'react';
+import { useQueryClient } from 'react-query';
+import { toast } from '@signozhq/ui/sonner';
+import { Typography } from '@signozhq/ui/typography';
+import logEvent from 'api/common/logEvent';
+import {
+	invalidateListDashboardsForUserV2,
+	useDeleteDashboardV2,
+} from 'api/generated/services/dashboard';
+import { useDeleteConfirm } from 'components/DeleteConfirmModal/useDeleteConfirm';
+import ROUTES from 'constants/routes';
+import { useDashboardPreferencesStore } from 'hooks/dashboard/useDashboardPreference';
+import history from 'lib/history';
+import { DashboardDetailEvents } from 'pages/DashboardPageV2/constants/events';
+import { useErrorModal } from 'providers/ErrorModalProvider';
+import APIError from 'types/api/error';
+
+import styles from './DashboardActions.module.scss';
+
+interface UseDeleteDashboardActionArgs {
+	dashboardId: string;
+	dashboardName: string;
+	panelCount: number;
+}
+
+interface UseDeleteDashboardAction {
+	/** Must be rendered in the calling component for the modal to appear. */
+	contextHolder: ReactNode;
+	confirmDeleteDashboard: () => void;
+}
+
+/**
+ * Deletes the open dashboard through the v2 endpoint behind the shared
+ * destructive confirmation, then drops its local preferences, refreshes the
+ * dashboard list cache and sends the user back to the list.
+ */
+export function useDeleteDashboardAction({
+	dashboardId,
+	dashboardName,
+	panelCount,
+}: UseDeleteDashboardActionArgs): UseDeleteDashboardAction {
+	const queryClient = useQueryClient();
+	const { showErrorModal } = useErrorModal();
+	const { contextHolder, confirmDelete } = useDeleteConfirm();
+	const removePreferences = useDashboardPreferencesStore(
+		(state) => state.removePreferences,
+	);
+
+	const { mutate: deleteDashboard } = useDeleteDashboardV2({
+		mutation: {
+			onSuccess: async (): Promise<void> => {
+				void logEvent(DashboardDetailEvents.Deleted, { dashboardId, panelCount });
+				removePreferences(dashboardId);
+				await invalidateListDashboardsForUserV2(queryClient);
+				toast.success('Dashboard deleted successfully');
+				history.replace(ROUTES.ALL_DASHBOARD);
+			},
+			onError: (error: unknown): void => {
+				showErrorModal(error as APIError);
+			},
+		},
+	});
+
+	const confirmDeleteDashboard = useCallback((): void => {
+		confirmDelete({
+			title: (
+				<Typography.Title level={5}>
+					Are you sure you want to delete the
+					<Typography.Text className={styles.deleteName}>
+						{' '}
+						{dashboardName}{' '}
+					</Typography.Text>
+					dashboard?
+				</Typography.Title>
+			),
+			content: 'This action cannot be undone.',
+			// Keeps the Delete button loading until the mutation settles, then closes.
+			onConfirm: () =>
+				new Promise<void>((resolve) => {
+					deleteDashboard(
+						{ pathParams: { id: dashboardId } },
+						{ onSettled: () => resolve() },
+					);
+				}),
+		});
+	}, [confirmDelete, dashboardName, deleteDashboard, dashboardId]);
+
+	return { contextHolder, confirmDeleteDashboard };
+}

frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/Header/Header.tsx

@@ -1,5 +1,6 @@
 import { useCallback } from 'react';
 import { SolidAlertTriangle, X } from '@signozhq/icons';
+import { Badge } from '@signozhq/ui/badge';
 import { Button } from '@signozhq/ui/button';
 import { DialogWrapper } from '@signozhq/ui/dialog';
 import { Divider } from '@signozhq/ui/divider';
@@ -13,6 +14,7 @@ import DisabledControlTooltip from '../../components/DisabledControlTooltip/Disa
 import styles from './Header.module.scss';
 
 interface HeaderProps {
+	/** Unsaved edits exist — shows the "Unsaved Changes" badge and gates the discard confirmation on close (not the Save button). */
 	isDirty: boolean;
 	isSaving: boolean;
 	showSwitchToView?: boolean;
@@ -66,6 +68,11 @@ function Header({
 				/>
 				<Divider type="vertical" />
 				<Typography.Text>Configure panel</Typography.Text>
+				{isDirty && (
+					<Badge color="warning" data-testid="panel-editor-v2-unsaved-badge">
+						Unsaved Changes
+					</Badge>
+				)}
 			</div>
 			<div className={styles.actions}>
 				<HeaderRightSection
@@ -88,7 +95,7 @@ function Header({
 						variant="solid"
 						color="primary"
 						data-testid="panel-editor-v2-save"
-						disabled={readOnly || !isDirty || isSaving}
+						disabled={readOnly || isSaving}
 						loading={!readOnly && isSaving}
 						onClick={readOnly ? undefined : onSave}
 					>

frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/Header/__tests__/Header.test.tsx

@@ -1,3 +1,4 @@
+import type { ComponentProps } from 'react';
 import { MemoryRouter } from 'react-router-dom';
 import { render, screen } from '@testing-library/react';
 import { TooltipProvider } from '@signozhq/ui/tooltip';
@@ -23,7 +24,9 @@ jest.mock('api/common/logEvent', () => ({
 
 const mockUseIsAIAssistantEnabled = useIsAIAssistantEnabled as jest.Mock;
 
-function renderHeader(): void {
+function renderHeader(
+	props: Partial<ComponentProps<typeof Header>> = {},
+): void {
 	// AppLayout supplies the TooltipProvider in the app; the header is rendered bare here.
 	render(
 		<MemoryRouter>
@@ -33,6 +36,7 @@ function renderHeader(): void {
 					isSaving={false}
 					onSave={jest.fn()}
 					onClose={jest.fn()}
+					{...props}
 				/>
 			</TooltipProvider>
 		</MemoryRouter>,
@@ -66,4 +70,34 @@ describe('PanelEditor Header', () => {
 		).not.toBeInTheDocument();
 		expect(screen.getByTestId('panel-editor-v2-save')).toBeInTheDocument();
 	});
+
+	it('keeps Save enabled even when there are no unsaved edits', () => {
+		mockUseIsAIAssistantEnabled.mockReturnValue(false);
+
+		renderHeader({ isDirty: false });
+
+		expect(screen.getByTestId('panel-editor-v2-save')).toBeEnabled();
+	});
+
+	it('disables Save only while read-only or saving', () => {
+		mockUseIsAIAssistantEnabled.mockReturnValue(false);
+
+		renderHeader({ isDirty: true, readOnly: true, readOnlyReason: 'Locked' });
+
+		expect(screen.getByTestId('panel-editor-v2-save')).toBeDisabled();
+	});
+
+	it('shows the Unsaved Changes badge only when there are unsaved edits', () => {
+		mockUseIsAIAssistantEnabled.mockReturnValue(false);
+
+		renderHeader({ isDirty: false });
+		expect(
+			screen.queryByTestId('panel-editor-v2-unsaved-badge'),
+		).not.toBeInTheDocument();
+
+		renderHeader({ isDirty: true });
+		expect(
+			screen.getByTestId('panel-editor-v2-unsaved-badge'),
+		).toBeInTheDocument();
+	});
 });

frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/__tests__/PanelEditorContainer.test.tsx

@@ -1,5 +1,6 @@
 import { render, screen, waitFor } from '@testing-library/react';
 import userEvent from '@testing-library/user-event';
+import { toast } from '@signozhq/ui/sonner';
 import type { DashboardtypesPanelDTO } from 'api/generated/services/sigNoz.schemas';
 import { PANEL_TYPES } from 'constants/queryBuilder';
 import { getPanelDefinition } from 'pages/DashboardPageV2/DashboardContainer/Panels/registry';
@@ -100,6 +101,12 @@ jest.mock('@signozhq/ui/resizable', () => ({
 jest.mock('@signozhq/ui/sonner', () => ({
 	toast: { success: jest.fn(), error: jest.fn() },
 }));
+const mockShowErrorModal = jest.fn();
+jest.mock('providers/ErrorModalProvider', () => ({
+	useErrorModal: (): { showErrorModal: jest.Mock } => ({
+		showErrorModal: mockShowErrorModal,
+	}),
+}));
 
 // Children mocked to capture props (and expose a Save trigger / footer slot).
 const mockHeaderProps = jest.fn();
@@ -261,13 +268,13 @@ describe('PanelEditorContainer composition', () => {
 		);
 	});
 
-	it('keeps a query-less new panel unsaveable but still serializes its seed query', () => {
+	it('keeps a query-less new panel clean but still serializes its seed query', () => {
 		setup(makePanel('signoz/TimeSeriesPanel'), { isNew: true });
 
 		expect(mockUseQuerySync).toHaveBeenCalledWith(
 			expect.objectContaining({ alwaysSerializeQuery: true }),
 		);
-		// No query and no edits yet → nothing to save, so Save stays disabled.
+		// No query and no edits yet → not dirty, so closing won't prompt to discard.
 		expect(mockHeaderProps).toHaveBeenCalledWith(
 			expect.objectContaining({ isDirty: false }),
 		);
@@ -290,7 +297,7 @@ describe('PanelEditorContainer composition', () => {
 		);
 	});
 
-	it('marks a new panel that already has a query saveable (e.g. list auto-runs one)', () => {
+	it('marks a new panel that already has a query dirty (e.g. list auto-runs one)', () => {
 		const seededQuery = {
 			spec: { plugin: { kind: 'signoz/BuilderQuery', spec: { signal: 'logs' } } },
 		};
@@ -327,6 +334,24 @@ describe('PanelEditorContainer composition', () => {
 		expect(mockSave).toHaveBeenCalledWith(panel.spec);
 	});
 
+	it('surfaces a save failure through the error modal', async () => {
+		// The raw thrown value flows straight to the modal, which normalizes it to an
+		// APIError itself (that normalization is covered by ErrorModalProvider's tests).
+		const failure = {
+			response: {
+				status: 400,
+				data: { error: { code: 'INVALID', message: 'Panel name already exists' } },
+			},
+		};
+		mockSave.mockRejectedValueOnce(failure);
+		setup(makePanel('signoz/TimeSeriesPanel'));
+
+		await userEvent.click(screen.getByTestId('editor-save'));
+
+		await waitFor(() => expect(mockShowErrorModal).toHaveBeenCalledWith(failure));
+		expect(toast.error).not.toHaveBeenCalled();
+	});
+
 	it('marks the saved panel to be scrolled into view on the dashboard', async () => {
 		setup(makePanel('signoz/TimeSeriesPanel'));
 

frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/hooks/__tests__/usePanelEditorQuerySync.integration.test.tsx

@@ -117,6 +117,49 @@ describe('usePanelEditorQuerySync (real query builder)', () => {
 		},
 	);
 
+	it('a NEW panel stays query-dirty after Stage & Run commits the edited query into the draft', async () => {
+		// Repro of the P0: new panel → edit → Run commits the query into the draft; the
+		// dirty baseline must not drift onto it, or Save re-disables.
+		const editedInUrl: Query = {
+			...initialQueriesMap[DataSource.METRICS],
+			id: 'edited-new-panel',
+			builder: {
+				...initialQueriesMap[DataSource.METRICS].builder,
+				queryData: [
+					{
+						...initialQueriesMap[DataSource.METRICS].builder.queryData[0],
+						legend: 'edited-legend',
+					},
+				],
+			},
+		};
+		const committed = toPerses(editedInUrl, panelType);
+
+		const { result, rerender } = renderHook(
+			({ draftQueries }: { draftQueries: DashboardtypesQueryDTO[] }) =>
+				usePanelEditorQuerySync({
+					draft: makePanel(draftQueries),
+					panelType,
+					setSpec: jest.fn(),
+					refetch: jest.fn(),
+					alwaysSerializeQuery: true,
+					signal: DataSource.METRICS as unknown as TelemetrytypesSignalDTO,
+					// savedQueries omitted — new panel.
+				}),
+			{
+				wrapper: makeUrlWrapper(editedInUrl),
+				initialProps: { draftQueries: [] as DashboardtypesQueryDTO[] },
+			},
+		);
+
+		// The edited query (from the URL) diverges from the seeded default → dirty.
+		await waitFor(() => expect(result.current.isQueryDirty).toBe(true));
+
+		// Stage & Run commits the edited query into the draft → must stay dirty.
+		rerender({ draftQueries: committed });
+		expect(result.current.isQueryDirty).toBe(true);
+	});
+
 	it('an untouched panel with a minimal/older stored query is NOT dirty (drift fix)', async () => {
 		// An older saved query carries only a few fields; the builder re-emits many more
 		// (source, stepInterval, filter, spaceAggregation, …). Comparing raw would read

frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/hooks/usePanelEditorQuerySync.ts

@@ -79,6 +79,11 @@ export function usePanelEditorQuerySync({
 	// in-editor edit in the URL survives a refresh / browser Back-Forward.
 	useShareBuilderUrl({ defaultValue: seedQuery });
 
+	// Frozen seed for the new-panel dirty baseline: Stage & Run commits the live query
+	// into `draft.spec.queries`, so re-reading `seedQuery` would drift the baseline onto
+	// the run query and Save would re-disable right after a run.
+	const initialSeedRef = useRef(seedQuery);
+
 	// Commit the live query into the draft (what the preview fetches).
 	const commitQuery = useCallback(
 		(query: Query): boolean => {
@@ -152,16 +157,19 @@ export function usePanelEditorQuerySync({
 	// the V5 envelope level. Anchoring to `savedQueries` (not the builder-seed) keeps a
 	// handed-off / URL-restored edit reading as dirty; routing both sides through the same
 	// `fromPerses → toPerses` round-trip stops builder-added defaults (absent from an older
-	// stored query) reading an untouched panel as modified. New panel: fall back to seed.
+	// stored query) reading an untouched panel as modified. New panel: fall back to the
+	// frozen initial seed (see `initialSeedRef`).
 	const baselineEnvelopes = useMemo(
 		() =>
 			toQueryEnvelopes(
 				toPerses(
-					savedQueries ? fromPerses(savedQueries, panelType) : seedQuery,
+					savedQueries
+						? fromPerses(savedQueries, panelType)
+						: initialSeedRef.current,
 					panelType,
 				),
 			),
-		[savedQueries, seedQuery, panelType],
+		[savedQueries, panelType],
 	);
 	const isQueryDirty = useMemo(
 		() =>

frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/index.tsx

@@ -19,6 +19,7 @@ import {
 	SectionKind,
 } from 'pages/DashboardPageV2/DashboardContainer/Panels/types/sections';
 import { getBuilderQueries } from 'pages/DashboardPageV2/DashboardContainer/Panels/utils/getBuilderQueries';
+import { useErrorModal } from 'providers/ErrorModalProvider';
 
 import { getExecStats } from '../queryV5/v5ResponseData';
 import { usePanelInteractions } from '../PanelsAndSectionsLayout/Panel/hooks/usePanelInteractions';
@@ -159,9 +160,10 @@ function PanelEditorContainer({
 		return section?.controls;
 	}, [panelDefinition]);
 
-	// New panels are savable once seeded with a query (List auto-seeds one). Read the
-	// seed `panel`, not the live `draft` — the staged-query sync commits the seed into
-	// the draft on open, which would falsely dirty an untouched query-less new panel.
+	// Unsaved-edits flag driving the discard confirmation on close (Save is always
+	// enabled). Read the seed `panel`, not the live `draft` — the staged-query sync
+	// commits the seed into the draft on open, which would falsely dirty an untouched
+	// query-less new panel.
 	const isDirty = useMemo(
 		() => isSpecDirty || isQueryDirty || (isNew && panel.spec.queries.length > 0),
 		[isSpecDirty, isQueryDirty, isNew, panel.spec.queries.length],
@@ -226,6 +228,7 @@ function PanelEditorContainer({
 	});
 
 	const setScrollTargetId = useScrollIntoViewStore((s) => s.setScrollTargetId);
+	const { showErrorModal } = useErrorModal();
 
 	const onSave = useCallback(async (): Promise<void> => {
 		if (!isEditable) {
@@ -236,12 +239,22 @@ function PanelEditorContainer({
 			const savedPanelId = await save(buildSaveSpec(draft.spec));
 			// Reveal the saved panel once the dashboard re-renders.
 			setScrollTargetId(savedPanelId);
-			toast.success('Panel saved');
+			toast.success('Panel saved', {
+				position: 'top-center',
+			});
 			onSaved();
-		} catch {
-			toast.error('Failed to save panel');
+		} catch (err) {
+			showErrorModal(err);
 		}
-	}, [isEditable, save, buildSaveSpec, draft.spec, setScrollTargetId, onSaved]);
+	}, [
+		isEditable,
+		save,
+		buildSaveSpec,
+		draft.spec,
+		setScrollTargetId,
+		onSaved,
+		showErrorModal,
+	]);
 
 	// Leaving an existing panel's editor (without saving) still returns to it, so
 	// the dashboard lands on that panel rather than scrolled to the top. A new,

frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/PanelStatus/__tests__/utils.test.ts

@@ -51,7 +51,7 @@ describe('panelStatusFromError', () => {
 
 	it('falls back to the error message when there is no structured body', () => {
 		expect(panelStatusFromError(new Error('boom'))).toStrictEqual({
-			code: 'unknown_error',
+			code: 'UPSTREAM_UNAVAILABLE',
 			message: 'boom',
 			docsUrl: undefined,
 			messages: [],

frontend/src/pages/DashboardsListPageV2/components/ActionsPopover/DeleteActionItem.tsx

@@ -12,13 +12,13 @@ import {
 	deleteDashboardV2,
 	invalidateListDashboardsForUserV2,
 } from 'api/generated/services/dashboard';
+import { useDeleteConfirm } from 'components/DeleteConfirmModal/useDeleteConfirm';
 import { DashboardListEvents } from 'pages/DashboardsListPageV2/constants/events';
 import { useAppContext } from 'providers/App/App';
 import { useErrorModal } from 'providers/ErrorModalProvider';
 import APIError from 'types/api/error';
 import { USER_ROLES } from 'types/roles';
 
-import { useDeleteConfirm } from '../DeleteConfirmModal/useDeleteConfirm';
 import styles from './ActionsPopover.module.scss';
 
 interface Props {

frontend/src/pages/DashboardsListPageV2/components/ViewsRail/ViewsRail.tsx

@@ -6,11 +6,11 @@ import { Typography } from '@signozhq/ui/typography';
 import { Bookmark, PenLine, Plus, Search, Trash2 } from '@signozhq/icons';
 import cx from 'classnames';
 
+import { useDeleteConfirm } from 'components/DeleteConfirmModal/useDeleteConfirm';
 import { DashboardListEvents } from 'pages/DashboardsListPageV2/constants/events';
 
 import type { SavedView } from '../../types';
 import { type BuiltinView } from '../../utils/views';
-import { useDeleteConfirm } from '../DeleteConfirmModal/useDeleteConfirm';
 import ViewNamePopover from './ViewNamePopover';
 
 import styles from './ViewsRail.module.scss';

frontend/src/types/api/querySuggestions/types.ts

@@ -1,4 +1,4 @@
-import { QUERY_BUILDER_KEY_TYPES } from 'constants/antlrQueryConstants';
+import { FieldDataType } from 'types/api/v5/queryRange';
 
 export interface QueryKeyDataSuggestionsProps {
 	label: string;
@@ -7,7 +7,12 @@ export interface QueryKeyDataSuggestionsProps {
 	apply?: string;
 	detail?: string;
 	fieldContext?: 'resource' | 'scope' | 'attribute' | 'span';
-	fieldDataType?: QUERY_BUILDER_KEY_TYPES;
+	/**
+	 * The field's type as the API reports it. Was declared as the antlr key-type enum
+	 * (`string | number | boolean`), which the endpoint never returns — every consumer cast it
+	 * back to `FieldDataType`.
+	 */
+	fieldDataType?: FieldDataType;
 	name: string;
 	signal: 'traces' | 'logs' | 'metrics';
 }
@@ -26,7 +31,7 @@ export interface QueryKeyRequestProps {
 	signal: 'traces' | 'logs' | 'metrics';
 	searchText: string;
 	fieldContext?: 'resource' | 'scope' | 'attribute' | 'span';
-	fieldDataType?: QUERY_BUILDER_KEY_TYPES;
+	fieldDataType?: FieldDataType;
 	metricName?: string;
 	metricNamespace?: string;
 	signalSource?: 'meter' | '';

frontend/src/utils/__tests__/fieldDataType.test.ts

@@ -0,0 +1,34 @@
+import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
+import type { FieldDataType } from 'types/api/v5/queryRange';
+
+import { fieldDataTypeToDataType } from '../fieldDataType';
+
+describe('fieldDataTypeToDataType', () => {
+	it('maps the reported numeric spellings onto the query-builder numerics', () => {
+		expect(fieldDataTypeToDataType('number')).toBe(DataTypes.Float64);
+		expect(fieldDataTypeToDataType('int64')).toBe(DataTypes.Int64);
+		expect(fieldDataTypeToDataType('float64')).toBe(DataTypes.Float64);
+	});
+
+	it('maps the list spellings onto the array types', () => {
+		expect(fieldDataTypeToDataType('[]string')).toBe(DataTypes.ArrayString);
+		expect(fieldDataTypeToDataType('[]bool')).toBe(DataTypes.ArrayBool);
+		expect(fieldDataTypeToDataType('[]int64')).toBe(DataTypes.ArrayInt64);
+		expect(fieldDataTypeToDataType('[]float64')).toBe(DataTypes.ArrayFloat64);
+		expect(fieldDataTypeToDataType('[]number')).toBe(DataTypes.ArrayFloat64);
+	});
+
+	it('passes through the spellings the two vocabularies share', () => {
+		expect(fieldDataTypeToDataType('string')).toBe(DataTypes.String);
+		expect(fieldDataTypeToDataType('bool')).toBe(DataTypes.bool);
+	});
+
+	it('returns EMPTY for empty, missing and not-yet-known types', () => {
+		expect(fieldDataTypeToDataType('')).toBe(DataTypes.EMPTY);
+		expect(fieldDataTypeToDataType(undefined)).toBe(DataTypes.EMPTY);
+		// The backend can store `[]json` / `[]dynamic`, which the API union doesn't list.
+		expect(fieldDataTypeToDataType('[]json' as FieldDataType)).toBe(
+			DataTypes.EMPTY,
+		);
+	});
+});

frontend/src/utils/errorUtils.ts

@@ -42,7 +42,12 @@ export function toAPIError(
 	try {
 		ErrorResponseHandlerForGeneratedAPIs(error);
 	} catch (apiError) {
-		if (apiError instanceof APIError) {
+		// UPSTREAM_UNAVAILABLE means the handler couldn't find a backend error code
+		// (non-envelope body); prefer the caller's context-specific defaultMessage.
+		if (
+			apiError instanceof APIError &&
+			apiError.getErrorCode() !== 'UPSTREAM_UNAVAILABLE'
+		) {
 			return apiError;
 		}
 	}

frontend/src/utils/fieldDataType.ts

@@ -0,0 +1,31 @@
+import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
+import type { FieldDataType } from 'types/api/v5/queryRange';
+
+/**
+ * The field metadata APIs and the query builder spell field types differently: the metadata
+ * collapses the numerics to `number` and writes lists as `[]string`, while `DataTypes` — which
+ * keys the operator and attribute-value lookups — uses `float64` and `array(string)`. Convert
+ * where a reported type becomes a query-builder one, so those lookups can't miss.
+ */
+const TO_DATA_TYPE: Record<FieldDataType, DataTypes> = {
+	'': DataTypes.EMPTY,
+	string: DataTypes.String,
+	bool: DataTypes.bool,
+	int64: DataTypes.Int64,
+	float64: DataTypes.Float64,
+	number: DataTypes.Float64,
+	'[]string': DataTypes.ArrayString,
+	'[]bool': DataTypes.ArrayBool,
+	'[]int64': DataTypes.ArrayInt64,
+	'[]float64': DataTypes.ArrayFloat64,
+	'[]number': DataTypes.ArrayFloat64,
+};
+
+/**
+ * Exhaustive over `FieldDataType`, so a type added to the API union fails to compile here
+ * rather than going missing at runtime. The fallback is for a spelling the union doesn't know
+ * yet — the backend can store `[]json` and `[]dynamic` — not for any of the cases above.
+ */
+export const fieldDataTypeToDataType = (
+	fieldDataType?: FieldDataType,
+): DataTypes => TO_DATA_TYPE[fieldDataType ?? ''] ?? DataTypes.EMPTY;

go.mod

@@ -4,7 +4,7 @@ go 1.25.7
 
 require (
 	dario.cat/mergo v1.0.2
-	github.com/AfterShip/clickhouse-sql-parser v0.4.16
+	github.com/AfterShip/clickhouse-sql-parser v0.5.2
 	github.com/ClickHouse/clickhouse-go/v2 v2.44.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/clickhouse-go-mock v0.14.0

go.sum

@@ -66,8 +66,8 @@ dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
 filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
-github.com/AfterShip/clickhouse-sql-parser v0.4.16 h1:gpl+wXclYUKT0p4+gBq22XeRYWwEoZ9f35vogqMvkLQ=
-github.com/AfterShip/clickhouse-sql-parser v0.4.16/go.mod h1:W0Z82wJWkJxz2RVun/RMwxue3g7ut47Xxl+SFqdJGus=
+github.com/AfterShip/clickhouse-sql-parser v0.5.2 h1:KCxdmvuVawVF4yl0ZS33q7olgmLZwvZG5BjWHp6o414=
+github.com/AfterShip/clickhouse-sql-parser v0.5.2/go.mod h1:Qi3qvPTfZb/aFwI5V4WFOahgjsLJa4MzVijIAfwOhDw=
 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=

pkg/flagger/registry.go

@@ -14,6 +14,8 @@ var (
 	FeatureEnableAIObservability  = featuretypes.MustNewName("enable_ai_observability")
 	FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
 	FeatureUseInfraMonitoringV2   = featuretypes.MustNewName("use_infra_monitoring_v2")
+
+	FeatureUsePrometheusClickhouseV2 = featuretypes.MustNewName("use_prometheus_clickhouse_v2")
 )
 
 func MustNewRegistry() featuretypes.Registry {
@@ -106,6 +108,14 @@ func MustNewRegistry() featuretypes.Registry {
 			DefaultVariant: featuretypes.MustNewName("disabled"),
 			Variants:       featuretypes.NewBooleanVariants(),
 		},
+		&featuretypes.Feature{
+			Name:           FeatureUsePrometheusClickhouseV2,
+			Kind:           featuretypes.KindBoolean,
+			Stage:          featuretypes.StageExperimental,
+			Description:    "Runs PromQL queries on the clickhousev2 provider alongside the served engine result and logs any difference; serving is unaffected.",
+			DefaultVariant: featuretypes.MustNewName("disabled"),
+			Variants:       featuretypes.NewBooleanVariants(),
+		},
 	)
 	if err != nil {
 		panic(err)

pkg/prometheus/clickhouseprometheusv2/capture.go

@@ -0,0 +1,85 @@
+package clickhouseprometheusv2
+
+import (
+	"context"
+	"sync"
+
+	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/util/annotations"
+)
+
+// statementRecorder collects the statements a PromQL evaluation would run.
+// Safe for concurrent use: the engine may Select selectors concurrently.
+type statementRecorder struct {
+	mu         sync.Mutex
+	statements []prometheus.CapturedStatement
+}
+
+func (r *statementRecorder) record(query string, args []any) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.statements = append(r.statements, prometheus.CapturedStatement{Query: query, Args: args})
+}
+
+func (r *statementRecorder) Statements() []prometheus.CapturedStatement {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	out := make([]prometheus.CapturedStatement, len(r.statements))
+	copy(out, r.statements)
+	return out
+}
+
+type captureQueryable struct {
+	client   *client
+	recorder *statementRecorder
+}
+
+func (c *captureQueryable) Querier(mint, maxt int64) (storage.Querier, error) {
+	return &captureQuerier{
+		querier:  querier{mint: mint, maxt: maxt, client: c.client},
+		recorder: c.recorder,
+	}, nil
+}
+
+// captureQuerier builds the same SQL as the live querier but records it and
+// returns no data. The fingerprint filter always takes the subquery form:
+// without executing the series lookup, the inline literal set is unknown.
+type captureQuerier struct {
+	querier
+	recorder *statementRecorder
+}
+
+func (c *captureQuerier) Select(ctx context.Context, _ bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
+
+	start, end := c.window(hints)
+
+	samplesQuery, args, err := buildSamplesQuery(start, end, metricNamesFromMatchers(matchers), matchers, c.lastSamplePerStepFor(ctx, hints))
+	if err != nil {
+		return storage.ErrSeriesSet(err)
+	}
+	c.recorder.record(samplesQuery, args)
+
+	return storage.EmptySeriesSet()
+}
+
+func (c *captureQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	return nil, nil, nil
+}
+
+func (c *captureQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	return nil, nil, nil
+}
+
+// metricNamesFromMatchers extracts the statically known metric name, if any.
+// The live path derives names from the matched series; the capture path has
+// no execution results, so only a __name__ equality contributes.
+func metricNamesFromMatchers(matchers []*labels.Matcher) []string {
+	for _, m := range matchers {
+		if m.Name == metricNameLabel && m.Type == labels.MatchEqual && m.Value != "" {
+			return []string{m.Value}
+		}
+	}
+	return nil
+}

pkg/prometheus/clickhouseprometheusv2/client.go

@@ -0,0 +1,180 @@
+package clickhouseprometheusv2
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"math"
+	"slices"
+
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
+	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/prometheus/prometheus/model/labels"
+	promValue "github.com/prometheus/prometheus/model/value"
+)
+
+// seriesLookup holds a series lookup's result: matched fingerprints with
+// their labels, and the distinct metric names seen on them.
+type seriesLookup struct {
+	fingerprints map[uint64]labels.Labels
+	metricNames  []string
+}
+
+// client executes the series, samples and raw queries against ClickHouse.
+type client struct {
+	settings       factory.ScopedProviderSettings
+	telemetryStore telemetrystore.TelemetryStore
+	lookbackMs     int64
+}
+
+func newClient(settings factory.ScopedProviderSettings, telemetryStore telemetrystore.TelemetryStore, cfg prometheus.Config) *client {
+	lookback := cfg.LookbackDelta
+	if lookback <= 0 {
+		// Mirror the engine: promql defaults an unset lookback to 5m.
+		lookback = defaultLookbackDelta
+	}
+	return &client{
+		settings:       settings,
+		telemetryStore: telemetryStore,
+		lookbackMs:     lookback.Milliseconds(),
+	}
+}
+
+func (c *client) withContext(ctx context.Context, functionName string) context.Context {
+	return ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
+		instrumentationtypes.TelemetrySignal:  telemetrytypes.SignalMetrics.StringValue(),
+		instrumentationtypes.CodeNamespace:    "clickhouse-prometheus-v2",
+		instrumentationtypes.CodeFunctionName: functionName,
+	})
+}
+
+func (c *client) selectSeries(ctx context.Context, query string, args []any) (*seriesLookup, error) {
+	ctx = c.withContext(ctx, "selectSeries")
+	rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	lookup := &seriesLookup{fingerprints: make(map[uint64]labels.Labels)}
+	names := make(map[string]struct{})
+
+	var fingerprint uint64
+	var labelsJSON string
+	for rows.Next() {
+		if err := rows.Scan(&fingerprint, &labelsJSON); err != nil {
+			return nil, err
+		}
+		lset, err := unmarshalLabels(labelsJSON)
+		if err != nil {
+			return nil, err
+		}
+		lookup.fingerprints[fingerprint] = lset
+		if name := lset.Get(metricNameLabel); name != "" {
+			names[name] = struct{}{}
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	for name := range names {
+		lookup.metricNames = append(lookup.metricNames, name)
+	}
+	slices.Sort(lookup.metricNames)
+
+	return lookup, nil
+}
+
+// unmarshalLabels parses the labels JSON column, dropping empty-valued
+// labels: empty means "absent" in Prometheus, but stored attribute JSON can
+// carry them.
+func unmarshalLabels(s string) (labels.Labels, error) {
+	m := make(map[string]string)
+	if err := json.Unmarshal([]byte(s), &m); err != nil {
+		return labels.EmptyLabels(), err
+	}
+	builder := labels.NewScratchBuilder(len(m))
+	for k, v := range m {
+		if v == "" {
+			continue
+		}
+		builder.Add(k, v)
+	}
+	builder.Sort()
+	return builder.Labels(), nil
+}
+
+// selectSamples assembles per-series sample slices from a samples query (raw
+// or last-sample-per-step; same column shape), whose rows arrive ordered by
+// (fingerprint, unix_milli). Fingerprints missing from the lookup are skipped:
+// the samples query's semi-join re-runs the series predicates and can match
+// series registered after the lookup ran — the lookup is the read snapshot.
+// Stale flags map to the engine's StaleNaN. Duplicate
+// timestamps pass through: uniqueness is ingest's job, and v1 feeds them to
+// the engine as-is over the same dirty data.
+func (c *client) selectSamples(ctx context.Context, query string, args []any, lookup *seriesLookup) ([]*series, error) {
+	ctx = c.withContext(ctx, "selectSamples")
+	rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var (
+		result       []*series
+		current      *series
+		fingerprint  uint64
+		prevFp       uint64
+		timestampMs  int64
+		val          float64
+		flags        uint32
+		first        = true
+		haveCurrent  bool
+		staleMarker  = math.Float64frombits(promValue.StaleNaN)
+		unknownCount int
+	)
+
+	for rows.Next() {
+		if err := rows.Scan(&fingerprint, &timestampMs, &val, &flags); err != nil {
+			return nil, err
+		}
+
+		if first || fingerprint != prevFp {
+			first = false
+			prevFp = fingerprint
+			lset, ok := lookup.fingerprints[fingerprint]
+			if !ok {
+				unknownCount++
+				haveCurrent = false
+				continue
+			}
+			current = &series{lset: lset}
+			result = append(result, current)
+			haveCurrent = true
+		}
+		if !haveCurrent {
+			continue
+		}
+
+		if flags&1 == 1 {
+			val = staleMarker
+		}
+		current.ts = append(current.ts, timestampMs)
+		current.vs = append(current.vs, val)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	if unknownCount > 0 {
+		c.settings.Logger().DebugContext(ctx, "skipped samples of fingerprints missing from series lookup",
+			slog.Int("unknown_fingerprints", unknownCount))
+	}
+
+	return result, nil
+}

pkg/prometheus/clickhouseprometheusv2/doc.go

@@ -0,0 +1,49 @@
+// Package clickhouseprometheusv2 is the second-generation ClickHouse-backed
+// Prometheus provider. It exists because the v1 provider fetches every raw
+// sample of a query's union window through the remote-read protobuf layer
+// and hands it to the engine — the cost is a function of ingested data, not
+// of the question asked. Here the stock promql engine evaluates over a
+// native storage.Querier: no translation layer, per-selector fetch windows,
+// and fetch reductions that are provably invisible to the engine. Every
+// reduction either preserves engine semantics exactly or is not performed.
+//
+// # Series lookup
+//
+// Matchers resolve to series once per selector (selectSeries) against the
+// time-series tables, which hold one row per (fingerprint, bucket) at
+// 1h/6h/1d/1w granularities; timeSeriesTableFor picks the table whose bucket
+// fits the window and rounds the window start down to the bucket boundary.
+// How matchers become SQL, and why regexes are anchored, is documented at
+// applySeriesConditions. Empty-valued labels come off at this boundary: an
+// empty value means "label absent" in Prometheus, but stored attribute JSON
+// can carry them.
+//
+// # Sample fetch
+//
+// Samples are fetched per selector using the engine's per-selector hints,
+// not the query-wide union window, so foo / foo offset 1d reads two narrow
+// windows instead of the widest one twice. Instant selectors of
+// subquery-free queries fetch only the last sample per step bucket — see
+// lastSamplePerStep for the correctness argument — while range selectors
+// always fetch raw: every sample feeds the range function. Row assembly maps
+// stale flags to the engine's StaleNaN and merges series with identical
+// label sets (sortAndMerge), because the engine assumes storages never emit
+// duplicates.
+//
+// # Sharding
+//
+// samples_v4 and time_series_v4 (and all their rollups) shard on the same
+// key — cityHash64(env, temporality, metric_name, fingerprint) — so a
+// series' samples and catalog rows live on the same shard. The samples
+// fetch exploits that: it restricts by a shard-local series subquery, not a
+// GLOBAL broadcast of the matched set. Delta-temporality
+// series stay invisible to PromQL exactly as they are in v1: the rollout
+// gate is parity with v1, and a Delta stream fed to rate() as-if-cumulative
+// would be wrong, not just new.
+//
+// # Observability
+//
+// Every statement carries a log_comment with
+// code.namespace=clickhouse-prometheus-v2 and code.function.name naming the
+// call site, so this provider's work is attributable in system.query_log.
+package clickhouseprometheusv2

pkg/prometheus/clickhouseprometheusv2/provider.go

@@ -0,0 +1,72 @@
+package clickhouseprometheusv2
+
+import (
+	"context"
+
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/prometheus/prometheus/storage"
+)
+
+// Provider ties the package together: its own engine and parser, and the
+// ClickHouse client behind the native storage.Querier. See the package
+// documentation for how the read path differs from v1. It is exported as a
+// concrete type — callers hold it directly, and an interface with a single
+// implementation would only hide that dependency.
+type Provider struct {
+	settings factory.ScopedProviderSettings
+	engine   *prometheus.Engine
+	parser   prometheus.Parser
+	client   *client
+}
+
+var (
+	_ prometheus.Prometheus        = (*Provider)(nil)
+	_ prometheus.StatementCapturer = (*Provider)(nil)
+)
+
+func NewFactory(telemetryStore telemetrystore.TelemetryStore) factory.ProviderFactory[prometheus.Prometheus, prometheus.Config] {
+	return factory.NewProviderFactory(factory.MustNewName("clickhousev2"), func(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config) (prometheus.Prometheus, error) {
+		return New(ctx, providerSettings, config, telemetryStore)
+	})
+}
+
+func New(_ context.Context, providerSettings factory.ProviderSettings, config prometheus.Config, telemetryStore telemetrystore.TelemetryStore) (*Provider, error) {
+	settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2")
+
+	engine := prometheus.NewEngine(settings.Logger(), config)
+	parser := prometheus.NewParser()
+	client := newClient(settings, telemetryStore, config)
+
+	return &Provider{
+		settings: settings,
+		engine:   engine,
+		parser:   parser,
+		client:   client,
+	}, nil
+}
+
+func (p *Provider) Engine() *prometheus.Engine {
+	return p.engine
+}
+
+func (p *Provider) Parser() prometheus.Parser {
+	return p.parser
+}
+
+func (p *Provider) Storage() storage.Queryable {
+	return p
+}
+
+func (p *Provider) Querier(mint, maxt int64) (storage.Querier, error) {
+	return &querier{mint: mint, maxt: maxt, client: p.client}, nil
+}
+
+// CapturingStorage implements prometheus.StatementCapturer: a storage that
+// records each selector's SQL without executing it, for the preview path.
+// A fresh recorder per call keeps concurrent dry-runs isolated.
+func (p *Provider) CapturingStorage() (storage.Queryable, prometheus.StatementRecorder) {
+	recorder := &statementRecorder{}
+	return &captureQueryable{client: p.client, recorder: recorder}, recorder
+}

pkg/prometheus/clickhouseprometheusv2/querier.go

@@ -0,0 +1,170 @@
+package clickhouseprometheusv2
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/huandu/go-sqlbuilder"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/util/annotations"
+)
+
+// defaultLookbackDelta mirrors promql's default when the config leaves the
+// lookback unset; the engine and the storage must agree on it for
+// last-sample-per-step bucket anchoring.
+const defaultLookbackDelta = 5 * time.Minute
+
+// querier is a native storage.Querier over ClickHouse: Select builds SQL
+// directly from the matchers and hints, with no remote-read protobuf layer.
+type querier struct {
+	mint, maxt int64
+	client     *client
+}
+
+var _ storage.Querier = (*querier)(nil)
+
+func (q *querier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
+	start, end := q.window(hints)
+
+	seriesQuery, seriesArgs, err := buildSeriesQuery(start, end, matchers)
+	if err != nil {
+		return storage.ErrSeriesSet(err)
+	}
+	lookup, err := q.client.selectSeries(ctx, seriesQuery, seriesArgs)
+	if err != nil {
+		return storage.ErrSeriesSet(err)
+	}
+	if len(lookup.fingerprints) == 0 {
+		return storage.EmptySeriesSet()
+	}
+
+	list, err := q.fetchSamples(ctx, start, end, matchers, lookup, q.lastSamplePerStepFor(ctx, hints))
+	if err != nil {
+		return storage.ErrSeriesSet(err)
+	}
+
+	// The engine assumes storages never emit duplicate label sets.
+	list = sortAndMerge(list)
+	return newSeriesSet(list)
+}
+
+func (q *querier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	sb := sqlbuilder.NewSelectBuilder()
+	if name == metricNameLabel {
+		sb.Select("DISTINCT metric_name AS value")
+	} else {
+		sb.Select(fmt.Sprintf("DISTINCT JSONExtractString(labels, %s) AS value", sb.Var(name)))
+	}
+	adjustedStart, table := timeSeriesTableFor(q.mint, q.maxt)
+	sb.From(fmt.Sprintf("%s.%s", databaseName, table))
+	if err := applySeriesConditions(sb, adjustedStart, q.maxt, matchers); err != nil {
+		return nil, nil, err
+	}
+	sb.Where("value != ''")
+	if hints != nil && hints.Limit > 0 {
+		sb.Limit(hints.Limit)
+	}
+
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	values, err := q.selectStrings(ctx, "LabelValues", query, args)
+	if err != nil {
+		return nil, nil, err
+	}
+	slices.Sort(values)
+	return values, nil, nil
+}
+
+func (q *querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
+	sb := sqlbuilder.NewSelectBuilder()
+	sb.Select("DISTINCT arrayJoin(JSONExtractKeys(labels)) AS name")
+	adjustedStart, table := timeSeriesTableFor(q.mint, q.maxt)
+	sb.From(fmt.Sprintf("%s.%s", databaseName, table))
+	if err := applySeriesConditions(sb, adjustedStart, q.maxt, matchers); err != nil {
+		return nil, nil, err
+	}
+	if hints != nil && hints.Limit > 0 {
+		sb.Limit(hints.Limit)
+	}
+
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	names, err := q.selectStrings(ctx, "LabelNames", query, args)
+	if err != nil {
+		return nil, nil, err
+	}
+	slices.Sort(names)
+	return names, nil, nil
+}
+
+func (q *querier) Close() error {
+	return nil
+}
+
+// window returns the per-selector fetch window from the hints (already
+// adjusted for offset, @, range and lookback); mint/maxt span the union of
+// all selectors and are the fallback.
+func (q *querier) window(hints *storage.SelectHints) (int64, int64) {
+	if hints != nil && hints.Start != 0 && hints.End != 0 && hints.Start <= hints.End {
+		return hints.Start, hints.End
+	}
+	return q.mint, q.maxt
+}
+
+// lastSamplePerStepFor decides whether the fetch can keep only the last
+// sample per step bucket (see lastSamplePerStep). Only instant selectors
+// (hints.Range == 0) of subquery-free queries qualify: range selectors need
+// every raw sample, and subquery selectors evaluate at the subquery's own
+// step while hints carry the top-level step — the subquery-free proof
+// arrives as QueryTraits in the context. The first evaluation timestamp is
+// recovered as hints.Start + lookback - 1ms, inverting how the engine
+// derives hints.Start.
+func (q *querier) lastSamplePerStepFor(ctx context.Context, hints *storage.SelectHints) *lastSamplePerStep {
+	if hints == nil || hints.Range != 0 || hints.Start <= 0 {
+		return nil
+	}
+	traits, ok := prometheus.QueryTraitsFromContext(ctx)
+	if !ok || !traits.SubqueryFree {
+		return nil
+	}
+	firstEval := hints.Start + q.client.lookbackMs - 1
+	if firstEval > hints.End {
+		// Defensive: never anchor a bucket past the window.
+		firstEval = hints.End
+	}
+	return &lastSamplePerStep{firstEvalMs: firstEval, stepMs: hints.Step}
+}
+
+// fetchSamples runs the samples query for the matched series (see
+// buildSamplesQuery).
+func (q *querier) fetchSamples(ctx context.Context, start, end int64, matchers []*labels.Matcher, lookup *seriesLookup, lastPerStep *lastSamplePerStep) ([]*series, error) {
+	query, args, err := buildSamplesQuery(start, end, lookup.metricNames, matchers, lastPerStep)
+	if err != nil {
+		return nil, err
+	}
+	return q.client.selectSamples(ctx, query, args, lookup)
+}
+
+func (q *querier) selectStrings(ctx context.Context, fn, query string, args []any) ([]string, error) {
+	ctx = q.client.withContext(ctx, fn)
+	rows, err := q.client.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var out []string
+	var v string
+	for rows.Next() {
+		if err := rows.Scan(&v); err != nil {
+			return nil, err
+		}
+		out = append(out, v)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return out, nil
+}

pkg/prometheus/clickhouseprometheusv2/seriesset.go

@@ -0,0 +1,184 @@
+package clickhouseprometheusv2
+
+import (
+	"sort"
+
+	"github.com/prometheus/prometheus/model/histogram"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+	"github.com/prometheus/prometheus/util/annotations"
+)
+
+// series is one time series with samples as parallel slices ordered by
+// timestamp. Deliberately not storage.NewListSeries: that boxes every sample
+// as an interface value, a per-sample allocation this fetch path exists to
+// avoid.
+type series struct {
+	lset labels.Labels
+	ts   []int64
+	vs   []float64
+}
+
+var _ storage.Series = (*series)(nil)
+
+func (s *series) Labels() labels.Labels {
+	return s.lset
+}
+
+func (s *series) Iterator(it chunkenc.Iterator) chunkenc.Iterator {
+	if fit, ok := it.(*floatIterator); ok {
+		fit.reset(s)
+		return fit
+	}
+	fit := &floatIterator{}
+	fit.reset(s)
+	return fit
+}
+
+// floatIterator implements chunkenc.Iterator over a series' sample slices.
+type floatIterator struct {
+	s *series
+	i int
+}
+
+var _ chunkenc.Iterator = (*floatIterator)(nil)
+
+func (it *floatIterator) reset(s *series) {
+	it.s = s
+	it.i = -1
+}
+
+func (it *floatIterator) Next() chunkenc.ValueType {
+	it.i++
+	if it.i >= len(it.s.ts) {
+		return chunkenc.ValNone
+	}
+	return chunkenc.ValFloat
+}
+
+func (it *floatIterator) Seek(t int64) chunkenc.ValueType { //nolint:govet // stdmethods flags io.Seeker; this is chunkenc.Iterator's Seek
+	if it.i < 0 {
+		it.i = 0
+	}
+	if it.i >= len(it.s.ts) {
+		return chunkenc.ValNone
+	}
+	// The current position, once valid, must not move backwards.
+	if it.s.ts[it.i] >= t {
+		return chunkenc.ValFloat
+	}
+	it.i += sort.Search(len(it.s.ts)-it.i, func(j int) bool {
+		return it.s.ts[it.i+j] >= t
+	})
+	if it.i >= len(it.s.ts) {
+		return chunkenc.ValNone
+	}
+	return chunkenc.ValFloat
+}
+
+func (it *floatIterator) At() (int64, float64) {
+	return it.s.ts[it.i], it.s.vs[it.i]
+}
+
+func (it *floatIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
+	return 0, nil
+}
+
+func (it *floatIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
+	return 0, nil
+}
+
+func (it *floatIterator) AtT() int64 {
+	return it.s.ts[it.i]
+}
+
+// AtST returns the current start timestamp; not tracked by this storage.
+func (it *floatIterator) AtST() int64 {
+	return 0
+}
+
+func (it *floatIterator) Err() error {
+	return nil
+}
+
+// seriesSet iterates a fully materialized, label-sorted list of series.
+type seriesSet struct {
+	series []*series
+	i      int
+}
+
+var _ storage.SeriesSet = (*seriesSet)(nil)
+
+func newSeriesSet(list []*series) *seriesSet {
+	return &seriesSet{series: list, i: -1}
+}
+
+func (s *seriesSet) Next() bool {
+	s.i++
+	return s.i < len(s.series)
+}
+
+func (s *seriesSet) At() storage.Series {
+	return s.series[s.i]
+}
+
+func (s *seriesSet) Err() error {
+	return nil
+}
+
+func (s *seriesSet) Warnings() annotations.Annotations {
+	return nil
+}
+
+// sortAndMerge orders series by label set and merges identical label sets
+// by timestamp (first sample wins ties): distinct fingerprints can carry
+// identical label sets, and the engine assumes storages never emit
+// duplicates.
+func sortAndMerge(list []*series) []*series {
+	if len(list) < 2 {
+		return list
+	}
+	sort.Slice(list, func(i, j int) bool {
+		return labels.Compare(list[i].lset, list[j].lset) < 0
+	})
+	out := list[:1]
+	for _, s := range list[1:] {
+		last := out[len(out)-1]
+		if labels.Compare(last.lset, s.lset) != 0 {
+			out = append(out, s)
+			continue
+		}
+		merged := mergeSamples(last, s)
+		out[len(out)-1] = merged
+	}
+	return out
+}
+
+func mergeSamples(a, b *series) *series {
+	ts := make([]int64, 0, len(a.ts)+len(b.ts))
+	vs := make([]float64, 0, len(a.ts)+len(b.ts))
+	i, j := 0, 0
+	for i < len(a.ts) && j < len(b.ts) {
+		switch {
+		case a.ts[i] < b.ts[j]:
+			ts = append(ts, a.ts[i])
+			vs = append(vs, a.vs[i])
+			i++
+		case a.ts[i] > b.ts[j]:
+			ts = append(ts, b.ts[j])
+			vs = append(vs, b.vs[j])
+			j++
+		default:
+			ts = append(ts, a.ts[i])
+			vs = append(vs, a.vs[i])
+			i++
+			j++
+		}
+	}
+	ts = append(ts, a.ts[i:]...)
+	vs = append(vs, a.vs[i:]...)
+	ts = append(ts, b.ts[j:]...)
+	vs = append(vs, b.vs[j:]...)
+	return &series{lset: a.lset, ts: ts, vs: vs}
+}

pkg/prometheus/clickhouseprometheusv2/sql.go

@@ -0,0 +1,164 @@
+package clickhouseprometheusv2
+
+import (
+	"fmt"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/huandu/go-sqlbuilder"
+	"github.com/prometheus/prometheus/model/labels"
+)
+
+// buildSeriesQuery renders the series lookup: one row per matched fingerprint
+// with its labels.
+func buildSeriesQuery(start, end int64, matchers []*labels.Matcher) (string, []any, error) {
+	adjustedStart, table := timeSeriesTableFor(start, end)
+	sb := sqlbuilder.NewSelectBuilder()
+	sb.Select("fingerprint", "any(labels)")
+	sb.From(fmt.Sprintf("%s.%s", databaseName, table))
+	if err := applySeriesConditions(sb, adjustedStart, end, matchers); err != nil {
+		return "", nil, err
+	}
+	sb.GroupBy("fingerprint")
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return query, args, nil
+}
+
+// buildSamplesQuery renders the samples fetch for the matched series: a
+// semi-join re-runs the series predicates against the shard-local series
+// table (complete by fingerprint co-locality, see localTimeSeriesTable — a
+// GLOBAL broadcast would ship the matched set to every shard, and ClickHouse
+// materializes the subquery's set per shard before the scan, so it still
+// engages the fingerprint primary-key column). metricNames (observed on the
+// matched series when the selector had no __name__ equality) narrows the
+// primary-key scan. A non-nil lastPerStep groups to the last sample per
+// step bucket.
+func buildSamplesQuery(start, end int64, metricNames []string, matchers []*labels.Matcher, lastPerStep *lastSamplePerStep) (string, []any, error) {
+	sb := sqlbuilder.NewSelectBuilder()
+	if lastPerStep != nil {
+		// Aliases must not shadow source columns: ClickHouse resolves aliases
+		// in WHERE too, and "max(unix_milli) AS unix_milli" would put an
+		// aggregate into the WHERE clause (error 184).
+		sb.Select("fingerprint", "max(unix_milli) AS ts", "argMax(value, unix_milli) AS val", "argMax(flags, unix_milli) AS fl")
+	} else {
+		sb.Select("fingerprint", "unix_milli", "value", "flags")
+	}
+	sb.From(fmt.Sprintf("%s.%s", databaseName, distributedSamplesV4))
+
+	switch len(metricNames) {
+	case 0:
+		// No name constraint derivable; the primary-key prefix goes unused.
+	case 1:
+		sb.Where(sb.EQ("metric_name", metricNames[0]))
+	default:
+		sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
+	}
+	// Semantically redundant (the fingerprints already come from these
+	// temporalities) but engages the leading primary-key column.
+	sb.Where("temporality IN ['Cumulative', 'Unspecified']")
+	sub := sqlbuilder.NewSelectBuilder()
+	sub.Select("fingerprint")
+	adjustedStart, table := timeSeriesTableFor(start, end)
+	sub.From(fmt.Sprintf("%s.%s", databaseName, localTimeSeriesTable(table)))
+	if err := applySeriesConditions(sub, adjustedStart, end, matchers); err != nil {
+		return "", nil, err
+	}
+	sb.Where(sb.In("fingerprint", sub))
+	sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
+
+	if lastPerStep != nil {
+		sb.GroupBy("fingerprint")
+		if expr := lastPerStep.bucketExpr(); expr != "" {
+			sb.GroupBy(expr)
+		}
+		sb.OrderBy("fingerprint", "ts")
+	} else {
+		sb.OrderBy("fingerprint", "unix_milli")
+	}
+
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return query, args, nil
+}
+
+// applySeriesConditions adds the WHERE conditions of a series table scan:
+// __name__ matchers (all four types) translate to the metric_name column,
+// every other matcher to a JSONExtractString condition on the labels column.
+// An equality matcher against "" matches series without the label, mirroring
+// PromQL, because JSONExtractString returns "" for missing keys. Regexes are
+// anchored: PromQL matchers match the whole value, ClickHouse match()
+// searches for a substring.
+func applySeriesConditions(sb *sqlbuilder.SelectBuilder, start, end int64, matchers []*labels.Matcher) error {
+	for _, m := range matchers {
+		if m.Name != metricNameLabel {
+			continue
+		}
+		switch m.Type {
+		case labels.MatchEqual:
+			sb.Where(sb.EQ("metric_name", m.Value))
+		case labels.MatchNotEqual:
+			sb.Where(sb.NE("metric_name", m.Value))
+		case labels.MatchRegexp:
+			sb.Where(fmt.Sprintf("match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
+		case labels.MatchNotRegexp:
+			sb.Where(fmt.Sprintf("NOT match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
+		default:
+			return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported matcher type %q for __name__", m.Type)
+		}
+	}
+
+	sb.Where("temporality IN ['Cumulative', 'Unspecified']")
+	// Inclusive upper bound: registration rows are hour-floored (and 6h/1d/1w
+	// for the rollup tables) by the exporter, so a series first registered in
+	// the bucket starting exactly at `end` would otherwise be invisible while
+	// its samples (<= end) are in range.
+	sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
+
+	for _, m := range matchers {
+		if m.Name == metricNameLabel {
+			continue
+		}
+		switch m.Type {
+		case labels.MatchEqual:
+			sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) = %s", sb.Var(m.Name), sb.Var(m.Value)))
+		case labels.MatchNotEqual:
+			sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) != %s", sb.Var(m.Name), sb.Var(m.Value)))
+		case labels.MatchRegexp:
+			sb.Where(fmt.Sprintf("match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
+		case labels.MatchNotRegexp:
+			sb.Where(fmt.Sprintf("NOT match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
+		default:
+			return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported matcher type %q", m.Type)
+		}
+	}
+
+	return nil
+}
+
+// anchorRegex turns a PromQL regex into its fully-anchored form (see
+// applySeriesConditions).
+func anchorRegex(v string) string {
+	return "^(?:" + v + ")$"
+}
+
+// lastSamplePerStep reduces an instant-selector fetch to the last sample of
+// each step bucket: bucket 0 is (start, firstEval], bucket i is
+// (firstEval+(i-1)·step, firstEval+i·step]. Anchoring buckets at the first
+// evaluation timestamp makes every bucket boundary an evaluation timestamp,
+// so a non-final sample of a bucket can never be the latest sample in any
+// (t-lookback, t] the engine resolves — the reduction is lossless. Real
+// timestamps are preserved, so the engine's own lookback and staleness
+// handling remain exact.
+type lastSamplePerStep struct {
+	firstEvalMs int64
+	stepMs      int64
+}
+
+func (t *lastSamplePerStep) bucketExpr() string {
+	if t.stepMs <= 0 {
+		// Instant query: a single evaluation at firstEval; one bucket.
+		return ""
+	}
+	return fmt.Sprintf(
+		"if(unix_milli <= %d, 0, intDiv(unix_milli - %d - 1, %d) + 1)",
+		t.firstEvalMs, t.firstEvalMs, t.stepMs,
+	)
+}

pkg/prometheus/clickhouseprometheusv2/sql_test.go

@@ -0,0 +1,139 @@
+package clickhouseprometheusv2
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func mustMatcher(t *testing.T, mt labels.MatchType, name, value string) *labels.Matcher {
+	t.Helper()
+	m, err := labels.NewMatcher(mt, name, value)
+	require.NoError(t, err)
+	return m
+}
+
+func TestTimeSeriesTableFor(t *testing.T) {
+	base := time.Date(2026, 7, 10, 3, 27, 0, 0, time.UTC).UnixMilli()
+
+	tests := []struct {
+		name      string
+		span      time.Duration
+		wantTable string
+		roundTo   time.Duration
+	}{
+		{"under 6h uses hourly table", 2 * time.Hour, distributedTimeSeriesV4, time.Hour},
+		{"under 1d uses 6h table", 12 * time.Hour, distributedTimeSeriesV46hrs, 6 * time.Hour},
+		{"under 1w uses 1d table", 3 * 24 * time.Hour, distributedTimeSeriesV41day, 24 * time.Hour},
+		{"over 1w uses 1w table", 10 * 24 * time.Hour, distributedTimeSeriesV41week, 7 * 24 * time.Hour},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			start, table := timeSeriesTableFor(base, base+tt.span.Milliseconds())
+			assert.Equal(t, tt.wantTable, table)
+			assert.Zero(t, start%tt.roundTo.Milliseconds())
+			assert.LessOrEqual(t, start, base)
+		})
+	}
+}
+
+func TestBuildSeriesQuery(t *testing.T) {
+	start := int64(1_700_000_000_000)
+	end := start + time.Hour.Milliseconds()
+	// The series table window rounds down to the table's bucket boundary.
+	adjustedStart := start - (start % time.Hour.Milliseconds())
+
+	t.Run("equality name and label matchers", func(t *testing.T) {
+		query, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
+			mustMatcher(t, labels.MatchEqual, "__name__", "http_requests_total"),
+			mustMatcher(t, labels.MatchEqual, "job", "api"),
+		})
+		require.NoError(t, err)
+		assert.Equal(t,
+			"SELECT fingerprint, any(labels) FROM signoz_metrics.distributed_time_series_v4 WHERE metric_name = ? AND temporality IN ['Cumulative', 'Unspecified'] AND unix_milli >= ? AND unix_milli <= ? AND JSONExtractString(labels, ?) = ? GROUP BY fingerprint",
+			query,
+		)
+		assert.Equal(t, []any{"http_requests_total", adjustedStart, end, "job", "api"}, args)
+	})
+
+	t.Run("regex matchers are anchored", func(t *testing.T) {
+		_, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
+			mustMatcher(t, labels.MatchEqual, "__name__", "up"),
+			mustMatcher(t, labels.MatchRegexp, "instance", "prod.*"),
+			mustMatcher(t, labels.MatchNotRegexp, "env", "dev|test"),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, []any{"up", adjustedStart, end, "instance", "^(?:prod.*)$", "env", "^(?:dev|test)$"}, args)
+	})
+
+	t.Run("regex name matcher uses metric_name column", func(t *testing.T) {
+		query, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
+			mustMatcher(t, labels.MatchRegexp, "__name__", "node_cpu.*|node_memory.*"),
+		})
+		require.NoError(t, err)
+		assert.Contains(t, query, "match(metric_name, ?)")
+		assert.NotContains(t, query, "JSONExtractString")
+		assert.Equal(t, []any{"^(?:node_cpu.*|node_memory.*)$", adjustedStart, end}, args)
+	})
+
+	t.Run("no name matcher omits metric_name condition", func(t *testing.T) {
+		query, _, err := buildSeriesQuery(start, end, []*labels.Matcher{
+			mustMatcher(t, labels.MatchEqual, "job", "api"),
+		})
+		require.NoError(t, err)
+		assert.NotContains(t, query, "metric_name")
+	})
+}
+
+func TestBuildSamplesQuery(t *testing.T) {
+	start := int64(1_700_000_000_000)
+	end := start + time.Hour.Milliseconds()
+	adjustedStart := start - (start % time.Hour.Milliseconds())
+	matchers := []*labels.Matcher{
+		mustMatcher(t, labels.MatchEqual, "__name__", "up"),
+		mustMatcher(t, labels.MatchEqual, "job", "api"),
+	}
+
+	t.Run("raw fetch filters by a shard-local semi-join", func(t *testing.T) {
+		query, args, err := buildSamplesQuery(start, end, []string{"up"}, matchers, nil)
+		require.NoError(t, err)
+		assert.Contains(t, query, "fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE ")
+		assert.NotContains(t, query, "GLOBAL IN")
+		assert.Contains(t, query, "ORDER BY fingerprint, unix_milli")
+		// Args follow placeholder order: samples metric name, the semi-join's
+		// series predicates, then the samples window bounds.
+		assert.Equal(t, []any{"up", "up", adjustedStart, end, "job", "api", start, end}, args)
+	})
+
+	t.Run("last-sample-per-step groups by step bucket anchored at first eval", func(t *testing.T) {
+		lastPerStep := &lastSamplePerStep{firstEvalMs: start + 299_999, stepMs: 60_000}
+		query, _, err := buildSamplesQuery(start, end, []string{"up"}, matchers, lastPerStep)
+		require.NoError(t, err)
+		assert.Contains(t, query, "argMax(value, unix_milli) AS val")
+		assert.Contains(t, query, "argMax(flags, unix_milli) AS fl")
+		assert.Contains(t, query, "GROUP BY fingerprint, if(unix_milli <= 1700000299999, 0, intDiv(unix_milli - 1700000299999 - 1, 60000) + 1)")
+		assert.Contains(t, query, "ORDER BY fingerprint, ts")
+		// Aliases must not shadow the source columns referenced in WHERE.
+		assert.NotContains(t, query, "AS unix_milli")
+		assert.NotContains(t, query, "AS value")
+		assert.NotContains(t, query, "AS flags")
+	})
+
+	t.Run("instant query keeps one bucket", func(t *testing.T) {
+		lastPerStep := &lastSamplePerStep{firstEvalMs: end, stepMs: 0}
+		query, _, err := buildSamplesQuery(start, end, []string{"up"}, matchers, lastPerStep)
+		require.NoError(t, err)
+		assert.Contains(t, query, "GROUP BY fingerprint ORDER BY fingerprint, ts")
+		assert.NotContains(t, query, "intDiv")
+	})
+
+	t.Run("multiple metric names from regex selector", func(t *testing.T) {
+		query, args, err := buildSamplesQuery(start, end, []string{"node_cpu", "node_memory"}, matchers, nil)
+		require.NoError(t, err)
+		assert.Contains(t, query, "metric_name IN (?, ?)")
+		assert.Equal(t, []any{"node_cpu", "node_memory", "up", adjustedStart, end, "job", "api", start, end}, args)
+	})
+}

pkg/prometheus/clickhouseprometheusv2/tables.go

@@ -0,0 +1,70 @@
+package clickhouseprometheusv2
+
+import "time"
+
+// TODO(srikanthccv): consolidate the metrics table names and the
+// window-to-table selection across the query builders (telemetrymetrics)
+// and the prometheus providers instead of each package carrying its own
+// copy.
+
+const (
+	// metricNameLabel is the reserved PromQL label holding the metric name.
+	metricNameLabel string = "__name__"
+
+	databaseName                 string = "signoz_metrics"
+	distributedTimeSeriesV4      string = "distributed_time_series_v4"
+	distributedTimeSeriesV46hrs  string = "distributed_time_series_v4_6hrs"
+	distributedTimeSeriesV41day  string = "distributed_time_series_v4_1day"
+	distributedTimeSeriesV41week string = "distributed_time_series_v4_1week"
+	distributedSamplesV4         string = "distributed_samples_v4"
+
+	localTimeSeriesV4      string = "time_series_v4"
+	localTimeSeriesV46hrs  string = "time_series_v4_6hrs"
+	localTimeSeriesV41day  string = "time_series_v4_1day"
+	localTimeSeriesV41week string = "time_series_v4_1week"
+)
+
+// localTimeSeriesTable maps a distributed time series table to its shard-local
+// table. Samples and time series shard on the same key
+// (cityHash64(env, temporality, metric_name, fingerprint)), so a query whose
+// top-level FROM is the distributed samples table can join or semi-join the
+// local time series table inside each shard: the shard rewrite runs the
+// subquery against the shard's own series rows, which are exactly the series
+// of the shard's samples. No broadcast, no initiator-side join.
+func localTimeSeriesTable(distributed string) string {
+	switch distributed {
+	case distributedTimeSeriesV46hrs:
+		return localTimeSeriesV46hrs
+	case distributedTimeSeriesV41day:
+		return localTimeSeriesV41day
+	case distributedTimeSeriesV41week:
+		return localTimeSeriesV41week
+	default:
+		return localTimeSeriesV4
+	}
+}
+
+var (
+	oneHourInMilliseconds  = time.Hour.Milliseconds()
+	sixHoursInMilliseconds = time.Hour.Milliseconds() * 6
+	oneDayInMilliseconds   = time.Hour.Milliseconds() * 24
+	oneWeekInMilliseconds  = time.Hour.Milliseconds() * 24 * 7
+)
+
+// timeSeriesTableFor returns the adjusted start and the time series table for
+// the window. Time series tables hold one row per (fingerprint, bucket), with
+// bucket granularities of 1h, 6h, 1d and 1w; the start is rounded down to the
+// bucket boundary so a window beginning mid-bucket still matches the bucket's
+// row.
+func timeSeriesTableFor(start, end int64) (int64, string) {
+	switch {
+	case end-start < sixHoursInMilliseconds:
+		return start - (start % oneHourInMilliseconds), distributedTimeSeriesV4
+	case end-start < oneDayInMilliseconds:
+		return start - (start % sixHoursInMilliseconds), distributedTimeSeriesV46hrs
+	case end-start < oneWeekInMilliseconds:
+		return start - (start % oneDayInMilliseconds), distributedTimeSeriesV41day
+	default:
+		return start - (start % oneWeekInMilliseconds), distributedTimeSeriesV41week
+	}
+}

pkg/prometheus/config.go

@@ -24,6 +24,10 @@ type Config struct {
 
 	// Timeout is the maximum time a query is allowed to run before being aborted.
 	Timeout time.Duration `mapstructure:"timeout"`
+
+	// ProviderName selects the storage provider: "clickhouse" (default) or
+	// "clickhousev2".
+	ProviderName string `mapstructure:"provider"`
 }
 
 func NewConfigFactory() factory.ConfigFactory {
@@ -37,7 +41,8 @@ func newConfig() factory.Config {
 			Path:          "",
 			MaxConcurrent: 20,
 		},
-		Timeout: 2 * time.Minute,
+		Timeout:      2 * time.Minute,
+		ProviderName: "clickhouse",
 	}
 }
 
@@ -45,9 +50,15 @@ func (c Config) Validate() error {
 	if c.Timeout <= 0 {
 		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::timeout must be greater than 0")
 	}
+	if c.ProviderName != "" && c.ProviderName != "clickhouse" && c.ProviderName != "clickhousev2" {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::provider must be one of [clickhouse, clickhousev2], got %q", c.ProviderName)
+	}
 	return nil
 }
 
 func (c Config) Provider() string {
-	return "clickhouse"
+	if c.ProviderName == "" {
+		return "clickhouse"
+	}
+	return c.ProviderName
 }

pkg/prometheus/prometheus.go

@@ -35,3 +35,9 @@ type StatementRecorder interface {
 type StatementCapturer interface {
 	CapturingStorage() (storage.Queryable, StatementRecorder)
 }
+
+// ProviderClickhouseV2 is the clickhousev2 provider name: the factory
+// registration, the prometheus::provider config value and the
+// X-SigNoz-PromQL-Provider request header all use it, so they cannot drift
+// apart.
+const ProviderClickhouseV2 = "clickhousev2"

pkg/prometheus/traits.go

@@ -0,0 +1,49 @@
+package prometheus
+
+import (
+	"context"
+
+	"github.com/prometheus/prometheus/promql/parser"
+)
+
+type queryTraitsKey struct{}
+
+// QueryTraits carries per-query facts a storage implementation cannot derive
+// from SelectHints alone. Call sites that parse the PromQL expression attach
+// traits to the context before handing it to the engine; storages treat a
+// missing traits value as "unknown" and stay conservative.
+type QueryTraits struct {
+	// SubqueryFree is true when the query contains no subquery expression.
+	// Subquery selectors are evaluated at the subquery's own step, but
+	// SelectHints.Step always carries the top-level step, so step-aligned
+	// storage optimizations (e.g. keeping only the last sample per step
+	// bucket) are safe only when this is true.
+	SubqueryFree bool
+}
+
+// DetectQueryTraits derives QueryTraits from a parsed PromQL expression.
+func DetectQueryTraits(expr parser.Expr) QueryTraits {
+	subqueryFree := true
+	parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error {
+		if _, ok := node.(*parser.SubqueryExpr); ok {
+			subqueryFree = false
+		}
+		return nil
+	})
+	return QueryTraits{SubqueryFree: subqueryFree}
+}
+
+// NewContextWithQueryTraits returns a context carrying the given traits.
+func NewContextWithQueryTraits(ctx context.Context, traits QueryTraits) context.Context {
+	return context.WithValue(ctx, queryTraitsKey{}, traits)
+}
+
+// QueryTraitsFromContext returns the traits attached to ctx, if any.
+//
+// Context is used here, unlike for backend selection, because traits must
+// cross the promql engine to reach storage.Querier.Select, and the engine's
+// interfaces offer no other channel; the alternative is a Prometheus fork.
+func QueryTraitsFromContext(ctx context.Context) (QueryTraits, bool) {
+	traits, ok := ctx.Value(queryTraitsKey{}).(QueryTraits)
+	return traits, ok
+}

pkg/querier/api.go

@@ -50,6 +50,7 @@ func (handler *handler) QueryRange(rw http.ResponseWriter, req *http.Request) {
 		render.Error(rw, err)
 		return
 	}
+	queryRangeRequest.PromQLProvider = req.Header.Get("X-SigNoz-PromQL-Provider")
 
 	// Validate the query request
 	if err := queryRangeRequest.Validate(); err != nil {

pkg/querier/clickhouse_query.go

@@ -100,9 +100,20 @@ func (q *chSQLQuery) renderVars(query string, vars map[string]qbtypes.VariableIt
 	return newQuery.String(), nil
 }
 
-// Statement renders the SQL without executing it, for the preview path.
-func (q *chSQLQuery) Statement(_ context.Context) (*qbtypes.Statement, error) {
+func (q *chSQLQuery) render(ctx context.Context) (string, error) {
 	rendered, err := q.renderVars(q.query.Query, q.vars, q.fromMS, q.toMS)
+	if err != nil {
+		return "", err
+	}
+
+	querybuilder.LogIfStatementIsNotValid(ctx, q.logger, rendered)
+
+	return rendered, nil
+}
+
+// Statement renders the SQL without executing it, for the preview path.
+func (q *chSQLQuery) Statement(ctx context.Context) (*qbtypes.Statement, error) {
+	rendered, err := q.render(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -124,7 +135,7 @@ func (q *chSQLQuery) Execute(ctx context.Context) (*qbtypes.Result, error) {
 		elapsed += p.Elapsed
 	}))
 
-	query, err := q.renderVars(q.query.Query, q.vars, q.fromMS, q.toMS)
+	query, err := q.render(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -135,11 +146,11 @@ func (q *chSQLQuery) Execute(ctx context.Context) (*qbtypes.Result, error) {
 	}
 	defer rows.Close()
 
-	// TODO: map the errors from ClickHouse to our error types
 	payload, err := consume(rows, q.kind, nil, qbtypes.Step{}, q.query.Name)
 	if err != nil {
 		return nil, err
 	}
+
 	return &qbtypes.Result{
 		Type:  q.kind,
 		Value: payload,

pkg/querier/preview.go

@@ -231,7 +231,7 @@ func (q *querier) buildPreviewProviders(
 			sub.CompositeQuery = qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{query}}
 		}
 
-		built, _, bErr := q.buildQueries(orgID, &sub, deps, missingMetricQuerySet, event)
+		built, _, bErr := q.buildQueries(orgID, &sub, deps, missingMetricQuerySet, event, promqlOptions{})
 		if bErr != nil {
 			errs[name] = bErr
 			continue

pkg/querier/promql_query.go

@@ -8,15 +8,19 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"sync"
 	"text/template"
 	"time"
 
+	"github.com/ClickHouse/clickhouse-go/v2"
+
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
 	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
@@ -98,6 +102,24 @@ type promqlQuery struct {
 	tr          qbv5.TimeRange
 	requestType qbv5.RequestType
 	vars        map[string]qbv5.VariableItem
+	opts        promqlOptions
+}
+
+// promqlOptions is how a PromQL query relates to the clickhousev2 provider
+// (see querier.promqlOptions for where the fields come from and why they are
+// flag-gated). Both providers are nil for a plain request, so a plain
+// request costs nothing extra.
+type promqlOptions struct {
+	// shadow, when set, runs the query on this provider after serving and
+	// logs any result difference; the response is never affected.
+	shadow *clickhouseprometheusv2.Provider
+	// shadowSlots is the querier-wide admission for shadow runs, shared by
+	// every query so the bound holds per process.
+	shadowSlots chan struct{}
+	// serve, when set, serves the response from this provider instead of the
+	// default path. Comparison callers fetch the default and the pinned
+	// result as two API calls and diff them.
+	serve *clickhouseprometheusv2.Provider
 }
 
 var _ qbv5.Query = (*promqlQuery)(nil)
@@ -110,6 +132,7 @@ func newPromqlQuery(
 	tr qbv5.TimeRange,
 	requestType qbv5.RequestType,
 	variables map[string]qbv5.VariableItem,
+	opts promqlOptions,
 ) *promqlQuery {
 	return &promqlQuery{
 		logger:      logger,
@@ -119,10 +142,19 @@ func newPromqlQuery(
 		tr:          tr,
 		requestType: requestType,
 		vars:        variables,
+		opts:        opts,
 	}
 }
 
 func (q *promqlQuery) Fingerprint() string {
+	// A pinned request must not share cache entries with default serving: a
+	// cached default result would satisfy the pin without running the pinned
+	// provider, and a pinned result would poison normal serving. No
+	// fingerprint means no caching at all — the pin exists to observe a
+	// provider, so a cache in front of it defeats the point.
+	if q.opts.serve != nil {
+		return ""
+	}
 	if q.requestType != qbv5.RequestTypeTimeSeries {
 		return ""
 	}
@@ -252,7 +284,16 @@ func (q *promqlQuery) PreviewStatements(ctx context.Context) ([]prometheus.Captu
 	start := int64(querybuilder.ToNanoSecs(q.tr.From))
 	end := int64(querybuilder.ToNanoSecs(q.tr.To))
 
+	// Attach the same query traits as Execute so the captured statements
+	// match what the live path would run.
+	if expr, parseErr := q.parser.ParseExpr(rendered); parseErr == nil {
+		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
+	}
+
 	capStorage, recorder := storer.CapturingStorage()
+	if capStorage == nil {
+		return nil, nil
+	}
 	qry, err := q.promEngine.Engine().NewRangeQuery(
 		ctx,
 		capStorage,
@@ -296,6 +337,41 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		return nil, err
 	}
 
+	// Attach query traits so the storage can prove step-aligned optimizations
+	// safe (see prometheus.QueryTraits). A parse failure surfaces below via
+	// the engine with the enhanced error message.
+	if expr, parseErr := q.parser.ParseExpr(query); parseErr == nil {
+		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
+	}
+
+	// Accumulate ClickHouse-side scan stats across every storage query this
+	// evaluation issues: progress options propagate to each ClickHouse query
+	// through the context.
+	var statsMu sync.Mutex
+	var rowsScanned, bytesScanned uint64
+	ctx = clickhouse.Context(ctx, clickhouse.WithProgress(func(p *clickhouse.Progress) {
+		statsMu.Lock()
+		rowsScanned += p.Rows
+		bytesScanned += p.Bytes
+		statsMu.Unlock()
+	}))
+
+	began := time.Now()
+
+	// A pinned provider serves directly from it: comparison callers fetch
+	// the default result and the pinned result as two API calls and diff
+	// them.
+	if q.opts.serve != nil {
+		matrix, err := q.serveFromProvider(ctx, query, start, end)
+		if err != nil {
+			if enhanced := tryEnhancePromQLExecError(err); enhanced != nil {
+				return nil, enhanced
+			}
+			return nil, err
+		}
+		return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned), nil
+	}
+
 	qry, err := q.promEngine.Engine().NewRangeQuery(
 		ctx,
 		q.promEngine.Storage(),
@@ -331,11 +407,42 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		return nil, errors.WrapInternalf(promErr, errors.CodeInternal, "error getting matrix from promql query %q", query)
 	}
 
-	excludeLabel := func(labelName string) bool {
-		if labelName == "__name__" {
-			return false
+	if q.opts.shadow != nil {
+		// Shadows detach from the request, so without admission a dashboard
+		// burst would stack unbounded ClickHouse work for up to the shadow
+		// timeout — the concurrency pattern behind the original outages.
+		// Non-blocking: at the cap the comparison is skipped, not queued;
+		// a sampled shadow stream is exactly as useful for rollout evidence.
+		select {
+		case q.opts.shadowSlots <- struct{}{}:
+			// The engine pools the result's sample slices on Close; the
+			// shadow comparison needs a stable copy of what was served.
+			served := copyMatrix(matrix)
+			servedIn := time.Since(began)
+			go func() {
+				defer func() { <-q.opts.shadowSlots }()
+				q.runShadowCompare(context.WithoutCancel(ctx), query, start, end, served, servedIn)
+			}()
+		default:
+			q.logger.DebugContext(ctx, "promql shadow skipped: at concurrency cap", slog.String("query", query))
 		}
-		return strings.HasPrefix(labelName, "__") || labelName == "fingerprint"
+	}
+
+	warnings, _ := res.Warnings.AsStrings(query, 10, 0)
+	return q.toResult(matrix, warnings, began, &statsMu, &rowsScanned, &bytesScanned), nil
+}
+
+// toResult converts an evaluated matrix into the v5 result shape, attaching
+// the ClickHouse scan stats accumulated during evaluation.
+func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) *qbv5.Result {
+	// Hide only known SigNoz storage keys: label names are user data and may
+	// legitimately start with "__" (e.g. __address__), so a blanket dunder
+	// strip mangles user labelsets. The __scope./__resource. prefixes cover
+	// every exporter version's keys.
+	excludeLabel := func(labelName string) bool {
+		return labelName == "__temporality__" ||
+			strings.HasPrefix(labelName, "__scope.") ||
+			strings.HasPrefix(labelName, "__resource.")
 	}
 
 	var series []*qbv5.TimeSeries
@@ -363,7 +470,13 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		series = append(series, &s)
 	}
 
-	warnings, _ := res.Warnings.AsStrings(query, 10, 0)
+	statsMu.Lock()
+	stats := qbv5.ExecStats{
+		RowsScanned:  *rowsScanned,
+		BytesScanned: *bytesScanned,
+		DurationMS:   uint64(time.Since(began).Milliseconds()),
+	}
+	statsMu.Unlock()
 
 	tsData := &qbv5.TimeSeriesData{
 		QueryName: q.query.Name,
@@ -397,6 +510,6 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		Type:     q.requestType,
 		Value:    payload,
 		Warnings: warnings,
-		// TODO: map promql stats?
-	}, nil
+		Stats:    stats,
+	}
 }

pkg/querier/promql_query_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/stretchr/testify/assert"
 )
@@ -440,3 +441,15 @@ func TestQuotedMetricOutsideBracesPattern(t *testing.T) {
 		})
 	}
 }
+
+// A pinned request must not share cache entries with default serving: a
+// cached default result would satisfy the pin without running the pinned
+// provider.
+func TestFingerprint_PinnedProviderBypassesCache(t *testing.T) {
+	q := &promqlQuery{
+		logger: slog.Default(),
+		query:  qbv5.PromQuery{Query: "up"},
+		opts:   promqlOptions{serve: &clickhouseprometheusv2.Provider{}},
+	}
+	assert.Empty(t, q.Fingerprint())
+}

pkg/querier/promql_shadow.go

@@ -0,0 +1,176 @@
+package querier
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+)
+
+// shadowTimeout bounds a shadow evaluation; a shadow run must never outlive
+// the request by much or pile up.
+const shadowTimeout = 2 * time.Minute
+
+// runShadowCompare executes the query on the clickhousev2 provider exactly
+// as it would serve (the engine over the v2 querier), compares against the
+// served result and logs the outcome. Serving is never affected: this runs
+// after the response, off the request context, and only logs. The mismatch
+// and failure logs are the rollout evidence — serving cuts over to v2 only
+// after they stay clean.
+func (q *promqlQuery) runShadowCompare(ctx context.Context, query string, startNs, endNs int64, served promql.Matrix, servedIn time.Duration) {
+	defer func() {
+		if r := recover(); r != nil {
+			q.logger.ErrorContext(ctx, "promql shadow comparison panicked", slog.Any("panic", r), slog.String("query", query))
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(ctx, shadowTimeout)
+	defer cancel()
+
+	// The request context carries the served response's scan-stats progress
+	// callback; without replacing it the shadow's ClickHouse progress would
+	// race into the served stats. The response itself was already sent.
+	ctx = clickhouse.Context(ctx, clickhouse.WithProgress(func(*clickhouse.Progress) {}))
+
+	if expr, parseErr := q.parser.ParseExpr(query); parseErr == nil {
+		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
+	}
+
+	start, end := time.Unix(0, startNs), time.Unix(0, endNs)
+	began := time.Now()
+	shadow, err := executeOnProvider(ctx, q.opts.shadow, query, start, end, q.query.Step.Duration)
+	shadowIn := time.Since(began)
+
+	logAttrs := []any{
+		slog.String("query", query),
+		slog.Int64("start_ms", startNs/int64(time.Millisecond)),
+		slog.Int64("end_ms", endNs/int64(time.Millisecond)),
+		slog.Duration("step", q.query.Step.Duration),
+		slog.Duration("served_in", servedIn),
+		slog.Duration("shadow_in", shadowIn),
+	}
+
+	if err != nil {
+		// A shadow failure would be a serving failure after rollout; surface
+		// it at the same level as a result mismatch.
+		q.logger.WarnContext(ctx, "promql shadow execution failed", append(logAttrs, slog.Any("error", err))...)
+		return
+	}
+
+	servedNorm := normalizeShadowMatrix(served)
+	shadowNorm := normalizeShadowMatrix(shadow)
+	if diff := diffShadowMatrices(servedNorm, shadowNorm); diff != "" {
+		q.logger.WarnContext(ctx, "promql shadow comparison mismatch", append(logAttrs,
+			slog.String("diff", diff),
+			slog.Int("served_series", len(servedNorm)),
+			slog.Int("shadow_series", len(shadowNorm)),
+		)...)
+		return
+	}
+	// Matches log the timings: served_in vs shadow_in across the fleet is
+	// the perf evidence for the cutover, gathered for free.
+	q.logger.DebugContext(ctx, "promql shadow comparison matched", logAttrs...)
+}
+
+// serveFromProvider evaluates the query the way the pinned provider would
+// serve it.
+func (q *promqlQuery) serveFromProvider(ctx context.Context, query string, startNs, endNs int64) (promql.Matrix, error) {
+	return executeOnProvider(ctx, q.opts.serve, query, time.Unix(0, startNs), time.Unix(0, endNs), q.query.Step.Duration)
+}
+
+// executeOnProvider evaluates the query the way the provider would serve it:
+// the engine over the provider's storage. The returned matrix is an owned
+// copy.
+func executeOnProvider(ctx context.Context, prov *clickhouseprometheusv2.Provider, query string, start, end time.Time, step time.Duration) (promql.Matrix, error) {
+	qry, err := prov.Engine().NewRangeQuery(ctx, prov.Storage(), nil, query, start, end, step)
+	if err != nil {
+		return nil, err
+	}
+	defer qry.Close()
+
+	res := qry.Exec(ctx)
+	if res.Err != nil {
+		return nil, res.Err
+	}
+	matrix, err := res.Matrix()
+	if err != nil {
+		return nil, err
+	}
+	// Close returns the result's sample slices to the engine pool.
+	return copyMatrix(matrix), nil
+}
+
+func copyMatrix(matrix promql.Matrix) promql.Matrix {
+	out := make(promql.Matrix, 0, len(matrix))
+	for _, s := range matrix {
+		floats := make([]promql.FPoint, len(s.Floats))
+		copy(floats, s.Floats)
+		out = append(out, promql.Series{Metric: s.Metric.Copy(), Floats: floats})
+	}
+	return out
+}
+
+// normalizeShadowMatrix sorts by label set for order-independent
+// comparison. Both providers now resolve series identity the same way
+// (empty-valued labels dropped at read, no synthetic fingerprint label
+// since the v1 series-identity fix), so labels need no normalization.
+func normalizeShadowMatrix(matrix promql.Matrix) promql.Matrix {
+	out := make(promql.Matrix, 0, len(matrix))
+	out = append(out, matrix...)
+	sort.Slice(out, func(i, j int) bool { return labels.Compare(out[i].Metric, out[j].Metric) < 0 })
+	return out
+}
+
+// diffShadowMatrices returns a description of the first difference, or "".
+// Values compare with relative tolerance: spatial aggregations accumulate
+// floats in storage order, which differs between the providers in the last
+// ULP.
+func diffShadowMatrices(served, shadow promql.Matrix) string {
+	const relTol = 1e-9
+	if len(served) != len(shadow) {
+		return fmt.Sprintf("series count: served=%d shadow=%d", len(served), len(shadow))
+	}
+	for i := range served {
+		if labels.Compare(served[i].Metric, shadow[i].Metric) != 0 {
+			return fmt.Sprintf("series %d labels: served=%s shadow=%s", i, served[i].Metric, shadow[i].Metric)
+		}
+		if len(served[i].Floats) != len(shadow[i].Floats) {
+			return fmt.Sprintf("series %s points: served=%d shadow=%d", served[i].Metric, len(served[i].Floats), len(shadow[i].Floats))
+		}
+		for j := range served[i].Floats {
+			a, b := served[i].Floats[j], shadow[i].Floats[j]
+			if a.T != b.T {
+				return fmt.Sprintf("series %s point %d ts: served=%d shadow=%d", served[i].Metric, j, a.T, b.T)
+			}
+			// NaN and infinities first: NaN != NaN and Inf-Inf arithmetic
+			// would otherwise make one-sided NaN and Inf-vs-finite compare
+			// as equal (NaN > x and Inf > Inf are both false).
+			if math.IsNaN(a.F) || math.IsNaN(b.F) {
+				if math.IsNaN(a.F) != math.IsNaN(b.F) {
+					return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
+				}
+				continue
+			}
+			if math.IsInf(a.F, 0) || math.IsInf(b.F, 0) {
+				if a.F != b.F {
+					return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
+				}
+				continue
+			}
+			diff := math.Abs(a.F - b.F)
+			scale := math.Max(math.Abs(a.F), math.Abs(b.F))
+			if diff > relTol*math.Max(scale, 1e-300) && diff > 1e-12 {
+				return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
+			}
+		}
+	}
+	return ""
+}

pkg/querier/promql_shadow_test.go

@@ -0,0 +1,67 @@
+package querier
+
+import (
+	"math"
+	"testing"
+
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizeShadowMatrix(t *testing.T) {
+	matrix := promql.Matrix{
+		{
+			Metric: labels.FromStrings("__name__", "up", "job", "api"),
+			Floats: []promql.FPoint{{T: 1000, F: 1}},
+		},
+		{
+			Metric: labels.FromStrings("a", "1"),
+			Floats: []promql.FPoint{{T: 1000, F: 2}},
+		},
+	}
+	norm := normalizeShadowMatrix(matrix)
+	// sorted by label set; labels pass through untouched — both providers
+	// resolve series identity identically since the v1 series-identity fix
+	assert.Equal(t, labels.FromStrings("__name__", "up", "job", "api"), norm[0].Metric)
+	assert.Equal(t, labels.FromStrings("a", "1"), norm[1].Metric)
+}
+
+func TestDiffShadowMatrices(t *testing.T) {
+	series := func(v float64) promql.Matrix {
+		return promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 1000, F: v}}}}
+	}
+
+	assert.Empty(t, diffShadowMatrices(series(1.5), series(1.5)))
+	// last-ULP differences from storage-order float accumulation are expected
+	assert.Empty(t, diffShadowMatrices(series(0.08888888888888889), series(0.08888888888888888)))
+	assert.Empty(t, diffShadowMatrices(series(math.NaN()), series(math.NaN())))
+
+	assert.Contains(t, diffShadowMatrices(series(1.5), series(1.6)), "value")
+	assert.Contains(t, diffShadowMatrices(series(1.5), promql.Matrix{}), "series count")
+	assert.Contains(t, diffShadowMatrices(
+		series(1.5),
+		promql.Matrix{{Metric: labels.FromStrings("a", "2"), Floats: []promql.FPoint{{T: 1000, F: 1.5}}}},
+	), "labels")
+	assert.Contains(t, diffShadowMatrices(
+		series(1.5),
+		promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 2000, F: 1.5}}}},
+	), "ts")
+}
+
+// One-sided NaN makes every float comparison false, and Inf-Inf arithmetic
+// yields Inf > Inf == false; without explicit handling both divergences log
+// as matched — a shadow comparator that cannot see them would green-light a
+// broken rollout.
+func TestDiffShadowMatrices_SpecialFloats(t *testing.T) {
+	point := func(v float64) promql.Matrix {
+		return promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 1000, F: v}}}}
+	}
+
+	assert.NotEmpty(t, diffShadowMatrices(point(math.NaN()), point(1.5)), "one-sided NaN must diff")
+	assert.NotEmpty(t, diffShadowMatrices(point(1.5), point(math.NaN())), "one-sided NaN must diff either way")
+	assert.NotEmpty(t, diffShadowMatrices(point(math.Inf(1)), point(1.5)), "Inf vs finite must diff")
+	assert.NotEmpty(t, diffShadowMatrices(point(math.Inf(1)), point(math.Inf(-1))), "opposite infinities must diff")
+	assert.Empty(t, diffShadowMatrices(point(math.Inf(1)), point(math.Inf(1))), "equal infinities match")
+	assert.Empty(t, diffShadowMatrices(point(math.NaN()), point(math.NaN())), "both NaN match")
+}

pkg/querier/querier.go

@@ -19,10 +19,12 @@ import (
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/query-service/utils"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
 	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
+	"github.com/SigNoz/signoz/pkg/types/featuretypes"
 	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
 	"github.com/SigNoz/signoz/pkg/types/metrictypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -36,11 +38,19 @@ var (
 )
 
 type querier struct {
-	logger                   *slog.Logger
-	fl                       flagger.Flagger
-	telemetryStore           telemetrystore.TelemetryStore
-	metadataStore            telemetrytypes.MetadataStore
-	promEngine               prometheus.Prometheus
+	logger         *slog.Logger
+	fl             flagger.Flagger
+	telemetryStore telemetrystore.TelemetryStore
+	metadataStore  telemetrytypes.MetadataStore
+	promEngine     prometheus.Prometheus
+	// promV2 is the clickhousev2 prometheus provider, wired only when the
+	// serving provider is the default one (nil otherwise). It reads the same
+	// ClickHouse data through a different implementation; PromQL queries
+	// shadow-compare against it behind the use_prometheus_clickhouse_v2 flag
+	// and can be pinned to it for a response (see promqlOptions). It never
+	// serves by default — that cutover happens only after the shadow logs
+	// stay clean.
+	promV2                   *clickhouseprometheusv2.Provider
 	traceStmtBuilder         qbtypes.StatementBuilder[qbtypes.TraceAggregation]
 	logStmtBuilder           qbtypes.StatementBuilder[qbtypes.LogAggregation]
 	auditStmtBuilder         qbtypes.StatementBuilder[qbtypes.LogAggregation]
@@ -51,8 +61,16 @@ type querier struct {
 	liveDataRefresh          time.Duration
 	builderConfig            builderConfig
 	maxConcurrentQueries     int
+	// shadowSlots bounds concurrent shadow comparisons per process; shadows
+	// detach from their requests, so nothing else limits how many pile up.
+	shadowSlots chan struct{}
 }
 
+// maxConcurrentShadows is deliberately small: a shadow is a full extra
+// ClickHouse evaluation, and a sampled stream of comparisons is exactly as
+// useful for rollout evidence as an exhaustive one under load.
+const maxConcurrentShadows = 8
+
 var _ Querier = (*querier)(nil)
 
 func New(
@@ -60,6 +78,7 @@ func New(
 	telemetryStore telemetrystore.TelemetryStore,
 	metadataStore telemetrytypes.MetadataStore,
 	promEngine prometheus.Prometheus,
+	promV2 *clickhouseprometheusv2.Provider,
 	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
@@ -81,6 +100,7 @@ func New(
 		telemetryStore:           telemetryStore,
 		metadataStore:            metadataStore,
 		promEngine:               promEngine,
+		promV2:                   promV2,
 		traceStmtBuilder:         traceStmtBuilder,
 		logStmtBuilder:           logStmtBuilder,
 		auditStmtBuilder:         auditStmtBuilder,
@@ -93,6 +113,7 @@ func New(
 			logTraceIDWindowPaddingMS: uint64(logTraceIDWindowPadding.Milliseconds()),
 		},
 		maxConcurrentQueries: maxConcurrentQueries,
+		shadowSlots:          make(chan struct{}, maxConcurrentShadows),
 	}
 }
 
@@ -132,7 +153,11 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 		missingMetricQuerySet[name] = true
 	}
 
-	queries, steps, err := q.buildQueries(orgID, req, dependencyQueries, missingMetricQuerySet, event)
+	promqlOpts, err := q.promqlOptions(ctx, orgID, req)
+	if err != nil {
+		return nil, err
+	}
+	queries, steps, err := q.buildQueries(orgID, req, dependencyQueries, missingMetricQuerySet, event, promqlOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -175,12 +200,41 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 	return qbResp, qbErr
 }
 
+// promqlOptions derives the PromQL execution options for a request. With the
+// org's use_prometheus_clickhouse_v2 flag on, queries are shadow-compared
+// against the clickhousev2 provider (serving unaffected, diffs logged; see
+// promql_shadow.go). The X-SigNoz-PromQL-Provider header may instead pin the
+// response to that provider — integration tests and support fetch both
+// results for comparison — so it is deliberately flag-gated too: without the
+// gate the header would be an unaudited switch onto a provider still under
+// validation.
+func (q *querier) promqlOptions(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (promqlOptions, error) {
+	enabled := q.fl.BooleanOrEmpty(ctx, flagger.FeatureUsePrometheusClickhouseV2, featuretypes.NewFlaggerEvaluationContext(orgID))
+	if req.PromQLProvider == "" {
+		if enabled && q.promV2 != nil {
+			return promqlOptions{shadow: q.promV2, shadowSlots: q.shadowSlots}, nil
+		}
+		return promqlOptions{}, nil
+	}
+	if req.PromQLProvider != prometheus.ProviderClickhouseV2 {
+		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "unknown promql provider %q", req.PromQLProvider)
+	}
+	if !enabled {
+		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "promql provider %q requires the use_prometheus_clickhouse_v2 flag", req.PromQLProvider)
+	}
+	if q.promV2 == nil {
+		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "promql provider %q is not available", req.PromQLProvider)
+	}
+	return promqlOptions{serve: q.promV2}, nil
+}
+
 func (q *querier) buildQueries(
 	orgID valuer.UUID,
 	req *qbtypes.QueryRangeRequest,
 	dependencyQueries map[string]bool,
 	missingMetricQuerySet map[string]bool,
 	event *qbtypes.QBEvent,
+	promqlOpts promqlOptions,
 ) (map[string]qbtypes.Query, map[string]qbtypes.Step, error) {
 
 	tmplVars := req.Variables
@@ -205,7 +259,7 @@ func (q *querier) buildQueries(
 			if !ok {
 				return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid promql query spec %T", query.Spec)
 			}
-			promqlQuery := newPromqlQuery(q.logger, q.promEngine, promQuery, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars)
+			promqlQuery := newPromqlQuery(q.logger, q.promEngine, promQuery, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars, promqlOpts)
 			queries[promQuery.Name] = promqlQuery
 			steps[promQuery.Name] = promQuery.Step
 		case qbtypes.QueryTypeClickHouseSQL:
@@ -848,7 +902,7 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
 	switch qt := originalQuery.(type) {
 	case *promqlQuery:
 		queryCopy := qt.query.Copy()
-		return newPromqlQuery(q.logger, q.promEngine, queryCopy, timeRange, qt.requestType, qt.vars)
+		return newPromqlQuery(q.logger, qt.promEngine, queryCopy, timeRange, qt.requestType, qt.vars, qt.opts)
 
 	case *chSQLQuery:
 		queryCopy := qt.query.Copy()

pkg/querier/querier_test.go

@@ -48,6 +48,7 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
 		nil, // telemetryStore
 		metadataStore,
 		nil,                // prometheus
+		nil,                // promV2
 		nil,                // traceStmtBuilder
 		nil,                // logStmtBuilder
 		nil,                // auditStmtBuilder
@@ -120,6 +121,7 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
 		telemetryStore,
 		metadataStore,
 		nil,                      // prometheus
+		nil,                      // promV2
 		nil,                      // traceStmtBuilder
 		nil,                      // logStmtBuilder
 		nil,                      // auditStmtBuilder

pkg/querier/signozquerier/provider.go

@@ -7,6 +7,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/telemetryaudit"
@@ -22,6 +23,7 @@ import (
 func NewFactory(
 	telemetryStore telemetrystore.TelemetryStore,
 	prometheus prometheus.Prometheus,
+	promV2 *clickhouseprometheusv2.Provider,
 	cache cache.Cache,
 	flagger flagger.Flagger,
 ) factory.ProviderFactory[querier.Querier, querier.Config] {
@@ -32,7 +34,7 @@ func NewFactory(
 			settings factory.ProviderSettings,
 			cfg querier.Config,
 		) (querier.Querier, error) {
-			return newProvider(ctx, settings, cfg, telemetryStore, prometheus, cache, flagger)
+			return newProvider(ctx, settings, cfg, telemetryStore, prometheus, promV2, cache, flagger)
 		},
 	)
 }
@@ -43,6 +45,7 @@ func newProvider(
 	cfg querier.Config,
 	telemetryStore telemetrystore.TelemetryStore,
 	prometheus prometheus.Prometheus,
+	promV2 *clickhouseprometheusv2.Provider,
 	cache cache.Cache,
 	flagger flagger.Flagger,
 ) (querier.Querier, error) {
@@ -180,6 +183,7 @@ func newProvider(
 		telemetryStore,
 		telemetryMetadataStore,
 		prometheus,
+		promV2,
 		traceStmtBuilder,
 		logStmtBuilder,
 		auditStmtBuilder,

pkg/query-service/app/http_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/modules/thirdpartyapi"
+	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/queryparser"
 
 	"log/slog"
@@ -1092,6 +1093,8 @@ func (aH *APIHandler) queryDashboardVarsV2(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	querybuilder.LogIfStatementIsNotValid(r.Context(), aH.logger, query)
+
 	dashboardVars, err := aH.reader.QueryDashboardVars(r.Context(), query)
 	if err != nil {
 		RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)

pkg/query-service/rules/managertestfactory.go

@@ -105,7 +105,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
 	}
 
 	// Create querier with test values
-	providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger)
+	providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, nil, cache, flagger)
 	mockQuerier, err := providerFactory.New(context.Background(), providerSettings, querier.Config{})
 	require.NoError(t, err)
 

pkg/query-service/rules/setups_test.go

@@ -47,6 +47,7 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
+		nil, // promV2
 		nil, // traceStmtBuilder
 		nil, // logStmtBuilder
 		nil, // auditStmtBuilder
@@ -100,6 +101,7 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
 		telemetryStore,
 		metadataStore,
 		nil,            // prometheus
+		nil,            // promV2
 		nil,            // traceStmtBuilder
 		logStmtBuilder, // logStmtBuilder
 		nil,            // auditStmtBuilder
@@ -149,6 +151,7 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
 		telemetryStore,
 		metadataStore,
 		nil,              // prometheus
+		nil,              // promV2
 		traceStmtBuilder, // traceStmtBuilder
 		nil,              // logStmtBuilder
 		nil,              // auditStmtBuilder

pkg/querybuilder/agg_rewrite.go

@@ -96,9 +96,9 @@ func (r *aggExprRewriter) Rewrite(
 	}
 
 	if visitor.isRate {
-		return fmt.Sprintf("%s/%d", sel.SelectItems[0].String(), rateInterval), visitor.chArgs, nil
+		return fmt.Sprintf("%s/%d", chparser.Format(sel.SelectItems[0]), rateInterval), visitor.chArgs, nil
 	}
-	return sel.SelectItems[0].String(), visitor.chArgs, nil
+	return chparser.Format(sel.SelectItems[0]), visitor.chArgs, nil
 }
 
 // RewriteMulti rewrites a slice of expressions.
@@ -207,7 +207,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
 	// Handle *If functions with predicate + values
 	if aggFunc.FuncCombinator {
 		// Map the predicate (last argument)
-		origPred := args[len(args)-1].String()
+		origPred := chparser.Format(args[len(args)-1])
 		whereClause, err := PrepareWhereClause(
 			origPred,
 			FilterExprVisitorOpts{
@@ -242,7 +242,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
 
 		// Map each value column argument
 		for i := 0; i < len(args)-1; i++ {
-			origVal := args[i].String()
+			origVal := chparser.Format(args[i])
 			fieldKey := telemetrytypes.GetFieldKeyFromKeyText(origVal)
 			expr, err := v.fieldMapper.ColumnExpressionFor(v.ctx, v.orgID, v.startNs, v.endNs, &fieldKey, dataType, v.fieldKeys)
 			if err != nil {
@@ -259,7 +259,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
 	} else {
 		// Non-If functions: map every argument as a column/value
 		for i, arg := range args {
-			orig := arg.String()
+			orig := chparser.Format(arg)
 			fieldKey := telemetrytypes.GetFieldKeyFromKeyText(orig)
 			expr, err := v.fieldMapper.ColumnExpressionFor(v.ctx, v.orgID, v.startNs, v.endNs, &fieldKey, dataType, v.fieldKeys)
 			if err != nil {

pkg/querybuilder/clickhouse_sql.go

@@ -0,0 +1,76 @@
+package querybuilder
+
+import (
+	"context"
+	"log/slog"
+	"strings"
+
+	chparser "github.com/AfterShip/clickhouse-sql-parser/parser"
+	"github.com/SigNoz/signoz/pkg/errors"
+)
+
+// internalDatabases hold server metadata, credentials and grants rather than telemetry.
+var internalDatabases = map[string]struct{}{
+	"system":             {},
+	"information_schema": {},
+}
+
+// The parser's grammar has gaps against SQL that ClickHouse itself accepts. See TestErrIfStatementIsNotValid_ShouldPassButFails.
+func ErrIfStatementIsNotValid(query string) (err error) {
+	defer func() {
+		// The parser has a history of panicking on malformed input rather than returning an error.
+		if recovered := recover(); recovered != nil {
+			err = errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid ClickHouse SQL (recovered): %v", recovered)
+		}
+	}()
+
+	stmts, parseErr := chparser.NewParser(query).ParseStmts()
+	if parseErr != nil {
+		// Wrapped rather than formatted in, so that callers can recover the parser's *ParseError and read the position off it.
+		return errors.WrapInvalidInputf(parseErr, errors.CodeInvalidInput, "invalid ClickHouse SQL: %s", parseErr.Error())
+	}
+
+	if len(stmts) != 1 {
+		return errors.NewInvalidInputf(errors.CodeInvalidInput, "ClickHouse SQL must contain exactly one statement, found %d statements", len(stmts))
+	}
+
+	selectQuery, ok := stmts[0].(*chparser.SelectQuery)
+	if !ok {
+		return errors.NewInvalidInputf(errors.CodeInvalidInput, "only SELECT statements are allowed in ClickHouse SQL queries")
+	}
+
+	visitor := &chparser.DefaultASTVisitor{Visit: func(node chparser.Expr) error {
+		switch expr := node.(type) {
+		case *chparser.TableFunctionExpr:
+			// Source table functions remain usable in ClickHouse read-only mode.
+			return errors.NewInvalidInputf(errors.CodeInvalidInput, "ClickHouse table functions are not allowed in SQL queries: %s", chparser.Format(expr.Name))
+
+		case *chparser.TableIdentifier:
+			// Reading these is unaffected by ClickHouse read-only mode.
+			if expr.Database == nil {
+				return nil
+			}
+
+			if _, ok := internalDatabases[strings.ToLower(expr.Database.Name)]; ok {
+				return errors.NewInvalidInputf(errors.CodeInvalidInput, "the ClickHouse %s database is not allowed in SQL queries", expr.Database.Name)
+			}
+
+		case *chparser.SettingExpr:
+			// A query-level setting takes precedence over the context setting.
+			if strings.EqualFold(expr.Name.Name, "readonly") {
+				return errors.NewInvalidInputf(errors.CodeInvalidInput, "the ClickHouse readonly setting cannot be overridden")
+			}
+		}
+
+		return nil
+	}}
+
+	return selectQuery.Accept(visitor)
+}
+
+// TODO(@therealpandey): remove this and move to ErrIfStatementIsNotValid.
+func LogIfStatementIsNotValid(ctx context.Context, logger *slog.Logger, query string) {
+	if err := ErrIfStatementIsNotValid(query); err != nil {
+		logger.WarnContext(ctx, "clickhouse sql is not valid", errors.Attr(err), slog.String("query", query))
+	}
+}

pkg/querybuilder/clickhouse_sql_test.go

@@ -0,0 +1,154 @@
+package querybuilder
+
+import (
+	"testing"
+
+	chparser "github.com/AfterShip/clickhouse-sql-parser/parser"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestErrIfStatementIsNotValid_Pass(t *testing.T) {
+	testCases := []struct {
+		name  string
+		query string
+	}{
+		// Shapes a telemetry read is allowed to take.
+		{"Select", "SELECT region AS r, zone FROM metrics WHERE metric_name = 'cpu' GROUP BY region, zone"},
+		{"TrailingSemicolon", "SELECT count() FROM signoz_logs.distributed_logs_v2;"},
+		{"CommonTableExpression", "WITH t AS (SELECT fingerprint FROM signoz_metrics.time_series_v4) SELECT * FROM t"},
+		{"Join", "SELECT * FROM t1 LEFT JOIN t2 ON t1.a = t2.b"},
+		{"GlobalIn", "SELECT a FROM t WHERE a GLOBAL IN (SELECT b FROM t2)"},
+		{"Union", "SELECT * FROM t UNION ALL SELECT * FROM t2"},
+		{"Intersect", "SELECT * FROM t INTERSECT SELECT * FROM t2"},
+		{"WindowFunction", "SELECT sum(v) OVER (PARTITION BY a ORDER BY t) FROM t"},
+		{"UnrelatedSetting", "SELECT * FROM t SETTINGS max_threads = 4"},
+		{"TerminatedBlockComment", "SELECT /* keep me */ count() FROM t"},
+		{"BlockCommentMarkerInsideStringLiteral", "SELECT count() FROM t WHERE body = '/* not a comment'"},
+		// The parser used to loop forever on this; it now reads the comment to the end of
+		// the input, so this doubles as a canary for that regression.
+		{"TrailingUnterminatedBlockComment", "SELECT count() FROM t /* unterminated"},
+		// The rule keys on the database, not on the table name.
+		{"TableNamedSystemInTelemetryDatabase", "SELECT * FROM signoz_logs.system"},
+		{"SignedLiteralAfterClosingParen", "SELECT (toUnixTimestamp(now()) - 3600)*1000000000"},
+		// order by interval
+		{"OrderByInterval", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval ORDER BY interval"},
+		{"OrderByIntervalAndDirection", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS `interval` ORDER BY `interval` ASC"},
+		{"TrimFunction", "SELECT trimBoth('/api/endpoint/', '/');"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			err := ErrIfStatementIsNotValid(testCase.query)
+			assert.NoError(t, err)
+		})
+	}
+}
+
+func TestErrIfStatementIsNotValid_Fail(t *testing.T) {
+	testCases := []struct {
+		name  string
+		query string
+	}{
+		// Not a single statement, or not a statement at all.
+		{"Empty", ""},
+		{"UnterminatedBlockCommentOnly", "/* x"},
+		{"Unparseable", "SELECT FROM WHERE"},
+		{"MultipleStatements", "SELECT 1; DROP TABLE signoz_logs.logs_v2"},
+		// Parses, but is not a SELECT.
+		{"Drop", "DROP TABLE signoz_logs.logs_v2"},
+		{"Insert", "INSERT INTO signoz_logs.logs_v2 SELECT * FROM signoz_logs.logs_v2"},
+		{"AlterDelete", "ALTER TABLE signoz_logs.logs_v2 DELETE WHERE 1 = 1"},
+		{"CreateTable", "CREATE TABLE evil (a Int) ENGINE = Memory"},
+		{"Grant", "GRANT ALL ON *.* TO admin"},
+		{"Set", "SET readonly = 0"},
+		// These the parser rejects outright rather than classifying.
+		{"ShowGrants", "SHOW GRANTS"},
+		{"IntoOutfile", "SELECT * FROM t INTO OUTFILE '/tmp/x.csv'"},
+		// Table functions, which read through something other than a telemetry table.
+		{"UrlTableFunction", "SELECT * FROM url('http://attacker.example/x', CSV, 'a String')"},
+		{"FileTableFunction", "SELECT * FROM file('/etc/passwd', CSV, 'a String')"},
+		{"ExecutableTableFunction", "SELECT * FROM executable('script.sh', CSV, 'a String')"},
+		{"TableFunctionInJoin", "SELECT * FROM t1 JOIN url('http://x', CSV, 'a String') u ON 1 = 1"},
+		{"TableFunctionInCommonTableExpression", "WITH c AS (SELECT * FROM url('http://x', CSV, 'a String')) SELECT * FROM c"},
+		{"TableFunctionInWhereSubquery", "SELECT * FROM t WHERE a IN (SELECT * FROM file('/etc/passwd', CSV, 'a String'))"},
+		{"TableFunctionInUnion", "SELECT * FROM t UNION ALL SELECT * FROM url('http://x', CSV, 'a String')"},
+		// Internal databases, which hold grants and server metadata rather than telemetry.
+		{"SystemUsers", "SELECT * FROM system.users"},
+		{"SystemUppercase", "SELECT * FROM SYSTEM.USERS"},
+		{"SystemQuoted", "SELECT count() FROM `system`.`tables`"},
+		{"SystemInSubquery", "SELECT * FROM (SELECT name FROM system.parts)"},
+		{"SystemInJoin", "SELECT * FROM signoz_logs.distributed_logs_v2 AS l JOIN system.users AS u ON 1 = 1"},
+		{"SystemInIntersect", "SELECT * FROM t INTERSECT SELECT * FROM system.users"},
+		{"InformationSchema", "SELECT * FROM information_schema.tables"},
+		// A query-level setting takes precedence over the one the caller applies.
+		{"ReadonlySettingOverride", "SELECT * FROM t SETTINGS readonly = 0"},
+		{"ReadonlySettingOverrideAmongOthers", "SELECT * FROM t SETTINGS max_threads = 4, readonly = 0"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			err := ErrIfStatementIsNotValid(testCase.query)
+			assert.Error(t, err)
+		})
+	}
+}
+
+// Queries the parser cannot read. ClickHouse runs all of them.
+func TestErrIfStatementIsNotValid_ShouldPassButFails(t *testing.T) {
+	testCases := []struct {
+		name  string
+		query string
+		// The construct the parser stops after, which is the one it cannot read.
+		expectedStopsAfter string
+		// The same construct written so the parser accepts it.
+		fix string
+	}{
+		{
+			name:               "IntervalAliasInOrderBy",
+			query:              "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, resource_string_service$$name AS `service.name`, attributes_string['http.route'] AS `http.route`, quantile(0.95)(duration_nano) / 1000000000 AS value FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_string_service$$name = 'svc-a' AND resources_string['deployment.environment'] = 'dev' AND attributes_string['http.route'] = '/v1' AND http_method = 'POST' AND timestamp BETWEEN toDateTime(1784601720) AND toDateTime(1784602620) AND ts_bucket_start BETWEEN 1784601720 - 1800 AND 1784602620 GROUP BY `service.name`, `http.route`, interval ORDER BY interval ASC",
+			expectedStopsAfter: "ORDER BY interval ASC",
+			fix:                "SELECT count() AS interval FROM t ORDER BY `interval` ASC",
+		},
+		{
+			name:               "IntervalAliasInOrderByDesc",
+			query:              "SELECT count() AS value, toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, serviceName, resourceTagsMap['deployment.environment'] AS environment, exceptionStacktrace FROM signoz_traces.distributed_signoz_error_index_v2 WHERE exceptionType != 'OSError' AND resourceTagsMap['deployment.environment'] = 'staging' AND timestamp BETWEEN toDateTime(1785186300) AND toDateTime(1785186600) GROUP BY serviceName, interval, environment, exceptionStacktrace ORDER BY interval DESC",
+			expectedStopsAfter: "ORDER BY interval DESC",
+			fix:                "SELECT count() AS interval FROM t ORDER BY `interval` DESC",
+		},
+		{
+			name:               "StandardTrimSyntax",
+			query:              "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 5 MINUTE) AS interval, resources_string['host.name'] as host_name, toFloat64(countIf( lower(trim(BOTH ' ' FROM replaceOne( JSONExtractString(body, 'Action'), 'health_status: ', '' ))) IN ('unhealthy','starting','failing') )) as value FROM signoz_logs.distributed_logs_v2 WHERE timestamp BETWEEN 1784602320000000000 AND 1784602620000000000 AND ts_bucket_start BETWEEN 1784602320 - 300 AND 1784602620 AND JSONExtractString(body, 'Type') = 'container' AND JSONExtractString(body, 'Actor', 'Attributes', 'name') IS NOT NULL AND resources_string['host.name'] IS NOT NULL AND resources_string['host.name'] = 'aihub-nightly' GROUP BY interval, host_name ORDER BY interval, host_name",
+			expectedStopsAfter: "trim(BOTH '",
+			fix:                "SELECT trimBoth(replaceOne( JSONExtractString(body, 'Action'), 'health_status: ', '' ), ' ')",
+		},
+		{
+			name:               "SignedLiteralAfterClosingParen",
+			query:              "SELECT now() AS ts, toFloat64(count()) AS value FROM ( SELECT attributes_string['TableName'] AS T, attributes_string['MissingId'] AS M, max(fromUnixTimestamp64Nano(timestamp)) AS last_seen, dateDiff('minute', min(fromUnixTimestamp64Nano(timestamp)), max(fromUnixTimestamp64Nano(timestamp))) AS age_min FROM signoz_logs.distributed_logs_v2 WHERE body='missing_map_record' AND timestamp >= (toUnixTimestamp(now())-3600)*1000000000 GROUP BY T, M ) WHERE age_min >= 20 AND last_seen >= now() - toIntervalMinute(8)",
+			expectedStopsAfter: "(toUnixTimestamp(now())",
+			fix:                "SELECT (toUnixTimestamp(now()) - 3600) * 1000000000",
+		},
+		{
+			name:               "SignedLiteralAfterClosingParenMinimal",
+			query:              "SELECT (1)-1",
+			expectedStopsAfter: "(1)",
+			fix:                "SELECT (1) - 1",
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			err := ErrIfStatementIsNotValid(testCase.query)
+
+			var parseErr *chparser.ParseError
+			require.ErrorAs(t, err, &parseErr, "expected a parser failure rather than a rule violation")
+
+			// The parser reports the offset it stopped at, which sits just past the construct
+			// it choked on, so the text leading up to it is what needs looking at.
+			consumed := testCase.query[:parseErr.Pos]
+			assert.Equal(t, testCase.expectedStopsAfter, consumed[max(0, len(consumed)-len(testCase.expectedStopsAfter)):])
+
+			assert.NoError(t, ErrIfStatementIsNotValid(testCase.fix))
+		})
+	}
+}

pkg/querybuilder/init.go

@@ -1 +0,0 @@
-package querybuilder

pkg/queryparser/queryfilterextractor/clickhouse.go

@@ -316,17 +316,17 @@ func (e *ClickHouseFilterExtractor) extractColumnStrByExpr(expr clickhouse.Expr)
 	// FunctionExpr is a function call like "toDate(timestamp)"
 	case *clickhouse.FunctionExpr:
 		// For function expressions, return the complete function call string
-		return ex.String()
+		return clickhouse.Format(ex)
 	// ColumnExpr is a column expression like "m.region", "toDate(timestamp)"
 	case *clickhouse.ColumnExpr:
 		// ColumnExpr wraps another expression - extract the underlying expression
 		if ex.Expr != nil {
 			return e.extractColumnStrByExpr(ex.Expr)
 		}
-		return ex.String()
+		return clickhouse.Format(ex)
 	default:
 		// For other expression types, return the string representation
-		return expr.String()
+		return clickhouse.Format(expr)
 	}
 }
 
@@ -524,7 +524,7 @@ func (e *ClickHouseFilterExtractor) extractFullExpression(expr clickhouse.Expr)
 	if expr == nil {
 		return ""
 	}
-	return expr.String()
+	return clickhouse.Format(expr)
 }
 
 // isSimpleColumnReference checks if an expression is just a simple column reference
@@ -657,7 +657,7 @@ func (e *ClickHouseFilterExtractor) extractCTEName(cte *clickhouse.CTEStmt) stri
 	case *clickhouse.Ident:
 		return name.Name
 	default:
-		return cte.Expr.String()
+		return clickhouse.Format(cte.Expr)
 	}
 }
 

pkg/signoz/provider.go

@@ -45,6 +45,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/pprof/nooppprof"
 	"github.com/SigNoz/signoz/pkg/prometheus"
 	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/querier/signozquerier"
 	"github.com/SigNoz/signoz/pkg/sharder"
@@ -244,6 +245,7 @@ func NewTelemetryStoreProviderFactories() factory.NamedMap[factory.ProviderFacto
 func NewPrometheusProviderFactories(telemetryStore telemetrystore.TelemetryStore) factory.NamedMap[factory.ProviderFactory[prometheus.Prometheus, prometheus.Config]] {
 	return factory.MustNewNamedMap(
 		clickhouseprometheus.NewFactory(telemetryStore),
+		clickhouseprometheusv2.NewFactory(telemetryStore),
 	)
 }
 
@@ -285,9 +287,9 @@ func NewStatsReporterProviderFactories(aggregator statsreporter.Aggregator, orgG
 	)
 }
 
-func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, cache cache.Cache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
+func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, promV2 *clickhouseprometheusv2.Provider, cache cache.Cache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
 	return factory.MustNewNamedMap(
-		signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger),
+		signozquerier.NewFactory(telemetryStore, prometheus, promV2, cache, flagger),
 	)
 }
 

pkg/signoz/signoz.go

@@ -40,6 +40,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/modules/tag/impltag"
 	"github.com/SigNoz/signoz/pkg/modules/user/impluser"
 	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/queryparser"
 	"github.com/SigNoz/signoz/pkg/ruler"
@@ -242,6 +243,11 @@ func New(
 
 	retentionGetter := implretention.NewGetter(implretention.NewStore(sqlstore))
 
+	// promV2 is the clickhousev2 provider handed to the querier for shadow
+	// comparison and pinned serving (declared before the serving provider,
+	// whose variable shadows the package name below).
+	var promV2 *clickhouseprometheusv2.Provider
+
 	// Initialize prometheus from the available prometheus provider factories
 	prometheus, err := factory.NewProviderFromNamedMap(
 		ctx,
@@ -254,12 +260,29 @@ func New(
 		return nil, err
 	}
 
+	// With the default provider, also stand up the clickhousev2 provider for
+	// the querier: PromQL queries shadow-compare against it behind the
+	// use_prometheus_clickhouse_v2 flag (see pkg/querier/promql_shadow.go).
+	// It never serves by default. An explicit
+	// prometheus::provider: clickhousev2 makes v2 the serving provider
+	// outright, so there is nothing to compare against.
+	if config.Prometheus.Provider() == "clickhouse" {
+		v2Config := config.Prometheus
+		// The v2 engine only evaluates shadow and pinned queries; disable its
+		// active query tracker so two trackers never share a file.
+		v2Config.ActiveQueryTrackerConfig.Enabled = false
+		promV2, err = clickhouseprometheusv2.New(ctx, providerSettings, v2Config, telemetrystore)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Initialize querier from the available querier provider factories
 	querier, err := factory.NewProviderFromNamedMap(
 		ctx,
 		providerSettings,
 		config.Querier,
-		NewQuerierProviderFactories(telemetrystore, prometheus, cache, flagger),
+		NewQuerierProviderFactories(telemetrystore, prometheus, promV2, cache, flagger),
 		config.Querier.Provider(),
 	)
 	if err != nil {

pkg/telemetrymetadata/stmt_parse.go

@@ -27,7 +27,7 @@ func (v *TelemetryFieldVisitor) VisitColumnDef(expr *parser.ColumnDef) error {
 	}
 
 	// Parse column name to extract context and data type
-	columnName := expr.Name.String()
+	columnName := parser.Format(expr.Name)
 
 	// Remove backticks if present
 	columnName = strings.TrimPrefix(columnName, "`")
@@ -75,7 +75,7 @@ func (v *TelemetryFieldVisitor) VisitColumnDef(expr *parser.ColumnDef) error {
 	// Extract field name from the DEFAULT expression
 	// The DEFAULT expression should be something like: resources_string['k8s.cluster.name']
 	// We need to extract the key inside the square brackets
-	defaultExprStr := expr.DefaultExpr.String()
+	defaultExprStr := parser.Format(expr.DefaultExpr)
 
 	// Look for the pattern: map['key']
 	startIdx := strings.Index(defaultExprStr, "['")

pkg/types/querybuildertypes/querybuildertypesv5/req.go

@@ -370,6 +370,14 @@ type QueryRangeRequest struct {
 	// NoCache is a flag to disable caching for the request.
 	NoCache bool `json:"noCache,omitempty"`
 
+	// PromQLProvider serves this request's PromQL queries via the named
+	// prometheus provider ("clickhousev2") instead of the default — the same
+	// data read through a different implementation. It is set from the
+	// X-SigNoz-PromQL-Provider header by the API handler, never from the
+	// body: a rollout-scoped comparison hook for integration tests and
+	// support should not become part of the public request schema.
+	PromQLProvider string `json:"-"`
+
 	FormatOptions *FormatOptions `json:"formatOptions,omitempty"`
 }
 

pkg/variables/clickhouse/processor.go

@@ -69,7 +69,7 @@ func (qp *QueryProcessor) ProcessQuery(query string, transformer FilterTransform
 	// Reconstruct the query
 	var resultBuilder strings.Builder
 	for _, stmt := range stmts {
-		resultBuilder.WriteString(stmt.String())
+		resultBuilder.WriteString(parser.Format(stmt))
 		resultBuilder.WriteString(";")
 	}
 

scripts/promqltestcorpus/.gitignore

@@ -0,0 +1,2 @@
+# go build artifact (go run . is the supported entry)
+/promqltestcorpus

scripts/promqltestcorpus/corpus.go

@@ -0,0 +1,660 @@
+// SigNoz corpus policy: which upstream cases are representable through the
+// API, the grid variants that steer coarse-step code paths, the API's value
+// rounding, and the frozen JSON model. Nothing in this file mirrors
+// upstream code; it encodes what our conformance harness needs.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/value"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/promql/parser"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+	"github.com/prometheus/prometheus/util/almost"
+	"github.com/prometheus/prometheus/util/teststorage"
+)
+
+// seriesDescParser is the slice of parser.Parser the loader needs.
+type seriesDescParser interface {
+	ParseSeriesDesc(input string) (labels.Labels, []parser.SequenceValue, error)
+}
+
+// Calendar and wall-clock functions are not invariant under time
+// translation, and the Python suite shifts every case to recent timestamps
+// (epoch-0 samples would sit 55 years past ClickHouse TTLs). Everything else
+// PromQL computes depends only on time differences.
+var timeDependentFuncs = map[string]bool{
+	"time": true, "timestamp": true, "month": true, "year": true,
+	"minute": true, "hour": true, "day_of_month": true, "day_of_week": true,
+	"day_of_year": true, "days_in_month": true,
+}
+
+const (
+	lookbackMs    = 300_000
+	instantStepMs = 1_000
+	maxSamples    = 50_000_000
+)
+
+type corpusSeries struct {
+	Labels  map[string]string `json:"labels"`
+	Samples [][2]any          `json:"samples"` // [offset_ms, value]
+}
+
+type corpusDataset struct {
+	ID     int            `json:"id"`
+	Source string         `json:"source"`
+	Series []corpusSeries `json:"series"`
+}
+
+type corpusPoint = [2]any // [offset_ms, value]
+
+type corpusResult struct {
+	Labels map[string]string `json:"labels"`
+	Points []corpusPoint     `json:"points"`
+}
+
+type corpusCase struct {
+	Dataset  int            `json:"dataset"`
+	Source   string         `json:"source"`
+	Variant  string         `json:"variant"`
+	Expr     string         `json:"expr"`
+	StartMs  int64          `json:"start_ms"`
+	EndMs    int64          `json:"end_ms"`
+	StepMs   int64          `json:"step_ms"`
+	Instant  bool           `json:"instant"`
+	Expected []corpusResult `json:"expected"`
+}
+
+type corpus struct {
+	Meta struct {
+		PrometheusVersion string `json:"prometheus_version"`
+		LookbackMs        int64  `json:"lookback_ms"`
+		InstantStepMs     int64  `json:"instant_step_ms"`
+		Note              string `json:"note"`
+	} `json:"meta"`
+	Datasets []corpusDataset `json:"datasets"`
+	Cases    []corpusCase    `json:"cases"`
+}
+
+func generate(files []string, engine *promql.Engine, seriesParser parser.Parser, exprParser parser.Parser, promVersion string) (*corpus, map[string]int, error) {
+	var c corpus
+	c.Meta.PrometheusVersion = promVersion
+	c.Meta.LookbackMs = lookbackMs
+	c.Meta.InstantStepMs = instantStepMs
+	c.Meta.Note = "expected values carry the API's 3-significant-decimal rounding (querybuildertypesv5 sanitizeValue); instant evals are encoded as start==end range queries"
+
+	skips := map[string]int{}
+	datasetIDs := map[string]int{}
+
+	for _, file := range files {
+		base := filepath.Base(file)
+		if base == "native_histograms.test" || base == "type_and_unit.test" {
+			// native histograms: the samples pipeline under test stores
+			// floats; type_and_unit: experimental __type__/__unit__ metadata
+			// labels our store does not materialize.
+			skips["file:"+strings.TrimSuffix(base, ".test")]++
+			continue
+		}
+		raw, err := os.ReadFile(file)
+		if err != nil {
+			return nil, nil, err
+		}
+		cmds := parseScript(string(raw))
+
+		segment := 0
+		var loads []command
+		segmentBad := "" // non-empty: reason the segment cannot be represented
+		for _, cmd := range cmds {
+			switch cmd.kind {
+			case "clear":
+				segment++
+				loads = nil
+				segmentBad = ""
+			case "skip":
+				skips["command:"+cmd.head]++
+			case "load":
+				if reason := checkLoad(seriesParser, cmd); reason != "" {
+					segmentBad = reason
+					skips["load:"+reason]++
+					continue
+				}
+				loads = append(loads, cmd)
+			case "eval":
+				if segmentBad != "" {
+					skips["segment:"+segmentBad]++
+					continue
+				}
+				if len(loads) == 0 {
+					skips["eval:no-data"]++
+					continue
+				}
+				ccs, reason, err := buildCases(engine, seriesParser, exprParser, cmd, loads, base, skips)
+				if err != nil {
+					return nil, nil, err
+				}
+				if reason != "" {
+					skips["eval:"+reason]++
+					continue
+				}
+				key := fmt.Sprintf("%s#%d#%d", base, segment, len(loads))
+				id, ok := datasetIDs[key]
+				if !ok {
+					ds, reason, err := dumpDataset(seriesParser, loads)
+					if err != nil {
+						return nil, nil, err
+					}
+					if reason != "" {
+						skips["dataset:"+reason]++
+						continue
+					}
+					id = len(c.Datasets)
+					datasetIDs[key] = id
+					ds.ID = id
+					ds.Source = key
+					c.Datasets = append(c.Datasets, *ds)
+				}
+				for _, cc := range ccs {
+					cc.Dataset = id
+					cc.Source = fmt.Sprintf("%s:%d", base, cmd.line)
+					c.Cases = append(c.Cases, cc)
+				}
+			}
+		}
+	}
+	if len(c.Cases) == 0 {
+		return nil, nil, fmt.Errorf("no corpus cases produced")
+	}
+	return &c, skips, nil
+}
+
+func writeCorpus(out string, c *corpus) error {
+	buf, err := json.MarshalIndent(c, "", " ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(out, buf, 0o644)
+}
+
+// checkLoad validates a load block is representable: parsable series
+// notation, float samples only ("load_with_nhcb" and histogram literals are
+// out of scope — the samples pipeline under test stores float samples).
+func checkLoad(p parser.Parser, cmd command) string {
+	fields := strings.Fields(cmd.head)
+	if len(fields) != 2 || fields[0] != "load" {
+		return "unsupported-load-variant"
+	}
+	if _, err := model.ParseDuration(fields[1]); err != nil {
+		return "bad-interval"
+	}
+	for _, line := range cmd.body {
+		metric, vals, err := p.ParseSeriesDesc(line)
+		if err != nil {
+			return "unparsable-series"
+		}
+		if metric.Get(model.MetricNameLabel) == "" {
+			return "unnamed-series"
+		}
+		for _, v := range vals {
+			if v.Histogram != nil {
+				return "histogram-samples"
+			}
+		}
+	}
+	return ""
+}
+
+// durationExprUsesRange reports whether a duration-expression tree contains
+// range(). Instant evals are encoded as one-step range queries (the API
+// rejects start == end), which changes what range() evaluates to, so they
+// cannot carry it; range evals keep it — each variant's oracle is computed
+// on the exact window it requests.
+func durationExprUsesRange(e parser.Expr) bool {
+	d, ok := e.(*parser.DurationExpr)
+	if !ok || d == nil {
+		return false
+	}
+	if d.Op == parser.RANGE {
+		return true
+	}
+	return durationExprUsesRange(d.LHS) || durationExprUsesRange(d.RHS)
+}
+
+// buildCases parses one eval header, filters unservable expressions, and
+// emits the base case plus grid variants — each with expectations computed by
+// the reference engine over the loads. The variants exist because upstream's
+// own grids are fine-stepped: without them the coarse-step code paths (the
+// window-sliver filter, the disjoint over_time form, the lifted instant/last
+// gates) would pass through this corpus untouched. A variant is just another
+// grid over the same data and expression; the engine is the oracle either way.
+func buildCases(engine *promql.Engine, seriesParser parser.Parser, exprParser parser.Parser, cmd command, loads []command, sourceFile string, skips map[string]int) ([]corpusCase, string, error) {
+	for _, line := range cmd.body {
+		if patExpect.MatchString(line) && strings.HasPrefix(line, "expect fail") {
+			return nil, "expect-fail", nil
+		}
+	}
+
+	base := corpusCase{Variant: "base"}
+	if m := patEvalInstant.FindStringSubmatch(cmd.head); m != nil {
+		at, err := parseTestDuration(m[2])
+		if err != nil {
+			return nil, "bad-duration", nil
+		}
+		base.Instant = true
+		base.StartMs, base.EndMs, base.StepMs = at, at, instantStepMs
+		base.Expr = m[3]
+	} else if m := patEvalRange.FindStringSubmatch(cmd.head); m != nil {
+		from, err1 := parseTestDuration(m[2])
+		to, err2 := parseTestDuration(m[3])
+		step, err3 := parseTestDuration(m[4])
+		if err1 != nil || err2 != nil || err3 != nil {
+			return nil, "bad-duration", nil
+		}
+		if step <= 0 || to < from {
+			return nil, "bad-grid", nil
+		}
+		base.StartMs, base.EndMs, base.StepMs = from, to, step
+		base.Expr = m[5]
+	} else {
+		return nil, "unrecognized", nil
+	}
+
+	expr, err := exprParser.ParseExpr(base.Expr)
+	if err != nil {
+		return nil, "needs-experimental-parser", nil
+	}
+	if vt := expr.Type(); vt != parser.ValueTypeVector && vt != parser.ValueTypeScalar {
+		return nil, "non-instant-type", nil
+	}
+	unservable := ""
+	hasSelector := false
+	hasSubquery := false
+	var maxRangeMs int64
+	parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error {
+		switch n := node.(type) {
+		case *parser.Call:
+			if timeDependentFuncs[n.Func.Name] {
+				unservable = "time-dependent"
+			}
+		case *parser.VectorSelector:
+			hasSelector = true
+			if n.Timestamp != nil || n.StartOrEnd != 0 {
+				unservable = "at-modifier"
+			}
+			if n.OriginalOffset < 0 {
+				// The server ships with negative offsets disabled.
+				unservable = "negative-offset"
+			}
+			if base.Instant && durationExprUsesRange(n.OriginalOffsetExpr) {
+				unservable = "range-duration-in-instant"
+			}
+			for _, m := range n.LabelMatchers {
+				if m.Name == "__type__" || m.Name == "__unit__" {
+					unservable = "type-unit-metadata"
+				}
+			}
+		case *parser.MatrixSelector:
+			if r := n.Range.Milliseconds(); r > maxRangeMs {
+				maxRangeMs = r
+			}
+			if base.Instant && durationExprUsesRange(n.RangeExpr) {
+				unservable = "range-duration-in-instant"
+			}
+		case *parser.SubqueryExpr:
+			hasSubquery = true
+			if n.Timestamp != nil || n.StartOrEnd != 0 {
+				unservable = "at-modifier"
+			}
+			if n.OriginalOffset < 0 {
+				unservable = "negative-offset"
+			}
+			if base.Instant && (durationExprUsesRange(n.RangeExpr) || durationExprUsesRange(n.StepExpr) || durationExprUsesRange(n.OriginalOffsetExpr)) {
+				unservable = "range-duration-in-instant"
+			}
+		}
+		return nil
+	})
+	if unservable != "" {
+		return nil, unservable, nil
+	}
+
+	stor, err := loadSeriesStorage(seriesParser, loads)
+	if err != nil {
+		return nil, "", err
+	}
+	defer func() { _ = stor.Close() }()
+
+	expected, reason := computeExpected(engine, stor, base.Expr, base.StartMs, base.EndMs, base.StepMs)
+	if reason != "" {
+		return nil, reason, nil
+	}
+	if err := crossCheckUpstream(engine, seriesParser, stor, cmd, base, sourceFile, skips); err != nil {
+		return nil, "", err
+	}
+	base.Expected = expected
+	out := []corpusCase{base}
+
+	// Grid variants. Subquery expressions keep their own inner grids; varying
+	// the outer grid there multiplies cases without steering the code paths
+	// the variants exist for, so they emit only the base.
+	if hasSubquery || !hasSelector {
+		return out, "", nil
+	}
+	type variant struct {
+		name                   string
+		startMs, endMs, stepMs int64
+	}
+	var variants []variant
+	if base.Instant {
+		// A coarse multi-point grid ending at the instant: step above the
+		// 5m lookback drives the lifted instant gate and the sliver filter.
+		const coarse = 600_000
+		variants = append(variants, variant{"instant-coarse", base.EndMs - 2*coarse, base.EndMs, coarse})
+	} else {
+		span := base.EndMs - base.StartMs
+		if maxRangeMs > 0 {
+			// Step wider than every window in the expression: the sliver
+			// filter and the disjoint over_time form become active.
+			if coarse := 2 * maxRangeMs; span >= coarse {
+				variants = append(variants, variant{"coarse-step", base.StartMs, base.EndMs, coarse})
+			}
+			// Whole-bucket tiling (range == 2 steps) drives the windowed
+			// over_time slide with W = 2.
+			if tiled := maxRangeMs / 2; tiled >= 1000 && maxRangeMs%2000 == 0 && tiled != base.StepMs && span >= tiled {
+				variants = append(variants, variant{"tiled", base.StartMs, base.EndMs, tiled})
+			}
+		}
+		// A start off every natural alignment shifts which samples each
+		// window sees; an end short of the lattice exercises the
+		// last-grid-point handling.
+		if span > 17_000 {
+			variants = append(variants, variant{"unaligned-start", base.StartMs + 17_000, base.EndMs, base.StepMs})
+		}
+		if lastIdx := span / base.StepMs; lastIdx >= 2 {
+			offEnd := base.StartMs + lastIdx*base.StepMs - base.StepMs/3
+			variants = append(variants, variant{"off-lattice-end", base.StartMs, offEnd, base.StepMs})
+		}
+	}
+	for _, v := range variants {
+		if v.stepMs <= 0 || v.endMs <= v.startMs || v.stepMs%1000 != 0 {
+			continue
+		}
+		expected, reason := computeExpected(engine, stor, base.Expr, v.startMs, v.endMs, v.stepMs)
+		if reason != "" {
+			continue
+		}
+		out = append(out, corpusCase{
+			Variant: v.name, Expr: base.Expr,
+			StartMs: v.startMs, EndMs: v.endMs, StepMs: v.stepMs,
+			Expected: expected,
+		})
+	}
+	return out, "", nil
+}
+
+// computeExpected evaluates the expression on one grid with the reference
+// engine and serializes the result with the API's value rounding.
+func computeExpected(engine *promql.Engine, stor *teststorage.TestStorage, expr string, startMs, endMs, stepMs int64) ([]corpusResult, string) {
+	qry, err := engine.NewRangeQuery(context.Background(), stor, nil, expr,
+		time.UnixMilli(startMs), time.UnixMilli(endMs), time.Duration(stepMs)*time.Millisecond)
+	if err != nil {
+		return nil, "engine-parse"
+	}
+	defer qry.Close()
+	res := qry.Exec(context.Background())
+	if res.Err != nil {
+		// Covers upstream's expected-error cases and engine features the
+		// range form cannot evaluate; a case we cannot compute is a case we
+		// cannot assert.
+		return nil, "engine-error"
+	}
+	matrix, ok := res.Value.(promql.Matrix)
+	if !ok {
+		return nil, "non-matrix-result"
+	}
+	expected := []corpusResult{}
+	for _, s := range matrix {
+		if len(s.Histograms) > 0 {
+			return nil, "histogram-result"
+		}
+		r := corpusResult{Labels: s.Metric.Map(), Points: []corpusPoint{}}
+		for _, p := range s.Floats {
+			r.Points = append(r.Points, corpusPoint{p.T, encodeFloat(roundToNonZeroDecimals(p.F, 3))})
+		}
+		expected = append(expected, r)
+	}
+	return expected, ""
+}
+
+// dumpDataset walks the loaded storage and serializes every float sample.
+func dumpDataset(seriesParser parser.Parser, loads []command) (*corpusDataset, string, error) {
+	stor, err := loadSeriesStorage(seriesParser, loads)
+	if err != nil {
+		return nil, "", err
+	}
+	defer func() { _ = stor.Close() }()
+
+	q, err := stor.Querier(math.MinInt64/2, math.MaxInt64/2)
+	if err != nil {
+		return nil, "querier", nil
+	}
+	defer q.Close()
+
+	ds := &corpusDataset{}
+	ss := q.Select(context.Background(), true, nil, labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".*"))
+	var it chunkenc.Iterator
+	for ss.Next() {
+		s := ss.At()
+		cs := corpusSeries{Labels: s.Labels().Map(), Samples: [][2]any{}}
+		it = s.Iterator(it)
+		for vt := it.Next(); vt != chunkenc.ValNone; vt = it.Next() {
+			if vt != chunkenc.ValFloat {
+				return nil, "histogram-samples", nil
+			}
+			ts, v := it.At()
+			if value.IsStaleNaN(v) {
+				cs.Samples = append(cs.Samples, [2]any{ts, "stale"})
+				continue
+			}
+			cs.Samples = append(cs.Samples, [2]any{ts, encodeFloat(v)})
+		}
+		ds.Series = append(ds.Series, cs)
+	}
+	if err := ss.Err(); err != nil {
+		return nil, "series-set", nil
+	}
+	return ds, "", nil
+}
+
+// parseTestDuration accepts promqltest's time notation: a Prometheus
+// duration ("5m", "1m30s"), a bare "0", or bare seconds.
+func parseTestDuration(s string) (int64, error) {
+	if d, err := model.ParseDuration(s); err == nil {
+		return int64(time.Duration(d) / time.Millisecond), nil
+	}
+	if n, err := strconv.ParseFloat(s, 64); err == nil {
+		return int64(n * 1000), nil
+	}
+	return 0, fmt.Errorf("unparsable duration %q", s)
+}
+
+func encodeFloat(f float64) any {
+	switch {
+	case math.IsNaN(f):
+		return "NaN"
+	case math.IsInf(f, 1):
+		return "Inf"
+	case math.IsInf(f, -1):
+		return "-Inf"
+	default:
+		return f
+	}
+}
+
+// roundToNonZeroDecimals mirrors querybuildertypesv5's sanitizeValue rounding
+// (pkg/types/querybuildertypes/querybuildertypesv5/resp.go) so the frozen
+// expectations equal what the API emits for the same float.
+func roundToNonZeroDecimals(val float64, n int) float64 {
+	if val == 0 || math.IsNaN(val) || math.IsInf(val, 0) {
+		return val
+	}
+	absVal := math.Abs(val)
+	if absVal >= 1 {
+		multiplier := math.Pow(10, float64(n))
+		rounded := math.Round(val*multiplier) / multiplier
+		if math.IsInf(rounded, 0) {
+			// Mirrors the overflow guard in querybuildertypesv5.
+			return val
+		}
+		if rounded == math.Trunc(rounded) {
+			return rounded
+		}
+		str := strconv.FormatFloat(rounded, 'f', -1, 64)
+		result, _ := strconv.ParseFloat(str, 64)
+		return result
+	}
+	order := math.Floor(math.Log10(absVal))
+	scale := math.Pow(10, -order+float64(n)-1)
+	rounded := math.Round(val*scale) / scale
+	str := strconv.FormatFloat(rounded, 'f', -1, 64)
+	result, _ := strconv.ParseFloat(str, 64)
+	return result
+}
+
+// crossCheckFileAllowlist names files whose written expectations assume
+// engine options we deliberately run differently, with the reason. Every
+// other mismatch between our engine-computed expectations and upstream's
+// hand-written ones aborts generation: the corpus must never contradict
+// the testdata it claims to represent.
+var crossCheckFileAllowlist = map[string]string{
+	"name_label_dropping.test": "expectations written for EnableDelayedNameRemoval; our engine matches the server default (off)",
+}
+
+// crossCheckUpstream validates the transcription chain — load parsing,
+// eval parsing, storage loading — by comparing the reference engine's raw
+// output on the base grid against the expectations upstream wrote under the
+// same eval, with upstream's own tolerance (almost.Equal, 1e-6 relative).
+// The corpus's authority is "what the reference engine computes over
+// upstream's data"; this pins that computation to upstream's own record of
+// it.
+func crossCheckUpstream(engine *promql.Engine, seriesParser parser.Parser, stor *teststorage.TestStorage, cmd command, base corpusCase, sourceFile string, skips map[string]int) error {
+	type expSeries struct {
+		labels labels.Labels
+		points map[int64]float64
+	}
+	var expected []expSeries
+	scalarOnly := false
+	var scalarValue float64
+	for _, line := range cmd.body {
+		if strings.HasPrefix(line, "expect") {
+			// expect fail/warn/info/ordered directives and "expect range
+			// vector"/"expect string" annotations, not series expectations.
+			continue
+		}
+		if f, err := strconv.ParseFloat(line, 64); err == nil && len(cmd.body) == 1 {
+			scalarOnly, scalarValue = true, f
+			break
+		}
+		metric, vals, err := seriesParser.ParseSeriesDesc(line)
+		if err != nil {
+			skips["crosscheck-skip:unparsable-expectation"]++
+			return nil
+		}
+		points := map[int64]float64{}
+		for k, v := range vals {
+			if v.Histogram != nil {
+				skips["crosscheck-skip:histogram-expectation"]++
+				return nil
+			}
+			if v.Omitted {
+				continue
+			}
+			points[base.StartMs+int64(k)*base.StepMs] = v.Value
+		}
+		expected = append(expected, expSeries{labels: metric, points: points})
+	}
+
+	qry, err := engine.NewRangeQuery(context.Background(), stor, nil, base.Expr,
+		time.UnixMilli(base.StartMs), time.UnixMilli(base.EndMs), time.Duration(base.StepMs)*time.Millisecond)
+	if err != nil {
+		return fmt.Errorf("crosscheck parse %q: %w", base.Expr, err)
+	}
+	defer qry.Close()
+	res := qry.Exec(context.Background())
+	if res.Err != nil {
+		return fmt.Errorf("crosscheck eval %q: %w", base.Expr, res.Err)
+	}
+	matrix, ok := res.Value.(promql.Matrix)
+	if !ok {
+		skips["crosscheck-skip:non-matrix"]++
+		return nil
+	}
+
+	mismatch := func(format string, args ...any) error {
+		if reason, ok := crossCheckFileAllowlist[sourceFile]; ok {
+			skips["crosscheck-allowlisted:"+sourceFile]++
+			_ = reason
+			return nil
+		}
+		return fmt.Errorf("%s:%d: corpus contradicts upstream expectation for %q: %s",
+			sourceFile, cmd.line, base.Expr, fmt.Sprintf(format, args...))
+	}
+
+	if scalarOnly {
+		if len(matrix) != 1 || matrix[0].Metric.Len() != 0 {
+			return mismatch("scalar expectation but %d series", len(matrix))
+		}
+		if len(matrix[0].Floats) == 0 || !almost.Equal(matrix[0].Floats[len(matrix[0].Floats)-1].F, scalarValue, defaultEpsilon) {
+			return mismatch("scalar %v != expected %v", matrix[0].Floats, scalarValue)
+		}
+		skips["crosscheck-ok"]++
+		return nil
+	}
+
+	if len(matrix) != len(expected) {
+		return mismatch("engine returned %d series, upstream wrote %d", len(matrix), len(expected))
+	}
+	for _, exp := range expected {
+		var got *promql.Series
+		for i := range matrix {
+			if labels.Equal(matrix[i].Metric, exp.labels) {
+				got = &matrix[i]
+				break
+			}
+		}
+		if got == nil {
+			return mismatch("series %s missing from engine result", exp.labels)
+		}
+		gotPoints := map[int64]float64{}
+		for _, p := range got.Floats {
+			gotPoints[p.T] = p.F
+		}
+		if len(gotPoints) != len(exp.points) {
+			return mismatch("series %s: %d points, upstream wrote %d", exp.labels, len(gotPoints), len(exp.points))
+		}
+		for ts, want := range exp.points {
+			gotV, ok := gotPoints[ts]
+			if !ok {
+				return mismatch("series %s: no point at %d", exp.labels, ts)
+			}
+			if math.IsNaN(want) && math.IsNaN(gotV) {
+				continue
+			}
+			if !almost.Equal(gotV, want, defaultEpsilon) {
+				return mismatch("series %s at %d: engine %v, upstream wrote %v", exp.labels, ts, gotV, want)
+			}
+		}
+	}
+	skips["crosscheck-ok"]++
+	return nil
+}

scripts/promqltestcorpus/go.mod

@@ -0,0 +1,103 @@
+module github.com/SigNoz/signoz/scripts/promqltestcorpus
+
+go 1.25.7
+
+require (
+	github.com/prometheus/common v0.67.5
+	github.com/prometheus/prometheus v0.311.3
+)
+
+require (
+	cloud.google.com/go/auth v0.18.2 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
+	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
+	github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
+	github.com/aws/smithy-go v1.24.2 // indirect
+	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/dennwc/varint v1.0.0 // indirect
+	github.com/edsrzf/mmap-go v1.2.0 // indirect
+	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
+	github.com/golang/snappy v1.0.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
+	github.com/googleapis/gax-go/v2 v2.18.0 // indirect
+	github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect
+	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
+	github.com/oklog/ulid/v2 v2.1.1 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.23.2 // indirect
+	github.com/prometheus/client_golang/exp v0.0.0-20260325093428-d8591d0db856 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/otlptranslator v1.0.0 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/prometheus/sigv4 v0.4.1 // indirect
+	github.com/stretchr/testify v1.11.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
+	go.opentelemetry.io/otel v1.42.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
+	go.opentelemetry.io/otel/trace v1.42.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
+	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
+	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/api v0.272.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c // indirect
+	google.golang.org/grpc v1.79.3 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apimachinery v0.35.3 // indirect
+	k8s.io/client-go v0.35.3 // indirect
+	k8s.io/klog/v2 v2.140.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
+	k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect
+	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+)

scripts/promqltestcorpus/go.sum

@@ -0,0 +1,449 @@
+cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
+cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
+cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
+cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
+cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
+cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/Code-Hex/go-generics-cache v1.5.1 h1:6vhZGc5M7Y/YD8cIUcY8kcuQLB4cHR7U+0KMqAA0KcU=
+github.com/Code-Hex/go-generics-cache v1.5.1/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
+github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
+github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
+github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
+github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
+github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
+github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0=
+github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0 h1:98Miqj16un1WLNyM1RjVDhXYumhqZrQfAeG8i4jPG6o=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0/go.mod h1:T6ndRfdhnXLIY5oKBHjYZDVj706los2zGdpThppquvA=
+github.com/aws/aws-sdk-go-v2/service/ecs v1.74.0 h1:YS5TXaEvzDb+sV+wdQFUtuCAk0GeFR9Ai6HFdxpz6q8=
+github.com/aws/aws-sdk-go-v2/service/ecs v1.74.0/go.mod h1:10kBgdaNJz0FO/+JWDUH+0rtSjkn5yafgavDDmmhFzs=
+github.com/aws/aws-sdk-go-v2/service/elasticache v1.51.12 h1:S066ajzfPRCSW4lsSHOYglne6SNi2CHt1u5omzW1RBg=
+github.com/aws/aws-sdk-go-v2/service/elasticache v1.51.12/go.mod h1:86SE4NcXxbxr8KTG3yOyDmd4HyiFmKl8TexXnhYJ+Bw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
+github.com/aws/aws-sdk-go-v2/service/kafka v1.49.1 h1:BgBatWcQIFqF1l6KGHjv66V0d/ISnWrTwxDx/Jf6EJM=
+github.com/aws/aws-sdk-go-v2/service/kafka v1.49.1/go.mod h1:pMpys+PlrN//vj8j5s0oOAMJjauj81VkHzIZxPVWOro=
+github.com/aws/aws-sdk-go-v2/service/lightsail v1.51.0 h1:cg6PxzoIide2wiEyLfikOFN+XwHafwR8p5+L9U1E8dQ=
+github.com/aws/aws-sdk-go-v2/service/lightsail v1.51.0/go.mod h1:YvX7hjUWecrKX8fBkbEncyddEW85xjNH+u5JHioITOw=
+github.com/aws/aws-sdk-go-v2/service/rds v1.117.0 h1:T1Xe9sYxSUUQOvd1RsFeVk/IXFPdqSiN0atXu/Hy/8A=
+github.com/aws/aws-sdk-go-v2/service/rds v1.117.0/go.mod h1:QbXW4coAMakHQhf1qhE0eVVCen9gwB/Kvn+HHHKhpGY=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
+github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
+github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
+github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w=
+github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
+github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
+github.com/digitalocean/godo v1.178.0 h1:+B4xGOaoFwwwpM7TKhoyGHdmFg5eF9zDB1YfOLvNJ2E=
+github.com/digitalocean/godo v1.178.0/go.mod h1:xQsWpVCCbkDrWisHA72hPzPlnC+4W5w/McZY5ij9uvU=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
+github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
+github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84=
+github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
+github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
+github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
+github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=
+github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A=
+github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds=
+github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
+github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM=
+github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
+github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA=
+github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0=
+github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8=
+github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4=
+github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU=
+github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ=
+github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4=
+github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0=
+github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4=
+github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU=
+github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y=
+github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk=
+github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo=
+github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU=
+github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA=
+github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY=
+github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s=
+github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE=
+github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48=
+github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg=
+github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0=
+github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg=
+github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8=
+github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0=
+github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw=
+github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE=
+github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw=
+github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc=
+github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
+github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
+github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
+github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
+github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
+github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
+github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
+github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
+github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
+github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
+github.com/googleapis/gax-go/v2 v2.18.0 h1:jxP5Uuo3bxm3M6gGtV94P4lliVetoCB4Wk2x8QA86LI=
+github.com/googleapis/gax-go/v2 v2.18.0/go.mod h1:uSzZN4a356eRG985CzJ3WfbFSpqkLTjsnhWGJR6EwrE=
+github.com/gophercloud/gophercloud/v2 v2.11.1 h1:jCs4vLH8sJgRqrPzqVfWgl7uI6JnIIlsgeIRM0uHjxY=
+github.com/gophercloud/gophercloud/v2 v2.11.1/go.mod h1:Rm0YvKQ4QYX2rY9XaDKnjRzSGwlG5ge4h6ABYnmkKQM=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
+github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7EAJ0BHIethd+J6LqxFNw5mSiI2bM=
+github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
+github.com/hashicorp/consul/api v1.32.1 h1:0+osr/3t/aZNAdJX558crU3PEjVrG4x6715aZHRgceE=
+github.com/hashicorp/consul/api v1.32.1/go.mod h1:mXUWLnxftwTmDv4W3lzxYCPD199iNLLUyLfLGFJbtl4=
+github.com/hashicorp/cronexpr v1.1.3 h1:rl5IkxXN2m681EfivTlccqIryzYJSXRGRNa0xeG7NA4=
+github.com/hashicorp/cronexpr v1.1.3/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
+github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
+github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
+github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=
+github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/nomad/api v0.0.0-20260324203407-b27b0c2e019a h1:HGwfgBNl90YBiHdbzZ/+8aMxO1UL9B/yNTAXa8iB8z8=
+github.com/hashicorp/nomad/api v0.0.0-20260324203407-b27b0c2e019a/go.mod h1:KkLNLU0Nyfh5jWsFoF/PsmMbKpRIAoIV4lmQoJWgKCk=
+github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
+github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
+github.com/hetznercloud/hcloud-go/v2 v2.36.0 h1:HlLL/aaVXUulqe+rsjoJmrxKhPi1MflL5O9iq5QEtvo=
+github.com/hetznercloud/hcloud-go/v2 v2.36.0/go.mod h1:MnN/QJEa/RYNQiiVoJjNHPntM7Z1wlYPgJ2HA40/cDE=
+github.com/ionos-cloud/sdk-go/v6 v6.3.6 h1:l/TtKgdQ1wUH3DDe2SfFD78AW+TJWdEbDpQhHkWd6CM=
+github.com/ionos-cloud/sdk-go/v6 v6.3.6/go.mod h1:nUGHP4kZHAZngCVr4v6C8nuargFrtvt7GrzH/hqn7c4=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
+github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo=
+github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
+github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE=
+github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A=
+github.com/knadh/koanf/v2 v2.3.3 h1:jLJC8XCRfLC7n4F+ZKKdBsbq1bfXTpuFhf4L7t94D94=
+github.com/knadh/koanf/v2 v2.3.3/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
+github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00=
+github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/linode/linodego v1.66.0 h1:rK8QJFaV53LWOEJvb/evhTg/dP5ElvtuZmx4iv4RJds=
+github.com/linode/linodego v1.66.0/go.mod h1:12ykGs9qsvxE+OU3SXuW2w+DTruWF35FPlXC7gGk2tU=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
+github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
+github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
+github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.148.0 h1:CiTjQE/Hh5xK2t56ogrDK4nl0+tJPNmASCs4zEYZ/xU=
+github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.148.0/go.mod h1:WUFkzTiOpt7EYyL67gv1GOf3RD8qKWGtin3lY9LYzW4=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.148.0 h1:1TLg6YrS3Au6F7xw3ws2Njbwj13IMqPplvGFi+18fWs=
+github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.148.0/go.mod h1:P8hZEDIQk4REgUWyLhSVRHwTxK6KkifKfg36BmmQ/DI=
+github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.148.0 h1:xgD/kNGp/wWY+bwY599Pc01OamYN17phRiTP934bM5Y=
+github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.148.0/go.mod h1:ZK7wvaefla9lB3bAW0rNKt7IzRPcTRQoOFqr4sZy/XM=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
+github.com/ovh/go-ovh v1.9.0 h1:6K8VoL3BYjVV3In9tPJUdT7qMx9h0GExN9EXx1r2kKE=
+github.com/ovh/go-ovh v1.9.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=
+github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_golang/exp v0.0.0-20260325093428-d8591d0db856 h1:1Y6bmpZb8peQCy1IpctnAhIFuyhrdtMaDnETChhSNns=
+github.com/prometheus/client_golang/exp v0.0.0-20260325093428-d8591d0db856/go.mod h1:Vf0QcmVhGqpjLxZOaWrFSep86vchQtJmbztFaMM4f6Q=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
+github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
+github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos=
+github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/prometheus v0.311.3 h1:3IrVxQv6v5i/ZCGi6OrYeBhtCwaPTn6Z3DYruXoYm3M=
+github.com/prometheus/prometheus v0.311.3/go.mod h1:gjsCxTKtHO1Q8T9333u1s+lUR1OjPyM7ruuGH8RvVyo=
+github.com/prometheus/sigv4 v0.4.1 h1:EIc3j+8NBea9u1iV6O5ZAN8uvPq2xOIUPcqCTivHuXs=
+github.com/prometheus/sigv4 v0.4.1/go.mod h1:eu+ZbRvsc5TPiHwqh77OWuCnWK73IdkETYY46P4dXOU=
+github.com/puzpuzpuz/xsync/v4 v4.4.0 h1:vlSN6/CkEY0pY8KaB0yqo/pCLZvp9nhdbBdjipT4gWo=
+github.com/puzpuzpuz/xsync/v4 v4.4.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.36 h1:ObX9hZmK+VmijreZO/8x9pQ8/P/ToHD/bdSb4Eg4tUo=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.36/go.mod h1:LEsDu4BubxK7/cWhtlQWfuxwL4rf/2UEpxXz1o1EMtM=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stackitcloud/stackit-sdk-go/core v0.23.0 h1:zPrOhf3Xe47rKRs1fg/AqKYUiJJRYjdcv+3qsS50mEs=
+github.com/stackitcloud/stackit-sdk-go/core v0.23.0/go.mod h1:osMglDby4csGZ5sIfhNyYq1bS1TxIdPY88+skE/kkmI=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/vultr/govultr/v3 v3.28.1 h1:KR3LhppYARlBujY7+dcrE7YKL0Yo9qXL+msxykKQrLI=
+github.com/vultr/govultr/v3 v3.28.1/go.mod h1:2zyUw9yADQaGwKnwDesmIOlBNLrm7edsCfWHFJpWKf8=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/collector/component v1.54.0 h1:LvtX0Tzz18n44OrUFVk77N1FNsejfWJqztB28hrmDM8=
+go.opentelemetry.io/collector/component v1.54.0/go.mod h1:yUMBYsySY/sDcXm8kOzEoZxt+JLdala6hxzSW0npOxY=
+go.opentelemetry.io/collector/confmap v1.54.0 h1:RUoxQ4uAYHTI57GfHh61D00tTQsXm9T88ozrAiicByc=
+go.opentelemetry.io/collector/confmap v1.54.0/go.mod h1:mQxG8bk0IWIt9gbWMvzE+cRkOuCuzbzkNGBq2YJ4wNM=
+go.opentelemetry.io/collector/confmap/xconfmap v0.148.0 h1:UW8MX5VlKJf67x4Et7J9kPwP9Rv4VSmJ+UUpgRcb//c=
+go.opentelemetry.io/collector/confmap/xconfmap v0.148.0/go.mod h1:4qTMr3V0uSXXac9wVs/UD5fIqRKw5yIl58+Vjsc6RHM=
+go.opentelemetry.io/collector/consumer v1.54.0 h1:RGGtUN+GbkV1px3T6XdUHmgJ+ldJ1hAHdesFzW/wgL0=
+go.opentelemetry.io/collector/consumer v1.54.0/go.mod h1:1PC6XINTL9DdT1bwvfMdHE72EB4RWU/WcPemUrhqKN8=
+go.opentelemetry.io/collector/featuregate v1.54.0 h1:ufo5Hy4Co9pcHVg24hyanm8qFG3TkkYbVyQXPVAbwDc=
+go.opentelemetry.io/collector/featuregate v1.54.0/go.mod h1:PS7zY/zaCb28EqciePVwRHVhc3oKortTFXsi3I6ee4g=
+go.opentelemetry.io/collector/internal/componentalias v0.148.0 h1:Y6MftNIZSzOr47TTj6A2z2UR3IwbeG46sAQshicGtDg=
+go.opentelemetry.io/collector/internal/componentalias v0.148.0/go.mod h1:uwKzfehzwRgHxdHgFXYSBHNBeWSSqsqQYGWr5fk08G0=
+go.opentelemetry.io/collector/pdata v1.54.0 h1:3LharKb792cQ3VrUGxd3IcpWwfu3ST+GSTU382jVz1s=
+go.opentelemetry.io/collector/pdata v1.54.0/go.mod h1:+MqC3VVOv/EX9YVFUo+mI4F0YmwJ+fXBYwjmu+mRiZ8=
+go.opentelemetry.io/collector/pipeline v1.54.0 h1:jYlCkdFLITVBdeB+IGS07zXWywEgvT3Ky46vdKKT+Ks=
+go.opentelemetry.io/collector/pipeline v1.54.0/go.mod h1:RD90NG3Jbk965Xaqym3JyHkuol4uZJjQVUkD9ddXJIs=
+go.opentelemetry.io/collector/processor v1.54.0 h1:zmHBFiEFmU9ZYuHhVP3lHIkbfy+ueapzGpTdXVMcWBg=
+go.opentelemetry.io/collector/processor v1.54.0/go.mod h1:L0lA6DZ0VbrtQBg44cmYfSpRlgm4zxW1I6QfBnRizPw=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.67.0 h1:c9r/G1CSw4dPI1jaNNG9RnQP+q4SvZnHciDQJVIvchU=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.67.0/go.mod h1:gO9smoZe9KnZcJCqcB0lMmQ4Z5VEifYmjMTpnwtTSuQ=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
+go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
+go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
+go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
+go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
+go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
+go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
+go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
+go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
+go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
+go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
+go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
+golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
+golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
+golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
+gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
+gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+google.golang.org/api v0.272.0 h1:eLUQZGnAS3OHn31URRf9sAmRk3w2JjMx37d2k8AjJmA=
+google.golang.org/api v0.272.0/go.mod h1:wKjowi5LNJc5qarNvDCvNQBn3rVK8nSy6jg2SwRwzIA=
+google.golang.org/genproto v0.0.0-20260217215200-42d3e9bedb6d h1:vsOm753cOAMkt76efriTCDKjpCbK18XGHMJHo0JUKhc=
+google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 h1:41r6JMbpzBMen0R/4TZeeAmGXSJC7DftGINUodzTkPI=
+google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c h1:xgCzyF2LFIO/0X2UAoVRiXKU5Xg6VjToG4i2/ecSswk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
+google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
+gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k=
+gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ=
+k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4=
+k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8=
+k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=
+k8s.io/client-go v0.35.3 h1:s1lZbpN4uI6IxeTM2cpdtrwHcSOBML1ODNTCCfsP1pg=
+k8s.io/client-go v0.35.3/go.mod h1:RzoXkc0mzpWIDvBrRnD+VlfXP+lRzqQjCmKtiwZ8Q9c=
+k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
+k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
+k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE=
+k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ=
+k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck=
+k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

scripts/promqltestcorpus/main.go

@@ -0,0 +1,119 @@
+// Command promqltestcorpus extracts an absolute-truth conformance corpus from
+// the upstream Prometheus promqltest testdata scripts.
+//
+// The integration suites compare our PromQL serving paths against each other
+// (parity) or against nothing (smoke); both are blind to a bug that moves the
+// oracle — anything that changes what the engine is fed. This corpus is the
+// third leg: the samples come from upstream's own load notation (parsed by
+// upstream's parser), the expected outputs are computed by the vendored
+// reference engine over those samples, and both are frozen to JSON. The
+// Python suite tests/integration/tests/promqlconformance replays ingestion
+// and asserts API responses against the frozen expectations — an oracle that
+// does not move when the querier or the transpiler changes.
+//
+// Regenerate (after bumping the vendored Prometheus) with:
+//
+//	cd scripts/promqltestcorpus && go run . \
+//	  -out ../../tests/integration/testdata/promqltestcorpus/corpus.json
+//
+// upstream.go holds verbatim copies of upstream's private .test-format
+// parsing; refresh it against promql/promqltest/test.go on every version
+// bump. Drift fails loudly: the generator parses the NEW module's testdata,
+// so unknown syntax surfaces here, and regeneration is already a mandatory
+// step of any bump.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/promql/parser"
+	"github.com/prometheus/prometheus/promql/promqltest"
+)
+
+func main() {
+	out := flag.String("out", os.Getenv("PROMQLTEST_CORPUS_OUT"), "path to write corpus.json")
+	flag.Parse()
+	if *out == "" {
+		log.Fatal("set -out (or PROMQLTEST_CORPUS_OUT) to the corpus destination")
+	}
+	if err := run(*out); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run(out string) error {
+	promDir, promVersion, err := prometheusModule()
+	if err != nil {
+		return err
+	}
+	testdataDir := filepath.Join(promDir, "promql", "promqltest", "testdata")
+	files, err := filepath.Glob(filepath.Join(testdataDir, "*.test"))
+	if err != nil {
+		return err
+	}
+	if len(files) == 0 {
+		return fmt.Errorf("no .test files under %s", testdataDir)
+	}
+
+	// NewTestEngine's options minus EnableDelayedNameRemoval: upstream's
+	// testdata assumes that feature, but the Prometheus server default and
+	// our engine (pkg/prometheus/engine.go) run with it off — the oracle
+	// must model the semantics we serve.
+	engine := promql.NewEngine(promql.EngineOpts{
+		MaxSamples:               maxSamples,
+		Timeout:                  100 * time.Second,
+		NoStepSubqueryIntervalFn: func(int64) int64 { return time.Minute.Milliseconds() },
+		EnableAtModifier:         true,
+		EnableNegativeOffset:     true,
+		LookbackDelta:            lookbackMs * time.Millisecond,
+		Parser:                   parser.NewParser(promqltest.TestParserOpts),
+	})
+	defer func() { _ = engine.Close() }()
+	seriesParser := parser.NewParser(promqltest.TestParserOpts)
+	// Standard options: an expression the server-side parser would reject is
+	// not servable, so it must not enter the corpus.
+	exprParser := parser.NewParser(parser.Options{})
+
+	c, skips, err := generate(files, engine, seriesParser, exprParser, promVersion)
+	if err != nil {
+		return err
+	}
+	if err := writeCorpus(out, c); err != nil {
+		return err
+	}
+
+	log.Printf("wrote %s: %d cases over %d datasets", out, len(c.Cases), len(c.Datasets))
+	keys := make([]string, 0, len(skips))
+	for k := range skips {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		log.Printf("skipped %5d  %s", skips[k], k)
+	}
+	return nil
+}
+
+// prometheusModule locates the vendored prometheus module in the module
+// cache; the generator always parses the testdata of the version this
+// module requires, so a version bump regenerates against the new scripts.
+func prometheusModule() (dir, version string, err error) {
+	outDir, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", "github.com/prometheus/prometheus").Output()
+	if err != nil {
+		return "", "", fmt.Errorf("locating prometheus module: %w", err)
+	}
+	outVer, err := exec.Command("go", "list", "-m", "-f", "{{.Version}}", "github.com/prometheus/prometheus").Output()
+	if err != nil {
+		return "", "", fmt.Errorf("resolving prometheus version: %w", err)
+	}
+	return strings.TrimSpace(string(outDir)), strings.TrimSpace(string(outVer)), nil
+}

scripts/promqltestcorpus/upstream.go

@@ -0,0 +1,163 @@
+// This file carries the upstream promqltest .test-format knowledge this
+// generator depends on. The patterns are verbatim copies of unexported
+// definitions in prometheus@v0.311.3 promql/promqltest/test.go (upstream
+// exposes no public API for parsing the format short of running assertions
+// through a testing.TB); the loader is adapted from loadCmd.set/append in
+// the same file. REFRESH THIS FILE against test.go on every prometheus
+// version bump.
+package main
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/util/teststorage"
+)
+
+// Copied verbatim from promql/promqltest/test.go (prometheus@v0.311.3).
+var (
+	patLoad        = regexp.MustCompile(`^load(?:_(with_nhcb))?\s+(.+?)$`)
+	patEvalInstant = regexp.MustCompile(`^eval(?:_(fail|warn|ordered|info))?\s+instant\s+(?:at\s+(.+?))?\s+(.+)$`)
+	patEvalRange   = regexp.MustCompile(`^eval(?:_(fail|warn|info))?\s+range\s+from\s+(.+)\s+to\s+(.+)\s+step\s+(.+?)\s+(.+)$`)
+	patExpect      = regexp.MustCompile(`^expect\s+(ordered|fail|warn|no_warn|info|no_info)(?:\s+(regex|msg):(.+))?$`)
+)
+
+// testStartTime is upstream's epoch for all load offsets (test.go).
+var testStartTime = time.Unix(0, 0).UTC()
+
+// command is one column-0 block of a .test script with its attached
+// continuation lines.
+type command struct {
+	kind string // "load" | "eval" | "clear" | "skip"
+	head string
+	body []string
+	line int
+}
+
+// parseScript tokenizes a .test script into column-0 commands with their
+// indented lines, classifying heads with upstream's own patterns (line
+// walking follows (*test).parse in test.go: blank lines and #-comments
+// reset, indentation attaches). eval_fail / eval_warn / eval_info /
+// eval_ordered assert errors, warnings or ordering — none of which cross
+// the API comparably — so their modifier forms are skipped.
+func parseScript(script string) []command {
+	var cmds []command
+	var cur *command
+	for i, line := range strings.Split(script, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" || strings.HasPrefix(trimmed, "#") {
+			cur = nil
+			continue
+		}
+		isTop := line[0] != ' ' && line[0] != '\t'
+		if !isTop {
+			if cur != nil {
+				cur.body = append(cur.body, trimmed)
+			}
+			continue
+		}
+		switch {
+		case trimmed == "clear":
+			cmds = append(cmds, command{kind: "clear", line: i + 1})
+			cur = nil
+		case patLoad.MatchString(trimmed):
+			// load_with_nhcb stays kind "load": checkLoad rejects the
+			// variant, which poisons the whole segment — evals over a
+			// partially-loaded dataset must not enter the corpus.
+			cmds = append(cmds, command{kind: "load", head: trimmed, line: i + 1})
+			cur = &cmds[len(cmds)-1]
+		case patEvalInstant.MatchString(trimmed):
+			if m := patEvalInstant.FindStringSubmatch(trimmed); m[1] != "" {
+				cmds = append(cmds, command{kind: "skip", head: "eval_" + m[1], line: i + 1})
+				cur = nil
+				break
+			}
+			cmds = append(cmds, command{kind: "eval", head: trimmed, line: i + 1})
+			cur = &cmds[len(cmds)-1]
+		case patEvalRange.MatchString(trimmed):
+			if m := patEvalRange.FindStringSubmatch(trimmed); m[1] != "" {
+				cmds = append(cmds, command{kind: "skip", head: "eval_" + m[1], line: i + 1})
+				cur = nil
+				break
+			}
+			cmds = append(cmds, command{kind: "eval", head: trimmed, line: i + 1})
+			cur = &cmds[len(cmds)-1]
+		default:
+			cmds = append(cmds, command{kind: "skip", head: strings.Fields(trimmed)[0], line: i + 1})
+			cur = nil
+		}
+	}
+	return cmds
+}
+
+// loadSeriesStorage builds a TSDB with the load blocks' samples, adapted
+// from loadCmd.set/append (test.go): each series' samples sit at
+// testStartTime + i*gap, omitted values leave gaps, and — like loadCmd.set's
+// hash-keyed defs map — a series redefined within one load block replaces
+// its earlier definition entirely (upstream testdata relies on this:
+// aggregators.test defines data{test="inf3",point="d"} twice). Only float
+// samples are supported; checkLoad guarantees no histogram series reach
+// here.
+func loadSeriesStorage(seriesParser seriesDescParser, loads []command) (*teststorage.TestStorage, error) {
+	type def struct {
+		metric  labels.Labels
+		samples []promql.Sample
+	}
+	defs := map[uint64]def{}
+	var order []uint64
+	for _, l := range loads {
+		fields := strings.Fields(l.head)
+		gapDur, err := model.ParseDuration(fields[1])
+		if err != nil {
+			return nil, fmt.Errorf("load interval %q: %w", fields[1], err)
+		}
+		gap := time.Duration(gapDur)
+		for _, line := range l.body {
+			metric, vals, err := seriesParser.ParseSeriesDesc(line)
+			if err != nil {
+				return nil, fmt.Errorf("series %q: %w", line, err)
+			}
+			samples := make([]promql.Sample, 0, len(vals))
+			ts := testStartTime
+			for _, v := range vals {
+				if !v.Omitted {
+					samples = append(samples, promql.Sample{T: ts.UnixMilli(), F: v.Value})
+				}
+				ts = ts.Add(gap)
+			}
+			h := metric.Hash()
+			if _, seen := defs[h]; !seen {
+				order = append(order, h)
+			}
+			defs[h] = def{metric: metric, samples: samples}
+		}
+	}
+
+	stor, err := teststorage.NewWithError()
+	if err != nil {
+		return nil, err
+	}
+	app := stor.Appender(context.Background())
+	for _, h := range order {
+		d := defs[h]
+		for _, s := range d.samples {
+			if _, err := app.Append(0, d.metric, s.T, s.F); err != nil {
+				return nil, err
+			}
+		}
+	}
+	if err := app.Commit(); err != nil {
+		return nil, err
+	}
+	return stor, nil
+}
+
+// defaultEpsilon is upstream's relative tolerance for sample values
+// (promql/promqltest/test.go).
+const defaultEpsilon = 0.000001

tests/fixtures/metrics.py

@@ -687,11 +687,17 @@ def insert_metrics_to_clickhouse(conn, metrics: list[Metrics]) -> None:
     Pure function so the seeder container can reuse the exact insert path
     used by the pytest fixture. `conn` is a clickhouse-connect Client.
     """
-    time_series_map: dict[int, MetricsTimeSeries] = {}
+    # One registration row per (series, hour bucket), unix_milli floored to
+    # the hour — the exporter's exact shape. Readers floor lookup windows to
+    # these buckets: skipping per-bucket re-registration or keeping raw
+    # mid-hour timestamps hides series in ways production never sees.
+    time_series_map: dict[tuple[int, int], MetricsTimeSeries] = {}
     for metric in metrics:
         fp = int(metric.time_series.fingerprint)
-        if fp not in time_series_map:
-            time_series_map[fp] = metric.time_series
+        hour_bucket = int(metric.time_series.unix_milli) // 3_600_000
+        if (fp, hour_bucket) not in time_series_map:
+            metric.time_series.unix_milli = np.int64(hour_bucket * 3_600_000)
+            time_series_map[(fp, hour_bucket)] = metric.time_series
 
     if len(time_series_map) > 0:
         conn.insert(

tests/fixtures/querier.py

@@ -168,6 +168,7 @@ def make_query_request(
     variables: dict | None = None,
     no_cache: bool = True,
     timeout: int = QUERY_TIMEOUT,
+    headers: dict | None = None,
 ) -> requests.Response:
     if format_options is None:
         format_options = {"formatTableResultForUI": False, "fillGaps": False}
@@ -187,7 +188,7 @@ def make_query_request(
     return requests.post(
         signoz.self.host_configs["8080"].get("/api/v5/query_range"),
         timeout=timeout,
-        headers={"authorization": f"Bearer {token}"},
+        headers={"authorization": f"Bearer {token}", **(headers or {})},
         json=payload,
     )
 

tests/integration/testdata/promqltestcorpus/known_divergences.json

@@ -0,0 +1,4 @@
+{
+  "note": "Divergences of the CURRENT promql serving path from the upstream reference engine, enforced exactly by 01_upstream_corpus.py in both directions. These document shipped defects, not test debt: the dominant class is the v1 remote-read fetch injecting a synthetic 'fingerprint' label into every series (pkg/prometheus/clickhouseprometheus/json.go), which breaks without() grouping and default vector matching. Entries must be REMOVED as the serving path is fixed or swapped.",
+  "divergences": {}
+}

tests/integration/testdata/promqltestcorpus/known_divergences_v2.json

@@ -0,0 +1,4 @@
+{
+  "note": "Divergences of the clickhousev2 provider (pinned via X-SigNoz-PromQL-Provider) from the upstream reference engine, enforced exactly by 01_upstream_corpus.py in both directions. This ledger is the rollout scorecard for the provider swap: the default provider cannot be replaced by clickhousev2 while anything is listed here. Entries must carry the defect's cause and be REMOVED as the provider is fixed.",
+  "divergences": {}
+}

tests/integration/tests/promqlconformance/01_upstream_corpus.py

@@ -0,0 +1,256 @@
+"""
+Upstream promqltest conformance: replay the frozen corpus extracted from
+Prometheus' own promql/promqltest testdata and assert our API returns the
+reference engine's answers.
+
+Unlike live-vs-live parity suites, the oracle here is a committed file
+(tests/integration/testdata/promqltestcorpus/corpus.json), generated by
+scripts/promqltestcorpus from upstream's load scripts and the vendored
+reference engine. It therefore keeps working when the serving path itself is
+the thing being changed — the one situation where comparing two live paths
+against each other is blind.
+
+Every case replays on both serving paths — the default provider, and the
+clickhousev2 provider pinned via the flag-gated X-SigNoz-PromQL-Provider
+header (see conftest.py) — and each leg is asserted against the same frozen
+expectations, each leg against its own known-divergences ledger. The legs
+are deliberately never asserted against each other: both can sit within one
+rounding quantum of the expected value yet differ from each other by up to
+two quanta when a true value straddles a rounding boundary, so a leg-vs-leg
+equality check would reintroduce exactly the boundary noise the quantum
+tolerance exists to absorb. Because both legs anchor to the same oracle over
+the same ingested bytes, a case failing on one leg while passing on the
+other already localizes the defect to that provider — and the printed
+DIVERGED lines for both legs are the side-by-side view for triage.
+
+Datasets are placed on disjoint time windows (2h isolation gaps, far beyond
+the 5m lookback) so one bulk ingest serves every case without cross-talk.
+Expected values carry the API's 3-significant-decimal rounding, mirrored by
+the generator; comparison allows one rounding quantum for ULP-at-boundary
+noise between storage iteration orders.
+"""
+
+import json
+import math
+import os
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from http import HTTPStatus
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.metrics import Metrics
+from fixtures.querier import get_all_series, make_query_request
+
+TESTDATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "testdata")
+CORPUS_FILE = os.path.join(TESTDATA_DIR, "promqltestcorpus", "corpus.json")
+
+# One ledger per leg, enforced exactly in both directions. The default leg's
+# ledger is empty and pinned there; the clickhousev2 ledger is the rollout
+# scorecard — the provider swap is measured by burning it down to empty.
+LEDGER_FILES = {
+    "default": os.path.join(TESTDATA_DIR, "promqltestcorpus", "known_divergences.json"),
+    "clickhousev2": os.path.join(TESTDATA_DIR, "promqltestcorpus", "known_divergences_v2.json"),
+}
+
+LEGS: list[tuple[str, dict | None]] = [
+    ("default", None),
+    ("clickhousev2", {"X-SigNoz-PromQL-Provider": "clickhousev2"}),
+]
+
+ISOLATION_GAP_MS = 2 * 3600 * 1000
+SPECIALS = {"NaN": math.nan, "Inf": math.inf, "-Inf": -math.inf}
+
+
+def _decode(v: float | str) -> float:
+    if isinstance(v, str):
+        return SPECIALS[v]
+    return float(v)
+
+
+def _values_close(a: float, b: float) -> bool:
+    if math.isnan(a) or math.isnan(b):
+        return math.isnan(a) and math.isnan(b)
+    if math.isinf(a) or math.isinf(b):
+        return a == b
+    if a == b:
+        return True
+    # Both sides carry the API's rounding (>=1: three decimal places; <1:
+    # three significant digits). A true value sitting exactly on a rounding
+    # boundary can round either way when the two computations differ at ULP
+    # level (float aggregation order over series is storage-iteration
+    # dependent), so allow one rounding quantum.
+    scale = max(abs(a), abs(b))
+    if scale >= 1:
+        # Values too large to round pass through unrounded; give those an
+        # ULP-class relative grace on top of the rounding quantum.
+        quantum = max(1e-3, scale * 1e-9)
+    else:
+        quantum = 10 ** (math.floor(math.log10(scale)) - 2)
+    return abs(a - b) <= quantum + 1e-12
+
+
+def _labelset(labels: dict[str, str]) -> tuple:
+    return tuple(sorted(labels.items()))
+
+
+def _response_series(data: dict) -> tuple[dict[tuple, dict[int, float]], list[tuple]]:
+    """Returns (series map, duplicate labelsets). A response carrying several
+    series with identical visible labels is itself a defect signal (e.g. a
+    hidden grouping label stripped on the way out) and must not be silently
+    collapsed into one entry."""
+    out: dict[tuple, dict[int, float]] = {}
+    duplicates: list[tuple] = []
+    # Empty results serialize with null aggregations/series/values fields.
+    for series in get_all_series(data, "A") or []:
+        lbls = {l["key"]["name"]: str(l["value"]) for l in series.get("labels") or []}
+        points = {int(v["timestamp"]): _decode(v["value"]) for v in series.get("values") or []}
+        key = _labelset(lbls)
+        if key in out:
+            duplicates.append(key)
+        out[key] = points
+    return out, duplicates
+
+
+def _case_failure(
+    signoz: types.SigNoz,
+    token: str,
+    case: dict,
+    base: int,
+    headers: dict | None,
+) -> str | None:
+    """Replays one corpus case on one leg; returns a failure line or None."""
+    start_ms = base + case["start_ms"]
+    end_ms = base + case["end_ms"]
+    step_s = max(1, case["step_ms"] // 1000)
+    req_start_ms = start_ms
+    if case["instant"]:
+        # The API rejects start == end; ask for one extra step backward
+        # and compare only at the instant timestamp. Nudging the start
+        # earlier instead of the end later keeps every window that the
+        # expected values were computed from untouched.
+        req_start_ms = start_ms - step_s * 1000
+    query = {
+        "type": "promql",
+        "spec": {"name": "A", "query": case["expr"], "step": step_s},
+    }
+
+    case_id = f"{case['source']}[{case['variant']}]"
+    response = make_query_request(signoz, token, req_start_ms, end_ms, [query], headers=headers)
+    if response.status_code != HTTPStatus.OK:
+        return f"{case_id}: HTTP {response.status_code} for {case['expr']!r}: {response.text[:200]}"
+
+    actual, duplicates = _response_series(response.json())
+    if duplicates:
+        return f"{case_id}: response carries multiple series with identical labels for {case['expr']!r}: {[dict(d) for d in duplicates[:3]]}"
+    if case["instant"]:
+        # Keep only the instant point; the extra grid step is a request
+        # encoding byproduct, not part of the assertion.
+        actual = {lset: {ts: v for ts, v in pts.items() if ts == end_ms} for lset, pts in actual.items()}
+        actual = {lset: pts for lset, pts in actual.items() if pts}
+    expected: dict[tuple, dict[int, float]] = {}
+    for res in case["expected"]:
+        points = {base + off_ms: _decode(v) for off_ms, v in res["points"]}
+        expected[_labelset(res["labels"])] = points
+
+    if set(actual) != set(expected):
+        missing = set(expected) - set(actual)
+        extra = set(actual) - set(expected)
+        return f"{case_id}: series mismatch for {case['expr']!r} (missing={sorted(missing)[:3]} extra={sorted(extra)[:3]}) actual={[(dict(k), {t - base: v for t, v in pts.items()}) for k, pts in actual.items()]}"
+
+    for lset, exp_points in expected.items():
+        act_points = actual[lset]
+        if set(act_points) != set(exp_points):
+            return f"{case_id}: timestamp mismatch for {case['expr']!r} series {dict(lset)} (expected {len(exp_points)} points, got {len(act_points)})"
+        for ts, exp_v in exp_points.items():
+            if not _values_close(act_points[ts], exp_v):
+                return f"{case_id}: value mismatch for {case['expr']!r} series {dict(lset)} at {ts}: expected {exp_v}, got {act_points[ts]}"
+    return None
+
+
+def test_upstream_promqltest_corpus(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_metrics: Callable[[list[Metrics]], None],
+) -> None:
+    with open(CORPUS_FILE, encoding="utf-8") as f:
+        corpus = json.load(f)
+
+    cases_by_dataset: dict[int, list[dict]] = {}
+    for case in corpus["cases"]:
+        cases_by_dataset.setdefault(case["dataset"], []).append(case)
+
+    # Lay datasets end to end on the timeline, newest last, ending safely in
+    # the past; spans are per-dataset so the whole corpus stays within days.
+    spans = {}
+    for ds in corpus["datasets"]:
+        sample_max = max((s["samples"][-1][0] for s in ds["series"] if s["samples"]), default=0)
+        case_max = max((c["end_ms"] for c in cases_by_dataset.get(ds["id"], [])), default=0)
+        spans[ds["id"]] = max(sample_max, case_max) + corpus["meta"]["lookback_ms"]
+
+    # Hour-aligned dataset bases: registration rows are hour-bucketed, so
+    # behavior depends on where samples fall relative to hour boundaries —
+    # the exact known-divergences enforcement needs that identical every run.
+    hour_ms = 3_600_000
+    advances = {ds["id"]: -(-(spans[ds["id"]] + ISOLATION_GAP_MS) // hour_ms) * hour_ms for ds in corpus["datasets"]}
+    total = sum(advances.values())
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    cursor = (int((now - timedelta(hours=1)).timestamp() * 1000) - total) // hour_ms * hour_ms
+
+    bases: dict[int, int] = {}
+    metrics: list[Metrics] = []
+    for ds in corpus["datasets"]:
+        bases[ds["id"]] = cursor
+        for series in ds["series"]:
+            labels = dict(series["labels"])
+            metric_name = labels.pop("__name__")
+            for off_ms, raw in series["samples"]:
+                stale = raw == "stale"
+                metrics.append(
+                    Metrics(
+                        metric_name=metric_name,
+                        labels=labels,
+                        timestamp=datetime.fromtimestamp((cursor + off_ms) / 1000, tz=UTC),
+                        value=0.0 if stale else _decode(raw),
+                        flags=1 if stale else 0,
+                    )
+                )
+        cursor += advances[ds["id"]]
+
+    insert_metrics(metrics)
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    failures: dict[str, list[str]] = {leg: [] for leg, _ in LEGS}
+    for case in corpus["cases"]:
+        for leg, headers in LEGS:
+            f_line = _case_failure(signoz, token, case, bases[case["dataset"]], headers)
+            if f_line:
+                failures[leg].append(f_line)
+
+    for leg, _ in LEGS:
+        for f_line in failures[leg]:
+            print("DIVERGED", f"[{leg}]", f_line)
+
+    # Known divergences are defects of that leg's serving path, frozen with
+    # reasons. Each set is enforced exactly in both directions: a NEW
+    # divergence is a regression, and a known divergence that starts passing
+    # must be removed from the file. Problems across both legs are collected
+    # before asserting so one leg's failure never hides the other's.
+    problems: list[str] = []
+    for leg, _ in LEGS:
+        known: dict[str, str] = {}
+        if os.path.exists(LEDGER_FILES[leg]):
+            with open(LEDGER_FILES[leg], encoding="utf-8") as f:
+                known = json.load(f)["divergences"]
+
+        failed_ids = {f_line.split(": ", 1)[0] for f_line in failures[leg]}
+        unexpected = [f_line for f_line in failures[leg] if f_line.split(": ", 1)[0] not in known]
+        now_passing = sorted(set(known) - failed_ids)
+
+        if unexpected:
+            problems.append(f"[{leg}] {len(unexpected)} corpus cases diverged beyond the known set:\n" + "\n".join(unexpected[:25]))
+        if now_passing:
+            problems.append(f"[{leg}] {len(now_passing)} known divergences now pass — remove them from {os.path.basename(LEDGER_FILES[leg])}: {now_passing[:25]}")
+
+    assert not problems, "\n\n".join(problems)

tests/integration/tests/promqlconformance/02_fingerprint_probe.py

@@ -0,0 +1,141 @@
+"""
+Regression tests for series identity in the PromQL serving path (PR #8563).
+
+#8563 fixed a real duplicate-labelset collision by injecting a synthetic
+per-series "fingerprint" label, which silently broke without() grouping and
+unaggregated vector matching; the adapter now merges fingerprints sharing a
+labelset instead. Pinned here:
+
+  1. Clean data: without() yields exactly the grouped series with correct
+     sums; "fingerprint" behaves as any absent label.
+  2. The #8563 incident: one series under two fingerprints (empty-valued vs
+     absent label). Both must come back as ONE merged series — not a
+     "duplicate series" error, not duplicate identical-labeled output.
+"""
+
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from http import HTTPStatus
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.metrics import Metrics
+from fixtures.querier import get_all_series, make_query_request
+
+METRIC = "probe_requests"
+EVOLVED_METRIC = "probe_schema_evolution"
+
+
+def _value_at(view_entry: tuple[dict, list], ts_ms: int) -> float:
+    for ts, v in view_entry[1]:
+        if ts == ts_ms:
+            return float(v)
+    raise AssertionError(f"no point at {ts_ms} in {view_entry}")
+
+
+def test_identical_labelsets_merge_and_grouping(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_metrics: Callable[[list[Metrics]], None],
+) -> None:
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    base = now - timedelta(minutes=30)
+
+    # Scenario 1: four clean series, 2 groups x 2 instances, 3 samples each.
+    labelsets = [
+        {"group": "canary", "instance": "0"},
+        {"group": "canary", "instance": "1"},
+        {"group": "production", "instance": "0"},
+        {"group": "production", "instance": "1"},
+    ]
+    metrics: list[Metrics] = []
+    for i, lbls in enumerate(labelsets):
+        for k in range(3):
+            metrics.append(
+                Metrics(
+                    metric_name=METRIC,
+                    labels=dict(lbls),
+                    timestamp=base + timedelta(minutes=k),
+                    value=float((i + 1) * 100 + k),
+                )
+            )
+
+    # Scenario 2 (PR #8563): one conceptual series under two fingerprints.
+    # The first three samples carry schema_url="" (empty value, dropped at
+    # read time); the next three drop the label entirely (new fingerprint).
+    for k in range(3):
+        metrics.append(
+            Metrics(
+                metric_name=EVOLVED_METRIC,
+                labels={"job": "api", "schema_url": ""},
+                timestamp=base + timedelta(minutes=k),
+                value=float(k + 1),
+            )
+        )
+    for k in range(3, 6):
+        metrics.append(
+            Metrics(
+                metric_name=EVOLVED_METRIC,
+                labels={"job": "api"},
+                timestamp=base + timedelta(minutes=k),
+                value=float(k + 1),
+            )
+        )
+    insert_metrics(metrics)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    def run(promql: str, start_ms: int, end_ms: int) -> list[tuple[dict, list]]:
+        q = {"type": "promql", "spec": {"name": "A", "query": promql, "step": 60}}
+        resp = make_query_request(signoz, token, start_ms, end_ms, [q])
+        assert resp.status_code == HTTPStatus.OK, f"{promql!r}: {resp.text[:300]}"
+        out = []
+        for series in get_all_series(resp.json(), "A") or []:
+            lbls = {l["key"]["name"]: str(l["value"]) for l in series.get("labels") or []}
+            vals = [(v["timestamp"], v["value"]) for v in series.get("values") or []]
+            out.append((lbls, vals))
+        return sorted(out, key=lambda x: sorted(x[0].items()))
+
+    end_ms = int((base + timedelta(minutes=2)).timestamp() * 1000)
+    start_ms = end_ms - 60_000
+
+    raw = run(METRIC, start_ms, end_ms)
+    assert len(raw) == 4, f"raw selector must show the 4 ingested series: {raw}"
+    assert len({tuple(sorted(l.items())) for l, _ in raw}) == 4
+    assert not any("fingerprint" in l for l, _ in raw), "no synthetic fingerprint label may appear in results"
+
+    count = run(f"count({METRIC})", start_ms, end_ms)
+    assert count and _value_at(count[0], end_ms) == 4
+
+    # without(instance): exactly one series per group, with the group sums —
+    # not per-fingerprint groups collapsing into duplicate labelsets.
+    without = run(f"sum without (instance) ({METRIC})", start_ms, end_ms)
+    assert [(l.get("group"), _value_at((l, v), end_ms)) for l, v in without] == [
+        ("canary", 304.0),
+        ("production", 704.0),
+    ], f"without(instance) must yield 2 correctly-summed groups: {without}"
+
+    # "fingerprint" is now just an absent label: adding it to without() must
+    # not change the result, and grouping by it collapses everything.
+    healed = run(f"sum without (instance, fingerprint) ({METRIC})", start_ms, end_ms)
+    assert [(l.get("group"), _value_at((l, v), end_ms)) for l, v in healed] == [
+        ("canary", 304.0),
+        ("production", 704.0),
+    ], f"without(instance, fingerprint) must equal without(instance): {healed}"
+
+    by_fp = run(f"sum by (fingerprint) ({METRIC})", start_ms, end_ms)
+    assert len(by_fp) == 1 and _value_at(by_fp[0], end_ms) == 304.0 + 704.0, f"by(fingerprint) must collapse to one group (label absent): {by_fp}"
+    assert "fingerprint" not in by_fp[0][0] or by_fp[0][0] == {}, by_fp
+
+    # Scenario 2: both fingerprints must come back as ONE merged series
+    # spanning the full range — no duplicate-series error, no duplicate
+    # identical-labeled output.
+    evo_start_ms = int(base.timestamp() * 1000)
+    evo_end_ms = int((base + timedelta(minutes=5)).timestamp() * 1000)
+    evolved = run(EVOLVED_METRIC, evo_start_ms, evo_end_ms)
+    assert len(evolved) == 1, f"label-evolution fingerprints must merge into one series: {evolved}"
+    lbls, _ = evolved[0]
+    assert lbls == {"__name__": EVOLVED_METRIC, "job": "api"}, evolved
+    got = [_value_at(evolved[0], evo_start_ms + m * 60_000) for m in range(6)]
+    assert got == [1.0, 2.0, 3.0, 4.0, 5.0, 6.0], f"merged series must carry both fingerprints' samples in order: {got}"

tests/integration/tests/promqlconformance/conftest.py

@@ -0,0 +1,39 @@
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz_promql_conformance(
+    network: Network,
+    migrator: types.Operation,  # pylint: disable=unused-argument
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.SigNoz:
+    """
+    Package-scoped SigNoz with use_prometheus_clickhouse_v2 on, so the corpus
+    can replay every case twice: once against the default provider and once
+    pinned to the clickhousev2 provider via the X-SigNoz-PromQL-Provider
+    header (which the flag gates). Each leg is asserted against the same
+    frozen expectations — see 01_upstream_corpus.py for why the legs are
+    never asserted against each other.
+    """
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz-promql-conformance",
+        env_overrides={
+            "SIGNOZ_FLAGGER_CONFIG_BOOLEAN_USE__PROMETHEUS__CLICKHOUSE__V2": True,
+        },
+    )

tests/integration/tests/querierlogs/11_aggregation_functions.py

@@ -20,12 +20,15 @@ from fixtures.querier import (
 # Numeric aggregation-function coverage for logs — the logs counterpart of
 # queriertraces/02_aggregation.py::test_traces_aggregate_functions. Logs querier
 # tests elsewhere only ever use count()/count_distinct(); here a grouped scalar
-# query computes sum / avg / min / max / p50 / p90 / p95 / p99 / countIf /
-# count_distinct over a numeric log attribute and asserts every value.
+# query computes sum / avg / min / max / p50 / p90 / p95 / p99 / countIf / rate /
+# rate_sum / count_distinct over a numeric log attribute and asserts every value.
 #
 # Log number attributes are stored as Float64, so aggregates come back as floats;
 # percentiles are matched with pytest.approx (ClickHouse quantile() and
 # numpy.percentile both linear-interpolate, but avoid ULP-level exact equality).
+#
+# The rate family divides by a window the caller does not otherwise see, so the
+# lookback is passed explicitly instead of taken from the request helper default.
 
 
 def test_logs_aggregate_functions(
@@ -41,11 +44,14 @@ def test_logs_aggregate_functions(
 
     Tests:
     A grouped scalar query computes count / sum / avg / min / max / p50 / p90 /
-    p95 / p99 over latency_ms, countIf over a numeric threshold, and
-    count_distinct over a string attribute — all matching values derived from the
-    inserted logs, ordered by count() desc.
+    p95 / p99 over latency_ms, countIf over a numeric threshold, rate and
+    rate_sum over the query window, and count_distinct over a string attribute —
+    all matching values derived from the inserted logs, ordered by count() desc.
     """
     now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    # A scalar rate divides by the whole query window, so both sides agree on it.
+    lookback_minutes = 5
+    rate_interval_seconds = lookback_minutes * 60
     # (service, latency_ms, endpoint)
     specs = [
         ("svc-a", 10, "/x"),
@@ -80,12 +86,14 @@ def test_logs_aggregate_functions(
             build_aggregation("p95(latency_ms)", "p95_l"),
             build_aggregation("p99(latency_ms)", "p99_l"),
             build_aggregation("countIf(latency_ms >= 25)", "slow"),
+            build_aggregation("rate()", "rate_all"),
+            build_aggregation("rate_sum(latency_ms)", "rate_sum_l"),
             build_aggregation("count_distinct(endpoint)", "endpoints"),
         ],
         group_by=[build_group_by_field("service.name", "string", "resource")],
         order=[build_order_by("count()", "desc")],
     )
-    response = make_scalar_query_request(signoz, token, now, [query])
+    response = make_scalar_query_request(signoz, token, now, [query], lookback_minutes=lookback_minutes)
 
     assert response.status_code == HTTPStatus.OK, response.text
     data = get_scalar_table_data(response.json())
@@ -109,6 +117,12 @@ def test_logs_aggregate_functions(
             float(np.percentile(latencies, 95)),  # p95(latency_ms)
             float(np.percentile(latencies, 99)),  # p99(latency_ms)
             sum(1 for latency in latencies if latency >= 25),  # countIf(latency_ms >= 25)
+            # Response floats are rounded to 3 decimals at or above 1 and to 3
+            # significant figures below it (roundToNonZeroDecimals). Every other
+            # value here is exact; the rates are the only ones that repeat, and
+            # they stay below 1 for this fixture, so mirror the latter form.
+            float(f"{len(latencies) / rate_interval_seconds:.3g}"),  # rate()
+            float(f"{sum(latencies) / rate_interval_seconds:.3g}"),  # rate_sum(latency_ms)
             len(set(endpoints)),  # count_distinct(endpoint)
         ]
         assert by_service[service] == pytest.approx(expected), f"{service}: {by_service[service]} != {expected}"

tests/integration/tests/queriermetrics/02_warnings.py

@@ -25,9 +25,16 @@ def test_histogram_p90_returns_warning_outside_data_window(
     now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
     metric_name = "test_p90_last_seen_bucket"
 
+    # Registration rows are written per (series, hour bucket) with
+    # hour-floored timestamps (the exporter's shape), and metadata lookups
+    # floor their window start to the hour. Data must therefore end a couple
+    # of hours back for the last-15m window to be genuinely outside every
+    # registration bucket; data merely 30 minutes stale shares an hour
+    # bucket with the floored window and does not warn (matching
+    # production behavior).
     metrics = Metrics.load_from_file(
         HISTOGRAM_FILE,
-        base_time=now - timedelta(minutes=90),
+        base_time=now - timedelta(hours=3),
         metric_name_override=metric_name,
     )
     insert_metrics(metrics)
@@ -42,8 +49,8 @@ def test_histogram_p90_returns_warning_outside_data_window(
 
     end_ms = int(now.timestamp() * 1000)
 
-    start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
-    response = make_query_request(signoz, token, start_2h, end_ms, [query])
+    start_4h = int((now - timedelta(hours=4)).timestamp() * 1000)
+    response = make_query_request(signoz, token, start_4h, end_ms, [query])
     assert response.status_code == HTTPStatus.OK
     assert response.json()["status"] == "success"
 

tests/integration/tests/queriertraces/02_aggregation.py

@@ -150,12 +150,17 @@ def test_traces_aggregate_functions(
     Tests:
     A grouped scalar query computes count / sum / avg / min / max / p50 / p90 over
     duration_nano, countIf over the intrinsic status_code and the calculated
-    response_status_code, and avg over a numeric attribute — all matching values
-    derived from the inserted spans. Under the corrupt variant the same-named
-    colliding attributes must not change any of these.
+    response_status_code, rate and rate_sum over the query window, and avg over a
+    numeric attribute — all matching values derived from the inserted spans. Under
+    the corrupt variant the same-named colliding attributes must not change any of
+    these.
     """
     extra_attrs, extra_resources = trace_noise(noise)
     now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    # A scalar rate divides by the whole query window, so both sides agree on it.
+    # Traces derives this separately from logs (telemetrytraces/statement_builder.go).
+    lookback_minutes = 5
+    rate_interval_seconds = lookback_minutes * 60
 
     def mk(service: str, dur_s: float, status: TracesStatusCode, rsc: str, latency: float) -> Traces:
         return Traces(
@@ -189,6 +194,8 @@ def test_traces_aggregate_functions(
         build_aggregation("p50(duration_nano)", "p50_d"),
         build_aggregation("p90(duration_nano)", "p90_d"),
         build_aggregation("countIf(status_code = 2)", "errs"),
+        build_aggregation("rate()", "rate_all"),
+        build_aggregation("rate_sum(duration_nano)", "rate_sum_d"),
         build_aggregation("avg(latency_ms)", "avg_lat"),
     ]
     query = build_traces_scalar_query(
@@ -196,7 +203,7 @@ def test_traces_aggregate_functions(
         group_by=[build_group_by_field("service.name", "string", "resource")],
         order=[build_order_by("count()", "desc")],
     )
-    response = make_scalar_query_request(signoz, token, now, [query])
+    response = make_scalar_query_request(signoz, token, now, [query], lookback_minutes=lookback_minutes)
 
     assert response.status_code == HTTPStatus.OK, response.text
     data = get_scalar_table_data(response.json())
@@ -214,6 +221,11 @@ def test_traces_aggregate_functions(
             float(np.percentile(durations, 50)),  # p50(duration_nano)
             float(np.percentile(durations, 90)),  # p90(duration_nano)
             sum(1 for s in group if int(s.status_code) == 2),  # countIf(status_code = 2)
+            # Response floats are rounded to 3 significant figures below 1 and to 3
+            # decimals at or above it (roundToNonZeroDecimals). The two rates land on
+            # either side of that boundary, so each mirrors its own form.
+            float(f"{len(group) / rate_interval_seconds:.3g}"),  # rate()
+            round(sum(durations) / rate_interval_seconds, 3),  # rate_sum(duration_nano)
             sum(latencies) / len(latencies),  # avg(latency_ms)
         )