fix: add back the flag in metadata

* refactor(infrastructure-monitoring-v2): remove pod phase and add pod status/counts

* fix(column-header): align to left

* fix(cell-value): ensure tooltip is correctly aligned

* fix(entity-traces): use - instead of N/A

* test(entity-traces): fix broken test

.devenv/docker/clickhouse/compose.yaml

@@ -1,6 +1,6 @@
 services:
   init-clickhouse:
-    image: clickhouse/clickhouse-server:25.5.6
+    image: clickhouse/clickhouse-server:25.12.5
     container_name: init-clickhouse
     command:
       - bash
@@ -18,7 +18,7 @@ services:
     volumes:
       - ${PWD}/fs/tmp/var/lib/clickhouse/user_scripts/:/var/lib/clickhouse/user_scripts/
   clickhouse:
-    image: clickhouse/clickhouse-server:25.5.6
+    image: clickhouse/clickhouse-server:25.12.5
     container_name: clickhouse
     volumes:
       - ${PWD}/fs/etc/clickhouse-server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml
@@ -67,7 +67,7 @@ services:
       timeout: 5s
       retries: 3
   telemetrystore-migrator:
-    image: signoz/signoz-otel-collector:v0.142.0
+    image: signoz/signoz-otel-collector:v0.144.6
     container_name: telemetrystore-migrator
     environment:
       - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://clickhouse:9000

.github/workflows/integrationci.yaml

@@ -53,6 +53,7 @@ jobs:
           - queriermetrics
           - querierscalar
           - queriercommon
+          - querierai
           - rawexportdata
           - role
           - rootuser
@@ -60,16 +61,17 @@ jobs:
           - querier_json_body
           - querier_skip_resource_fingerprint
           - ttl
+          - clickhousecluster
+          - metricreduction
         sqlstore-provider:
           - postgres
           - sqlite
         sqlite-mode:
           - wal
         clickhouse-version:
-          - 25.5.6
           - 25.12.5
         schema-migrator-version:
-          - v0.144.3
+          - v0.144.6
         postgres-version:
           - 15
     if: |

docs/api/openapi.yml

@@ -8565,6 +8565,7 @@ components:
     TelemetrytypesSource:
       enum:
       - meter
+      - ai
       type: string
     TelemetrytypesTelemetryFieldKey:
       properties:

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -3631,6 +3631,7 @@ export enum Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQue
 }
 export enum TelemetrytypesSourceDTO {
 	meter = 'meter',
+	ai = 'ai',
 }
 export interface Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQuerybuildertypesQuerybuildertypesv5LogAggregationDTO {
 	/**

frontend/src/container/InfraMonitoringHostsV2/utils.tsx

@@ -11,8 +11,6 @@ import { INFRA_MONITORING_ATTR_KEYS } from 'container/InfraMonitoringK8sV2/const
 import { CellValueTooltip } from 'container/InfraMonitoringK8sV2/components';
 import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
 import { DataSource } from 'types/common/queryBuilder';
-import TanStackTable from 'components/TanStackTableView';
-
 const HOSTNAME_DOCS_URL =
 	'https://signoz.io/docs/infrastructure-monitoring/hostmetrics/#host-name-is-blankempty';
 
@@ -23,11 +21,7 @@ export function HostnameCell({
 }): React.ReactElement {
 	const isEmpty = !hostName || !hostName.trim();
 	if (!isEmpty) {
-		return (
-			<CellValueTooltip value={hostName}>
-				<TanStackTable.Text>{hostName}</TanStackTable.Text>
-			</CellValueTooltip>
-		);
+		return <CellValueTooltip value={hostName} />;
 	}
 	return (
 		<>

frontend/src/container/InfraMonitoringK8sV2/Base/ColumnHeader.module.scss

@@ -11,6 +11,6 @@
 }
 
 .columnHeaderLabel {
-	text-align: center;
+	text-align: left;
 	padding: var(--spacing-2) var(--spacing-2) var(--spacing-2) 0px;
 }

frontend/src/container/InfraMonitoringK8sV2/Base/K8sBaseDetails.tsx

@@ -86,6 +86,23 @@ export interface K8sDetailsFilters {
 	end: number;
 }
 
+export interface CustomTabRenderProps<T> {
+	entity: T;
+	timeRange: { startTime: number; endTime: number };
+	selectedInterval: Time;
+	handleTimeChange: (
+		interval: Time | CustomTimeType,
+		dateTimeRange?: [number, number],
+	) => void;
+}
+
+export interface CustomTab<T> {
+	key: string;
+	label: string;
+	icon: React.ReactNode;
+	render: (props: CustomTabRenderProps<T>) => React.ReactNode;
+}
+
 export interface K8sBaseDetailsProps<T> {
 	category: InfraMonitoringEntity;
 	eventCategory: string;
@@ -122,20 +139,7 @@ export interface K8sBaseDetailsProps<T> {
 		showTraces?: boolean;
 		showEvents?: boolean;
 	};
-	customTabs?: Array<{
-		key: string;
-		label: string;
-		icon: React.ReactNode;
-		render: (props: {
-			entity: T;
-			timeRange: { startTime: number; endTime: number };
-			selectedInterval: Time;
-			handleTimeChange: (
-				interval: Time | CustomTimeType,
-				dateTimeRange?: [number, number],
-			) => void;
-		}) => React.ReactNode;
-	}>;
+	customTabs?: Array<CustomTab<T>>;
 }
 
 // eslint-disable-next-line sonarjs/cognitive-complexity
@@ -271,6 +275,33 @@ export default function K8sBaseDetails<T>({
 	const [selectedView, setSelectedView] = useInfraMonitoringView();
 	const effectiveView = hideDetailViewTabs ? VIEW_TYPES.METRICS : selectedView;
 
+	const validTabs = useMemo(() => {
+		const tabs: string[] = [];
+		if (tabVisibility.showMetrics) {
+			tabs.push(VIEW_TYPES.METRICS);
+		}
+		if (tabVisibility.showLogs) {
+			tabs.push(VIEW_TYPES.LOGS);
+		}
+		if (tabVisibility.showTraces) {
+			tabs.push(VIEW_TYPES.TRACES);
+		}
+		if (tabVisibility.showEvents) {
+			tabs.push(VIEW_TYPES.EVENTS);
+		}
+		if (customTabs) {
+			tabs.push(...customTabs.map((t) => t.key));
+		}
+		return tabs;
+	}, [tabVisibility, customTabs]);
+
+	useEffect(() => {
+		if (!hideDetailViewTabs && !validTabs.includes(selectedView)) {
+			const firstValid = validTabs[0] || VIEW_TYPES.METRICS;
+			void setSelectedView(firstValid);
+		}
+	}, [hideDetailViewTabs, selectedView, validTabs, setSelectedView]);
+
 	const [, setLogFiltersParam] = useInfraMonitoringLogFilters();
 	const [, setTracesFiltersParam] = useInfraMonitoringTracesFilters();
 	const [, setEventsFiltersParam] = useInfraMonitoringEventsFilters();
@@ -306,7 +337,10 @@ export default function K8sBaseDetails<T>({
 		}
 	}, [getMinMaxTime, selectedTime]);
 
-	const handleTabChange = (value: string): void => {
+	const handleTabChange = (value: string | null): void => {
+		if (!value) {
+			return;
+		}
 		setSelectedView(value);
 		setLogFiltersParam(null);
 		setTracesFiltersParam(null);

frontend/src/container/InfraMonitoringK8sV2/Base/__tests__/K8sBaseDetails.tabValidation.test.tsx

@@ -0,0 +1,221 @@
+import { Box } from '@signozhq/icons';
+import { screen } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+import { NuqsTestingAdapter } from 'nuqs/adapters/testing';
+import { act, render, waitFor } from 'tests/test-utils';
+
+import {
+	InfraMonitoringEntity,
+	INFRA_MONITORING_K8S_PARAMS_KEYS,
+	VIEW_TYPES,
+} from '../../constants';
+import K8sBaseDetails from '../K8sBaseDetails';
+
+jest.mock('container/TopNav/DateTimeSelectionV2/index.tsx', () => ({
+	__esModule: true,
+	default: (): JSX.Element => <div data-testid="mock-datetime" />,
+}));
+
+type TestEntity = {
+	name: string;
+	namespace: string;
+	cluster: string;
+};
+
+const mockEntity: TestEntity = {
+	name: 'test-pod',
+	namespace: 'default',
+	cluster: 'test-cluster',
+};
+
+function createBaseProps() {
+	return {
+		category: InfraMonitoringEntity.PODS,
+		eventCategory: 'Pod',
+		getSelectedItemExpression: (): string => 'k8s.pod.name = "test-pod"',
+		fetchEntityData: jest
+			.fn()
+			.mockResolvedValue({ data: mockEntity, error: null }),
+		getEntityName: (e: TestEntity): string => e.name,
+		getInitialLogTracesExpression: (): string => 'k8s.pod.name = "test-pod"',
+		getInitialEventsExpression: (): string => 'k8s.pod.name = "test-pod"',
+		metadataConfig: [
+			{ label: 'Name', getValue: (e: TestEntity): string => e.name },
+		],
+		entityWidgetInfo: [{ title: 'CPU', yAxisUnit: 'percent' }],
+		getEntityQueryPayload: jest.fn().mockReturnValue([]),
+		queryKeyPrefix: 'testPod',
+	};
+}
+
+interface RenderOptions {
+	view?: string;
+	tabsConfig?: {
+		showMetrics?: boolean;
+		showLogs?: boolean;
+		showTraces?: boolean;
+		showEvents?: boolean;
+	};
+	customTabs?: Array<{
+		key: string;
+		label: string;
+		icon: React.ReactNode;
+		render: () => React.ReactNode;
+	}>;
+}
+
+function renderK8sBaseDetails({
+	view = VIEW_TYPES.METRICS,
+	tabsConfig,
+	customTabs,
+}: RenderOptions = {}) {
+	const searchParams: Record<string, string> = {
+		[INFRA_MONITORING_K8S_PARAMS_KEYS.SELECTED_ITEM]: 'test-pod',
+		[INFRA_MONITORING_K8S_PARAMS_KEYS.VIEW]: view,
+	};
+
+	return render(
+		<NuqsTestingAdapter searchParams={searchParams}>
+			<K8sBaseDetails<TestEntity>
+				{...createBaseProps()}
+				tabsConfig={tabsConfig}
+				customTabs={customTabs}
+			/>
+		</NuqsTestingAdapter>,
+	);
+}
+
+function getSelectedTabText(): string | null {
+	const selectedTab = document.querySelector('[aria-checked="true"]');
+	return selectedTab?.textContent ?? null;
+}
+
+describe('K8sBaseDetails - Tab Validation', () => {
+	it('should reset view to METRICS when selected view is invalid', async () => {
+		act(() => {
+			renderK8sBaseDetails({ view: 'invalid-tab' });
+		});
+
+		await waitFor(() => {
+			expect(screen.getAllByText('test-pod').length).toBeGreaterThan(0);
+		});
+
+		await waitFor(() => {
+			expect(getSelectedTabText()).toContain('Metrics');
+		});
+	});
+
+	it('should reset to first available tab when METRICS is disabled and view is invalid', async () => {
+		act(() => {
+			renderK8sBaseDetails({
+				view: 'invalid-tab',
+				tabsConfig: { showMetrics: false },
+			});
+		});
+
+		await waitFor(() => {
+			expect(screen.getAllByText('test-pod').length).toBeGreaterThan(0);
+		});
+
+		await waitFor(() => {
+			expect(getSelectedTabText()).toContain('Logs');
+		});
+	});
+
+	it('should reset to custom tab when all standard tabs disabled and custom tab exists', async () => {
+		const customTabKey = 'pod-metrics';
+
+		act(() => {
+			renderK8sBaseDetails({
+				view: 'invalid-tab',
+				tabsConfig: {
+					showMetrics: false,
+					showLogs: false,
+					showTraces: false,
+					showEvents: false,
+				},
+				customTabs: [
+					{
+						key: customTabKey,
+						label: 'Pod Metrics',
+						icon: <Box size={14} />,
+						render: (): React.ReactNode => <div>Custom Tab</div>,
+					},
+				],
+			});
+		});
+
+		await waitFor(() => {
+			expect(screen.getAllByText('test-pod').length).toBeGreaterThan(0);
+		});
+
+		await waitFor(() => {
+			expect(getSelectedTabText()).toContain('Pod Metrics');
+		});
+	});
+
+	it('should NOT reset view when selected view is valid', async () => {
+		act(() => {
+			renderK8sBaseDetails({ view: VIEW_TYPES.LOGS });
+		});
+
+		await waitFor(() => {
+			expect(screen.getAllByText('test-pod').length).toBeGreaterThan(0);
+		});
+
+		await waitFor(() => {
+			expect(getSelectedTabText()).toContain('Logs');
+		});
+	});
+
+	it('should NOT reset view when custom tab is selected and exists', async () => {
+		const customTabKey = 'pod-metrics';
+
+		act(() => {
+			renderK8sBaseDetails({
+				view: customTabKey,
+				customTabs: [
+					{
+						key: customTabKey,
+						label: 'Pod Metrics',
+						icon: <Box size={14} />,
+						render: (): React.ReactNode => <div>Custom Tab</div>,
+					},
+				],
+			});
+		});
+
+		await waitFor(() => {
+			expect(screen.getAllByText('test-pod').length).toBeGreaterThan(0);
+		});
+
+		await waitFor(() => {
+			expect(getSelectedTabText()).toContain('Pod Metrics');
+		});
+	});
+
+	it('should keep the selected tab when the active tab is clicked again (untoggle guard)', async () => {
+		const user = userEvent.setup();
+
+		act(() => {
+			renderK8sBaseDetails({ view: VIEW_TYPES.LOGS });
+		});
+
+		await waitFor(() => {
+			expect(screen.getAllByText('test-pod').length).toBeGreaterThan(0);
+		});
+
+		await waitFor(() => {
+			expect(getSelectedTabText()).toContain('Logs');
+		});
+
+		const selectedTab = document.querySelector('[aria-checked="true"]');
+		expect(selectedTab).not.toBeNull();
+
+		await user.click(selectedTab as Element);
+
+		await waitFor(() => {
+			expect(getSelectedTabText()).toContain('Logs');
+		});
+	});
+});

frontend/src/container/InfraMonitoringK8sV2/Clusters/table.config.tsx

@@ -8,7 +8,7 @@ import { ExpandButtonWrapper } from 'container/InfraMonitoringK8sV2/components';
 import ColumnHeader from '../Base/ColumnHeader';
 import EntityGroupHeader from '../Base/EntityGroupHeader';
 import K8sGroupCell from '../Base/K8sGroupCell';
-import { formatBytes, getPodPhaseStatusItems } from '../commonUtils';
+import { formatBytes, getPodStatusItems } from '../commonUtils';
 import {
 	CellValueTooltip,
 	GroupedStatusCounts,
@@ -77,11 +77,7 @@ export const k8sClustersColumnsConfig: ClusterTableColumnConfig[] = [
 		visibilityBehavior: 'hidden-on-expand',
 		cell: ({ value }): React.ReactNode => {
 			const clusterName = value as string;
-			return (
-				<CellValueTooltip value={clusterName}>
-					<TanStackTable.Text>{clusterName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={clusterName} />;
 		},
 	},
 	{
@@ -121,23 +117,25 @@ export const k8sClustersColumnsConfig: ClusterTableColumnConfig[] = [
 		},
 	},
 	{
-		id: 'podCountsByPhase',
+		id: 'podCountsByStatus',
 		header: (): React.ReactNode => (
-			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/clusters#pod-counts-by-phase">
-				Pod Phases
+			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/clusters#pod-counts-by-status">
+				Pod Status
 			</ColumnHeader>
 		),
-		accessorFn: (row): InframonitoringtypesClusterRecordDTO['podCountsByPhase'] =>
-			row.podCountsByPhase,
+		accessorFn: (
+			row,
+		): InframonitoringtypesClusterRecordDTO['podCountsByStatus'] =>
+			row.podCountsByStatus,
 		width: { min: 250 },
 		enableSort: false,
 		cell: ({ row }): React.ReactNode => {
-			const podCountsByPhase = row.podCountsByPhase;
-			if (!podCountsByPhase) {
+			const podCountsByStatus = row.podCountsByStatus;
+			if (!podCountsByStatus) {
 				return <TanStackTable.Text>-</TanStackTable.Text>;
 			}
 			return (
-				<GroupedStatusCounts items={getPodPhaseStatusItems(row.podCountsByPhase)} />
+				<GroupedStatusCounts items={getPodStatusItems(row.podCountsByStatus)} />
 			);
 		},
 	},

frontend/src/container/InfraMonitoringK8sV2/DaemonSets/K8sDaemonSetsList.tsx

@@ -1,4 +1,4 @@
-import { useCallback } from 'react';
+import { useCallback, useMemo } from 'react';
 import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
 import { listDaemonSets } from 'api/generated/services/inframonitoring';
 import { RenderErrorResponseDTO } from 'api/generated/services/sigNoz.schemas';
@@ -10,6 +10,7 @@ import {
 } from 'api/generated/services/sigNoz.schemas';
 import { InfraMonitoringEvents } from 'constants/events';
 import APIError from 'types/api/error';
+
 import K8sBaseDetails, { K8sDetailsFilters } from '../Base/K8sBaseDetails';
 import { K8sBaseList } from '../Base/K8sBaseList';
 import { K8sBaseFilters } from '../Base/types';
@@ -18,6 +19,7 @@ import { SelectedItemParams } from '../hooks';
 import {
 	daemonSetWidgetInfo,
 	getDaemonSetMetricsQueryPayload,
+	getDaemonSetPodMetricsQueryPayload,
 	k8sDaemonSetDetailsMetadataConfig,
 	k8sDaemonSetGetEntityName,
 	k8sDaemonSetGetSelectedItemExpression,
@@ -29,6 +31,8 @@ import {
 	getK8sDaemonSetRowKey,
 	k8sDaemonSetsColumnsConfig,
 } from './table.config';
+import { createPodMetricsTab } from 'container/InfraMonitoringK8sV2/EntityDetailsUtils/createPodMetricsTab';
+
 function K8sDaemonSetsList({
 	controlListPrefix,
 }: {
@@ -112,6 +116,17 @@ function K8sDaemonSetsList({
 		},
 		[],
 	);
+	const customTabs = useMemo(
+		() => [
+			createPodMetricsTab<InframonitoringtypesDaemonSetRecordDTO>({
+				getQueryPayload: getDaemonSetPodMetricsQueryPayload,
+				category: InfraMonitoringEntity.DAEMONSETS,
+				queryKey: 'daemonSetPodMetrics',
+			}),
+		],
+		[],
+	);
+
 	return (
 		<>
 			<K8sBaseList<InframonitoringtypesDaemonSetRecordDTO, SelectedItemParams>
@@ -135,6 +150,7 @@ function K8sDaemonSetsList({
 				entityWidgetInfo={daemonSetWidgetInfo}
 				getEntityQueryPayload={getDaemonSetMetricsQueryPayload}
 				queryKeyPrefix="daemonset"
+				customTabs={customTabs}
 			/>
 		</>
 	);

frontend/src/container/InfraMonitoringK8sV2/DaemonSets/constants.ts

@@ -7,7 +7,10 @@ import { DataSource, ReduceOperators } from 'types/common/queryBuilder';
 import { v4 } from 'uuid';
 
 import { K8sDetailsMetadataConfig } from '../Base/K8sBaseDetails';
-import { INFRA_MONITORING_ATTR_KEYS } from '../constants';
+import {
+	getPodUtilizationByPodQueryPayloads,
+	INFRA_MONITORING_ATTR_KEYS,
+} from '../constants';
 import { SelectedItemParams } from '../hooks';
 import {
 	buildEventsExpression,
@@ -676,3 +679,29 @@ export const getDaemonSetMetricsQueryPayload = (
 		},
 	];
 };
+
+export const getDaemonSetPodMetricsQueryPayload = (
+	daemonSet: InframonitoringtypesDaemonSetRecordDTO,
+	start: number,
+	end: number,
+	dotMetricsEnabled: boolean,
+): GetQueryResultsProps[] => {
+	const k8sDaemonSetNameKey = dotMetricsEnabled
+		? 'k8s.daemonset.name'
+		: 'k8s_daemonset_name';
+
+	return getPodUtilizationByPodQueryPayloads(
+		{
+			workloadNameKey: k8sDaemonSetNameKey,
+			workloadNameValue:
+				daemonSet.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_DAEMONSET_NAME] ?? '',
+			clusterName:
+				daemonSet.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_CLUSTER_NAME] ?? '',
+			namespaceName:
+				daemonSet.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_NAMESPACE_NAME] ?? '',
+		},
+		start,
+		end,
+		dotMetricsEnabled,
+	);
+};

frontend/src/container/InfraMonitoringK8sV2/DaemonSets/table.config.tsx

@@ -7,7 +7,7 @@ import ColumnHeader from '../Base/ColumnHeader';
 import EntityGroupHeader from '../Base/EntityGroupHeader';
 import K8sGroupCell from '../Base/K8sGroupCell';
 import { SelectedItemParams } from '../hooks';
-import { formatBytes, getPodPhaseStatusItems } from '../commonUtils';
+import { formatBytes, getPodStatusItems } from '../commonUtils';
 import {
 	CellValueTooltip,
 	EntityProgressBar,
@@ -87,11 +87,7 @@ export const k8sDaemonSetsColumnsConfig: DaemonSetTableColumnConfig[] = [
 		visibilityBehavior: 'hidden-on-expand',
 		cell: ({ value }): React.ReactNode => {
 			const daemonsetName = value as string;
-			return (
-				<CellValueTooltip value={daemonsetName}>
-					<TanStackTable.Text>{daemonsetName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={daemonsetName} />;
 		},
 	},
 	{
@@ -108,35 +104,29 @@ export const k8sDaemonSetsColumnsConfig: DaemonSetTableColumnConfig[] = [
 		enableResize: true,
 		cell: ({ value }): React.ReactNode => {
 			const namespaceName = value as string;
-			return (
-				<CellValueTooltip value={namespaceName}>
-					<TanStackTable.Text>{namespaceName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={namespaceName} />;
 		},
 	},
 	{
-		id: 'pod_counts_by_phase',
+		id: 'pod_counts_by_status',
 		header: (): React.ReactNode => (
-			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/daemonsets#pod-counts-by-phase">
-				Pod Phases
+			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/daemonsets#pod-counts-by-status">
+				Pod Status
 			</ColumnHeader>
 		),
 		accessorFn: (
 			row,
-		): InframonitoringtypesDaemonSetRecordDTO['podCountsByPhase'] =>
-			row.podCountsByPhase,
+		): InframonitoringtypesDaemonSetRecordDTO['podCountsByStatus'] =>
+			row.podCountsByStatus,
 		width: { min: 250 },
 		enableSort: false,
 		enableResize: true,
 		cell: ({ row }): React.ReactNode => {
-			const podCountsByPhase = row.podCountsByPhase;
-			if (!podCountsByPhase) {
+			const podCountsByStatus = row.podCountsByStatus;
+			if (!podCountsByStatus) {
 				return <TanStackTable.Text>-</TanStackTable.Text>;
 			}
-			return (
-				<GroupedStatusCounts items={getPodPhaseStatusItems(podCountsByPhase)} />
-			);
+			return <GroupedStatusCounts items={getPodStatusItems(podCountsByStatus)} />;
 		},
 	},
 	{

frontend/src/container/InfraMonitoringK8sV2/Deployments/K8sDeploymentsList.tsx

@@ -1,4 +1,4 @@
-import { useCallback } from 'react';
+import { useCallback, useMemo } from 'react';
 import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
 import { listDeployments } from 'api/generated/services/inframonitoring';
 import { RenderErrorResponseDTO } from 'api/generated/services/sigNoz.schemas';
@@ -19,6 +19,7 @@ import { SelectedItemParams } from '../hooks';
 import {
 	deploymentWidgetInfo,
 	getDeploymentMetricsQueryPayload,
+	getDeploymentPodMetricsQueryPayload,
 	k8sDeploymentDetailsMetadataConfig,
 	k8sDeploymentGetEntityName,
 	k8sDeploymentGetSelectedItemExpression,
@@ -30,6 +31,7 @@ import {
 	getK8sDeploymentRowKey,
 	k8sDeploymentsColumnsConfig,
 } from './table.config';
+import { createPodMetricsTab } from 'container/InfraMonitoringK8sV2/EntityDetailsUtils/createPodMetricsTab';
 
 function K8sDeploymentsList({
 	controlListPrefix,
@@ -118,6 +120,17 @@ function K8sDeploymentsList({
 		[],
 	);
 
+	const customTabs = useMemo(
+		() => [
+			createPodMetricsTab<InframonitoringtypesDeploymentRecordDTO>({
+				getQueryPayload: getDeploymentPodMetricsQueryPayload,
+				category: InfraMonitoringEntity.DEPLOYMENTS,
+				queryKey: 'deploymentPodMetrics',
+			}),
+		],
+		[],
+	);
+
 	return (
 		<>
 			<K8sBaseList<InframonitoringtypesDeploymentRecordDTO, SelectedItemParams>
@@ -142,6 +155,7 @@ function K8sDeploymentsList({
 				entityWidgetInfo={deploymentWidgetInfo}
 				getEntityQueryPayload={getDeploymentMetricsQueryPayload}
 				queryKeyPrefix="deployment"
+				customTabs={customTabs}
 			/>
 		</>
 	);

frontend/src/container/InfraMonitoringK8sV2/Deployments/constants.ts

@@ -7,7 +7,10 @@ import { DataSource, ReduceOperators } from 'types/common/queryBuilder';
 import { v4 } from 'uuid';
 
 import { K8sDetailsMetadataConfig } from '../Base/K8sBaseDetails';
-import { INFRA_MONITORING_ATTR_KEYS } from '../constants';
+import {
+	getPodUtilizationByPodQueryPayloads,
+	INFRA_MONITORING_ATTR_KEYS,
+} from '../constants';
 import { SelectedItemParams } from '../hooks';
 import {
 	buildEventsExpression,
@@ -675,3 +678,29 @@ export const getDeploymentMetricsQueryPayload = (
 		},
 	];
 };
+
+export const getDeploymentPodMetricsQueryPayload = (
+	deployment: InframonitoringtypesDeploymentRecordDTO,
+	start: number,
+	end: number,
+	dotMetricsEnabled: boolean,
+): GetQueryResultsProps[] => {
+	const k8sDeploymentNameKey = dotMetricsEnabled
+		? 'k8s.deployment.name'
+		: 'k8s_deployment_name';
+
+	return getPodUtilizationByPodQueryPayloads(
+		{
+			workloadNameKey: k8sDeploymentNameKey,
+			workloadNameValue:
+				deployment.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_DEPLOYMENT_NAME] ?? '',
+			clusterName:
+				deployment.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_CLUSTER_NAME] ?? '',
+			namespaceName:
+				deployment.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_NAMESPACE_NAME] ?? '',
+		},
+		start,
+		end,
+		dotMetricsEnabled,
+	);
+};

frontend/src/container/InfraMonitoringK8sV2/Deployments/table.config.tsx

@@ -7,7 +7,7 @@ import ColumnHeader from '../Base/ColumnHeader';
 import EntityGroupHeader from '../Base/EntityGroupHeader';
 import K8sGroupCell from '../Base/K8sGroupCell';
 import { SelectedItemParams } from '../hooks';
-import { formatBytes, getPodPhaseStatusItems } from '../commonUtils';
+import { formatBytes, getPodStatusItems } from '../commonUtils';
 import {
 	CellValueTooltip,
 	EntityProgressBar,
@@ -88,11 +88,7 @@ export const k8sDeploymentsColumnsConfig: TableColumnDef<InframonitoringtypesDep
 			visibilityBehavior: 'hidden-on-expand',
 			cell: ({ value }): React.ReactNode => {
 				const deploymentName = value as string;
-				return (
-					<CellValueTooltip value={deploymentName}>
-						<TanStackTable.Text>{deploymentName}</TanStackTable.Text>
-					</CellValueTooltip>
-				);
+				return <CellValueTooltip value={deploymentName} />;
 			},
 		},
 		{
@@ -112,24 +108,22 @@ export const k8sDeploymentsColumnsConfig: TableColumnDef<InframonitoringtypesDep
 			),
 		},
 		{
-			id: 'podCountsByPhase',
+			id: 'podCountsByStatus',
 			header: (): React.ReactNode => (
-				<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/deployments#pod-counts-by-phase">
-					Pod Phases
+				<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/deployments#pod-counts-by-status">
+					Pod Status
 				</ColumnHeader>
 			),
-			accessorFn: (row): object | undefined => row.podCountsByPhase,
+			accessorFn: (row): object | undefined => row.podCountsByStatus,
 			width: { min: 250 },
 			enableSort: false,
 			enableResize: true,
 			cell: ({ row }): React.ReactNode => {
-				const podCountsByPhase = row.podCountsByPhase;
-				if (!podCountsByPhase) {
+				const podCountsByStatus = row.podCountsByStatus;
+				if (!podCountsByStatus) {
 					return <TanStackTable.Text>-</TanStackTable.Text>;
 				}
-				return (
-					<GroupedStatusCounts items={getPodPhaseStatusItems(podCountsByPhase)} />
-				);
+				return <GroupedStatusCounts items={getPodStatusItems(podCountsByStatus)} />;
 			},
 		},
 		{

frontend/src/container/InfraMonitoringK8sV2/EntityDetailsUtils/EntityTraces/__tests__/EntityTraces.table.test.tsx

@@ -87,7 +87,7 @@ describe('EntityTraces - Table Rendering', () => {
 		expect(badge).toHaveAttribute('data-variant', 'outline');
 	});
 
-	it('should render N/A when http method is empty', async () => {
+	it('should render - when http method is empty', async () => {
 		mockQueryRangeV5WithTracesResponse({
 			customTraces: [{ httpMethod: '', responseStatusCode: '200' }],
 		});
@@ -96,7 +96,7 @@ describe('EntityTraces - Table Rendering', () => {
 			renderEntityTraces();
 		});
 
-		await expect(screen.findByText('N/A')).resolves.toBeInTheDocument();
+		await expect(screen.findByText('-')).resolves.toBeInTheDocument();
 		expect(screen.queryByTestId('httpMethod')).not.toBeInTheDocument();
 	});
 

frontend/src/container/InfraMonitoringK8sV2/EntityDetailsUtils/EntityTraces/traceListColumns.module.scss

@@ -4,4 +4,8 @@
 
 .cellText {
 	color: var(--l2-foreground);
+
+	&[data-novalue='true'] {
+		opacity: 0.6;
+	}
 }

frontend/src/container/InfraMonitoringK8sV2/EntityDetailsUtils/EntityTraces/traceListColumns.tsx

@@ -89,8 +89,12 @@ export const getTraceListColumns = (
 				if (value === '') {
 					return (
 						<BlockLink to={getTraceLink(itemData)} openInNewTab>
-							<Typography data-testid={key} className={styles.cellText}>
-								N/A
+							<Typography
+								data-testid={key}
+								className={styles.cellText}
+								data-novalue="true"
+							>
+								-
 							</Typography>
 						</BlockLink>
 					);
@@ -102,7 +106,9 @@ export const getTraceListColumns = (
 					if (!httpMethod) {
 						return (
 							<BlockLink to={getTraceLink(itemData)} openInNewTab>
-								<Typography className={styles.cellText}>N/A</Typography>
+								<Typography className={styles.cellText} data-novalue="true">
+									-
+								</Typography>
 							</BlockLink>
 						);
 					}
@@ -129,8 +135,11 @@ export const getTraceListColumns = (
 					if (!isValidCode) {
 						return (
 							<BlockLink to={getTraceLink(itemData)} openInNewTab>
-								<Typography className={styles.cellText}>
-									{numericCode === 0 || !statusCode ? 'N/A' : statusCode}
+								<Typography
+									className={styles.cellText}
+									data-novalue={numericCode === 0 || !statusCode}
+								>
+									{numericCode === 0 || !statusCode ? '-' : statusCode}
 								</Typography>
 							</BlockLink>
 						);

frontend/src/container/InfraMonitoringK8sV2/EntityDetailsUtils/createPodMetricsTab.tsx

@@ -0,0 +1,47 @@
+import { Container } from '@signozhq/icons';
+import { GetQueryResultsProps } from 'lib/dashboard/getQueryResults';
+
+import { CustomTab } from '../Base/K8sBaseDetails';
+import {
+	InfraMonitoringEntity,
+	podUtilizationByPodWidgetInfo,
+	VIEW_TYPES,
+} from '../constants';
+
+import EntityMetrics from './EntityMetrics';
+
+interface CreatePodMetricsTabParams<T> {
+	getQueryPayload: (
+		entity: T,
+		start: number,
+		end: number,
+		dotMetricsEnabled: boolean,
+	) => GetQueryResultsProps[];
+	category: InfraMonitoringEntity;
+	queryKey: string;
+}
+
+export function createPodMetricsTab<T>({
+	getQueryPayload,
+	category,
+	queryKey,
+}: CreatePodMetricsTabParams<T>): CustomTab<T> {
+	return {
+		key: VIEW_TYPES.POD_METRICS,
+		label: 'Pod Metrics',
+		icon: <Container size={14} />,
+		render: ({ entity, timeRange, selectedInterval, handleTimeChange }) => (
+			<EntityMetrics
+				entity={entity}
+				selectedInterval={selectedInterval}
+				timeRange={timeRange}
+				handleTimeChange={handleTimeChange}
+				isModalTimeSelection
+				entityWidgetInfo={podUtilizationByPodWidgetInfo}
+				getEntityQueryPayload={getQueryPayload}
+				category={category}
+				queryKey={queryKey}
+			/>
+		),
+	};
+}

frontend/src/container/InfraMonitoringK8sV2/Jobs/K8sJobsList.tsx

@@ -1,4 +1,4 @@
-import { useCallback } from 'react';
+import { useCallback, useMemo } from 'react';
 import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
 import { listJobs } from 'api/generated/services/inframonitoring';
 import { RenderErrorResponseDTO } from 'api/generated/services/sigNoz.schemas';
@@ -18,6 +18,7 @@ import { InfraMonitoringEntity } from '../constants';
 import { SelectedItemParams } from '../hooks';
 import {
 	getJobMetricsQueryPayload,
+	getJobPodMetricsQueryPayload,
 	jobWidgetInfo,
 	k8sJobDetailsMetadataConfig,
 	k8sJobGetEntityName,
@@ -30,6 +31,7 @@ import {
 	getK8sJobRowKey,
 	k8sJobsColumnsConfig,
 } from './table.config';
+import { createPodMetricsTab } from 'container/InfraMonitoringK8sV2/EntityDetailsUtils/createPodMetricsTab';
 
 function K8sJobsList({
 	controlListPrefix,
@@ -118,6 +120,17 @@ function K8sJobsList({
 		[],
 	);
 
+	const customTabs = useMemo(
+		() => [
+			createPodMetricsTab<InframonitoringtypesJobRecordDTO>({
+				getQueryPayload: getJobPodMetricsQueryPayload,
+				category: InfraMonitoringEntity.JOBS,
+				queryKey: 'jobPodMetrics',
+			}),
+		],
+		[],
+	);
+
 	return (
 		<>
 			<K8sBaseList<InframonitoringtypesJobRecordDTO, SelectedItemParams>
@@ -142,6 +155,7 @@ function K8sJobsList({
 				entityWidgetInfo={jobWidgetInfo}
 				getEntityQueryPayload={getJobMetricsQueryPayload}
 				queryKeyPrefix="job"
+				customTabs={customTabs}
 			/>
 		</>
 	);

frontend/src/container/InfraMonitoringK8sV2/Jobs/constants.ts

@@ -7,7 +7,10 @@ import { DataSource, ReduceOperators } from 'types/common/queryBuilder';
 import { v4 } from 'uuid';
 
 import { K8sDetailsMetadataConfig } from '../Base/K8sBaseDetails';
-import { INFRA_MONITORING_ATTR_KEYS } from '../constants';
+import {
+	getPodUtilizationByPodQueryPayloads,
+	INFRA_MONITORING_ATTR_KEYS,
+} from '../constants';
 import { SelectedItemParams } from '../hooks';
 import {
 	buildEventsExpression,
@@ -429,3 +432,25 @@ export const getJobMetricsQueryPayload = (
 		},
 	];
 };
+
+export const getJobPodMetricsQueryPayload = (
+	job: InframonitoringtypesJobRecordDTO,
+	start: number,
+	end: number,
+	dotMetricsEnabled: boolean,
+): GetQueryResultsProps[] => {
+	const k8sJobNameKey = dotMetricsEnabled ? 'k8s.job.name' : 'k8s_job_name';
+
+	return getPodUtilizationByPodQueryPayloads(
+		{
+			workloadNameKey: k8sJobNameKey,
+			workloadNameValue: job.jobName ?? '',
+			clusterName: job.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_CLUSTER_NAME] ?? '',
+			namespaceName:
+				job.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_NAMESPACE_NAME] ?? '',
+		},
+		start,
+		end,
+		dotMetricsEnabled,
+	);
+};

frontend/src/container/InfraMonitoringK8sV2/Jobs/table.config.tsx

@@ -7,7 +7,7 @@ import ColumnHeader from '../Base/ColumnHeader';
 import EntityGroupHeader from '../Base/EntityGroupHeader';
 import K8sGroupCell from '../Base/K8sGroupCell';
 import { SelectedItemParams } from '../hooks';
-import { formatBytes, getPodPhaseStatusItems } from '../commonUtils';
+import { formatBytes, getPodStatusItems } from '../commonUtils';
 import {
 	CellValueTooltip,
 	EntityProgressBar,
@@ -81,11 +81,7 @@ export const k8sJobsColumnsConfig: JobTableColumnConfig[] = [
 		visibilityBehavior: 'hidden-on-expand',
 		cell: ({ value }): React.ReactNode => {
 			const jobName = value as string;
-			return (
-				<CellValueTooltip value={jobName}>
-					<TanStackTable.Text>{jobName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={jobName} />;
 		},
 	},
 	{
@@ -102,33 +98,27 @@ export const k8sJobsColumnsConfig: JobTableColumnConfig[] = [
 		enableResize: true,
 		cell: ({ value }): React.ReactNode => {
 			const namespaceName = value as string;
-			return (
-				<CellValueTooltip value={namespaceName}>
-					<TanStackTable.Text>{namespaceName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={namespaceName} />;
 		},
 	},
 	{
-		id: 'pod_counts_by_phase',
+		id: 'pod_counts_by_status',
 		header: (): React.ReactNode => (
-			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/jobs#pod-counts-by-phase">
-				Pod Phases
+			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/jobs#pod-counts-by-status">
+				Pod Status
 			</ColumnHeader>
 		),
-		accessorFn: (row): InframonitoringtypesJobRecordDTO['podCountsByPhase'] =>
-			row.podCountsByPhase,
+		accessorFn: (row): InframonitoringtypesJobRecordDTO['podCountsByStatus'] =>
+			row.podCountsByStatus,
 		width: { min: 250 },
 		enableSort: false,
 		enableResize: true,
 		cell: ({ row }): React.ReactNode => {
-			const podCountsByPhase = row.podCountsByPhase;
-			if (!podCountsByPhase) {
+			const podCountsByStatus = row.podCountsByStatus;
+			if (!podCountsByStatus) {
 				return <TanStackTable.Text>-</TanStackTable.Text>;
 			}
-			return (
-				<GroupedStatusCounts items={getPodPhaseStatusItems(podCountsByPhase)} />
-			);
+			return <GroupedStatusCounts items={getPodStatusItems(podCountsByStatus)} />;
 		},
 	},
 	{

frontend/src/container/InfraMonitoringK8sV2/Namespaces/K8sNamespacesList.tsx

@@ -1,4 +1,4 @@
-import { useCallback } from 'react';
+import { useCallback, useMemo } from 'react';
 import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
 import { listNamespaces } from 'api/generated/services/inframonitoring';
 import { RenderErrorResponseDTO } from 'api/generated/services/sigNoz.schemas';
@@ -18,6 +18,7 @@ import { InfraMonitoringEntity } from '../constants';
 import { SelectedItemParams } from '../hooks';
 import {
 	getNamespaceMetricsQueryPayload,
+	getNamespacePodMetricsQueryPayload,
 	k8sNamespaceDetailsCountsConfig,
 	k8sNamespaceDetailsMetadataConfig,
 	k8sNamespaceGetCountsFilterExpression,
@@ -32,6 +33,7 @@ import {
 	getK8sNamespaceRowKey,
 	k8sNamespacesColumnsConfig,
 } from './table.config';
+import { createPodMetricsTab } from 'container/InfraMonitoringK8sV2/EntityDetailsUtils/createPodMetricsTab';
 
 function K8sNamespacesList({
 	controlListPrefix,
@@ -120,6 +122,17 @@ function K8sNamespacesList({
 		[],
 	);
 
+	const customTabs = useMemo(
+		() => [
+			createPodMetricsTab<InframonitoringtypesNamespaceRecordDTO>({
+				getQueryPayload: getNamespacePodMetricsQueryPayload,
+				category: InfraMonitoringEntity.NAMESPACES,
+				queryKey: 'namespacePodMetrics',
+			}),
+		],
+		[],
+	);
+
 	return (
 		<>
 			<K8sBaseList<InframonitoringtypesNamespaceRecordDTO, SelectedItemParams>
@@ -146,6 +159,7 @@ function K8sNamespacesList({
 				entityWidgetInfo={namespaceWidgetInfo}
 				getEntityQueryPayload={getNamespaceMetricsQueryPayload}
 				queryKeyPrefix="namespace"
+				customTabs={customTabs}
 			/>
 		</>
 	);

frontend/src/container/InfraMonitoringK8sV2/Namespaces/constants.ts

@@ -11,6 +11,7 @@ import {
 	K8sDetailsMetadataConfig,
 } from '../Base/K8sBaseDetails';
 import {
+	getPodUtilizationByPodQueryPayloads,
 	INFRA_MONITORING_ATTR_KEYS,
 	InfraMonitoringEntity,
 } from '../constants';
@@ -1752,3 +1753,26 @@ export const getNamespaceMetricsQueryPayload = (
 		},
 	];
 };
+
+export const getNamespacePodMetricsQueryPayload = (
+	namespace: InframonitoringtypesNamespaceRecordDTO,
+	start: number,
+	end: number,
+	dotMetricsEnabled: boolean,
+): GetQueryResultsProps[] => {
+	const k8sNamespaceNameKey = dotMetricsEnabled
+		? 'k8s.namespace.name'
+		: 'k8s_namespace_name';
+
+	return getPodUtilizationByPodQueryPayloads(
+		{
+			workloadNameKey: k8sNamespaceNameKey,
+			workloadNameValue: namespace.namespaceName ?? '',
+			clusterName:
+				namespace.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_CLUSTER_NAME] ?? '',
+		},
+		start,
+		end,
+		dotMetricsEnabled,
+	);
+};

frontend/src/container/InfraMonitoringK8sV2/Namespaces/table.config.tsx

@@ -6,7 +6,7 @@ import { ExpandButtonWrapper } from 'container/InfraMonitoringK8sV2/components';
 import ColumnHeader from '../Base/ColumnHeader';
 import EntityGroupHeader from '../Base/EntityGroupHeader';
 import K8sGroupCell from '../Base/K8sGroupCell';
-import { formatBytes, getPodPhaseStatusItems } from '../commonUtils';
+import { formatBytes, getPodStatusItems } from '../commonUtils';
 import {
 	CellValueTooltip,
 	GroupedStatusCounts,
@@ -83,11 +83,7 @@ export const k8sNamespacesColumnsConfig: NamespaceTableColumnConfig[] = [
 		visibilityBehavior: 'hidden-on-expand',
 		cell: ({ value }): React.ReactNode => {
 			const namespaceName = value as string;
-			return (
-				<CellValueTooltip value={namespaceName}>
-					<TanStackTable.Text>{namespaceName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={namespaceName} />;
 		},
 	},
 	{
@@ -106,25 +102,25 @@ export const k8sNamespacesColumnsConfig: NamespaceTableColumnConfig[] = [
 		),
 	},
 	{
-		id: 'podCountsByPhase',
+		id: 'podCountsByStatus',
 		header: (): React.ReactNode => (
-			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/namespaces#pod-counts-by-phase">
-				Pod Phases
+			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/namespaces#pod-counts-by-status">
+				Pod Status
 			</ColumnHeader>
 		),
 		accessorFn: (
 			row,
-		): InframonitoringtypesNamespaceRecordDTO['podCountsByPhase'] =>
-			row.podCountsByPhase,
+		): InframonitoringtypesNamespaceRecordDTO['podCountsByStatus'] =>
+			row.podCountsByStatus,
 		width: { min: 250 },
 		enableSort: false,
 		cell: ({ row }): React.ReactNode => {
-			const podCountsByPhase = row.podCountsByPhase;
-			if (!podCountsByPhase) {
+			const podCountsByStatus = row.podCountsByStatus;
+			if (!podCountsByStatus) {
 				return <TanStackTable.Text>-</TanStackTable.Text>;
 			}
 			return (
-				<GroupedStatusCounts items={getPodPhaseStatusItems(row.podCountsByPhase)} />
+				<GroupedStatusCounts items={getPodStatusItems(row.podCountsByStatus)} />
 			);
 		},
 	},

frontend/src/container/InfraMonitoringK8sV2/Nodes/table.config.tsx

@@ -7,7 +7,7 @@ import { ExpandButtonWrapper } from 'container/InfraMonitoringK8sV2/components';
 import ColumnHeader from '../Base/ColumnHeader';
 import EntityGroupHeader from '../Base/EntityGroupHeader';
 import K8sGroupCell from '../Base/K8sGroupCell';
-import { formatBytes, getPodPhaseStatusItems } from '../commonUtils';
+import { formatBytes, getPodStatusItems } from '../commonUtils';
 import { INFRA_MONITORING_ATTR_KEYS } from '../constants';
 import {
 	CellValueTooltip,
@@ -85,11 +85,7 @@ export const k8sNodesColumnsConfig: NodeTableColumnConfig[] = [
 		visibilityBehavior: 'hidden-on-expand',
 		cell: ({ value }): React.ReactNode => {
 			const nodeName = value as string;
-			return (
-				<CellValueTooltip value={nodeName}>
-					<TanStackTable.Text>{nodeName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={nodeName} />;
 		},
 	},
 	{
@@ -132,23 +128,23 @@ export const k8sNodesColumnsConfig: NodeTableColumnConfig[] = [
 		},
 	},
 	{
-		id: 'podCountsByPhase',
+		id: 'podCountsByStatus',
 		header: (): React.ReactNode => (
-			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/nodes#pod-counts-by-phase">
-				Pod Phases
+			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/nodes#pod-counts-by-status">
+				Pod Status
 			</ColumnHeader>
 		),
-		accessorFn: (row): InframonitoringtypesNodeRecordDTO['podCountsByPhase'] =>
-			row.podCountsByPhase,
+		accessorFn: (row): InframonitoringtypesNodeRecordDTO['podCountsByStatus'] =>
+			row.podCountsByStatus,
 		width: { min: 250 },
 		enableSort: false,
 		cell: ({ row }): React.ReactNode => {
-			const podCountsByPhase = row.podCountsByPhase;
-			if (!podCountsByPhase) {
+			const podCountsByStatus = row.podCountsByStatus;
+			if (!podCountsByStatus) {
 				return <TanStackTable.Text>-</TanStackTable.Text>;
 			}
 			return (
-				<GroupedStatusCounts items={getPodPhaseStatusItems(row.podCountsByPhase)} />
+				<GroupedStatusCounts items={getPodStatusItems(row.podCountsByStatus)} />
 			);
 		},
 	},

frontend/src/container/InfraMonitoringK8sV2/Pods/table.config.tsx

@@ -1,9 +1,9 @@
 import { Container } from '@signozhq/icons';
-import { Badge, BadgeColor } from '@signozhq/ui/badge';
+import { Badge } from '@signozhq/ui/badge';
 import { TooltipSimple } from '@signozhq/ui/tooltip';
 import {
-	InframonitoringtypesPodPhaseDTO,
 	InframonitoringtypesPodRecordDTO,
+	InframonitoringtypesPodStatusDTO,
 } from 'api/generated/services/sigNoz.schemas';
 import TanStackTable, { TableColumnDef } from 'components/TanStackTableView';
 import { ExpandButtonWrapper } from 'container/InfraMonitoringK8sV2/components';
@@ -11,7 +11,11 @@ import { ExpandButtonWrapper } from 'container/InfraMonitoringK8sV2/components';
 import ColumnHeader from '../Base/ColumnHeader';
 import EntityGroupHeader from '../Base/EntityGroupHeader';
 import K8sGroupCell from '../Base/K8sGroupCell';
-import { formatBytes, getPodPhaseStatusItems } from '../commonUtils';
+import {
+	formatBytes,
+	getPodStatusItems,
+	POD_STATUS_COLORS,
+} from '../commonUtils';
 import {
 	CellValueTooltip,
 	EntityProgressBar,
@@ -40,15 +44,6 @@ export function getK8sPodItemKey(
 	return pod.podUID;
 }
 
-const POD_PHASE_COLORS: Record<string, BadgeColor> = {
-	running: 'forest',
-	pending: 'amber',
-	succeeded: 'robin',
-	failed: 'cherry',
-	unknown: 'vanilla',
-	no_data: 'vanilla',
-};
-
 export type PodTableColumnConfig =
 	TableColumnDef<InframonitoringtypesPodRecordDTO>;
 export const k8sPodColumnsConfig: PodTableColumnConfig[] = [
@@ -93,34 +88,30 @@ export const k8sPodColumnsConfig: PodTableColumnConfig[] = [
 		visibilityBehavior: 'hidden-on-expand',
 		cell: ({ value }): React.ReactNode => {
 			const podName = value as string;
-			return (
-				<CellValueTooltip value={podName}>
-					<TanStackTable.Text>{podName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={podName} />;
 		},
 	},
 	{
-		id: 'podPhase',
+		id: 'podStatus',
 		header: (): React.ReactNode => (
-			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/pods#pod-phase">
-				Phase
+			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/pods#pod-status">
+				Status
 			</ColumnHeader>
 		),
-		accessorFn: (row): string => row.podPhase,
-		width: { min: 120 },
+		accessorFn: (row): string => row.podStatus,
+		width: { min: 160 },
 		enableSort: false,
 		visibilityBehavior: 'hidden-on-expand',
 		cell: ({ row }): React.ReactNode => {
-			if (!row.podPhase) {
+			if (!row.podStatus) {
 				return <></>;
 			}
 
-			const color = POD_PHASE_COLORS[row.podPhase] || POD_PHASE_COLORS.unknown;
+			const color = POD_STATUS_COLORS[row.podStatus] || POD_STATUS_COLORS.unknown;
 			const label =
-				row.podPhase === InframonitoringtypesPodPhaseDTO.no_data
+				row.podStatus === InframonitoringtypesPodStatusDTO.no_data
 					? 'No Data'
-					: row.podPhase.charAt(0).toUpperCase() + row.podPhase.slice(1);
+					: row.podStatus.charAt(0).toUpperCase() + row.podStatus.slice(1);
 			return (
 				<Badge color={color} variant="outline">
 					{label}
@@ -129,24 +120,24 @@ export const k8sPodColumnsConfig: PodTableColumnConfig[] = [
 		},
 	},
 	{
-		id: 'podCountsByPhase',
+		id: 'podCountsByStatus',
 		header: (): React.ReactNode => (
-			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/pods#pod-phase">
-				Phases
+			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/pods#pod-status">
+				Status
 			</ColumnHeader>
 		),
-		accessorFn: (row): InframonitoringtypesPodRecordDTO['podCountsByPhase'] =>
-			row.podCountsByPhase,
+		accessorFn: (row): InframonitoringtypesPodRecordDTO['podCountsByStatus'] =>
+			row.podCountsByStatus,
 		width: { min: 250 },
 		enableSort: false,
 		visibilityBehavior: 'hidden-on-collapse',
 		cell: ({ row }): React.ReactNode => {
-			const podCountsByPhase = row.podCountsByPhase;
-			if (!podCountsByPhase) {
+			const podCountsByStatus = row.podCountsByStatus;
+			if (!podCountsByStatus) {
 				return <TanStackTable.Text>-</TanStackTable.Text>;
 			}
 			return (
-				<GroupedStatusCounts items={getPodPhaseStatusItems(row.podCountsByPhase)} />
+				<GroupedStatusCounts items={getPodStatusItems(row.podCountsByStatus)} />
 			);
 		},
 	},
@@ -172,6 +163,28 @@ export const k8sPodColumnsConfig: PodTableColumnConfig[] = [
 			return <TanStackTable.Text>{formatAge(age)}</TanStackTable.Text>;
 		},
 	},
+	{
+		id: 'podRestarts',
+		header: (): React.ReactNode => (
+			<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/pods#restarts">
+				Restarts
+			</ColumnHeader>
+		),
+		accessorFn: (row): number => row.podRestarts,
+		width: { min: 100 },
+		enableSort: true,
+		cell: ({ value }): React.ReactNode => {
+			const restarts = value as number;
+			if (restarts === -1) {
+				return (
+					<TooltipSimple title="No data">
+						<Typography.Text>-</Typography.Text>
+					</TooltipSimple>
+				);
+			}
+			return <TanStackTable.Text>{restarts}</TanStackTable.Text>;
+		},
+	},
 	{
 		id: 'cpu_request',
 		header: (): React.ReactNode => (

frontend/src/container/InfraMonitoringK8sV2/StatefulSets/K8sStatefulSetsList.tsx

@@ -1,4 +1,4 @@
-import { useCallback } from 'react';
+import { useCallback, useMemo } from 'react';
 import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
 import { listStatefulSets } from 'api/generated/services/inframonitoring';
 import { RenderErrorResponseDTO } from 'api/generated/services/sigNoz.schemas';
@@ -18,6 +18,7 @@ import { InfraMonitoringEntity } from '../constants';
 import { SelectedItemParams } from '../hooks';
 import {
 	getStatefulSetMetricsQueryPayload,
+	getStatefulSetPodMetricsQueryPayload,
 	k8sStatefulSetDetailsMetadataConfig,
 	k8sStatefulSetGetEntityName,
 	k8sStatefulSetGetSelectedItemExpression,
@@ -30,6 +31,7 @@ import {
 	getK8sStatefulSetRowKey,
 	k8sStatefulSetsColumnsConfig,
 } from './table.config';
+import { createPodMetricsTab } from 'container/InfraMonitoringK8sV2/EntityDetailsUtils/createPodMetricsTab';
 
 function K8sStatefulSetsList({
 	controlListPrefix,
@@ -118,6 +120,17 @@ function K8sStatefulSetsList({
 		[],
 	);
 
+	const customTabs = useMemo(
+		() => [
+			createPodMetricsTab<InframonitoringtypesStatefulSetRecordDTO>({
+				getQueryPayload: getStatefulSetPodMetricsQueryPayload,
+				category: InfraMonitoringEntity.STATEFULSETS,
+				queryKey: 'statefulSetPodMetrics',
+			}),
+		],
+		[],
+	);
+
 	return (
 		<>
 			<K8sBaseList<InframonitoringtypesStatefulSetRecordDTO, SelectedItemParams>
@@ -142,6 +155,7 @@ function K8sStatefulSetsList({
 				entityWidgetInfo={statefulSetWidgetInfo}
 				getEntityQueryPayload={getStatefulSetMetricsQueryPayload}
 				queryKeyPrefix="statefulSet"
+				customTabs={customTabs}
 			/>
 		</>
 	);

frontend/src/container/InfraMonitoringK8sV2/StatefulSets/constants.ts

@@ -7,7 +7,10 @@ import { DataSource, ReduceOperators } from 'types/common/queryBuilder';
 import { v4 } from 'uuid';
 
 import { K8sDetailsMetadataConfig } from '../Base/K8sBaseDetails';
-import { INFRA_MONITORING_ATTR_KEYS } from '../constants';
+import {
+	getPodUtilizationByPodQueryPayloads,
+	INFRA_MONITORING_ATTR_KEYS,
+} from '../constants';
 import { SelectedItemParams } from '../hooks';
 import {
 	buildEventsExpression,
@@ -859,3 +862,29 @@ export const getStatefulSetMetricsQueryPayload = (
 		},
 	];
 };
+
+export const getStatefulSetPodMetricsQueryPayload = (
+	statefulSet: InframonitoringtypesStatefulSetRecordDTO,
+	start: number,
+	end: number,
+	dotMetricsEnabled: boolean,
+): GetQueryResultsProps[] => {
+	const k8sStatefulSetNameKey = dotMetricsEnabled
+		? INFRA_MONITORING_ATTR_KEYS.K8S_STATEFULSET_NAME
+		: 'k8s_statefulset_name';
+
+	return getPodUtilizationByPodQueryPayloads(
+		{
+			workloadNameKey: k8sStatefulSetNameKey,
+			workloadNameValue:
+				statefulSet.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_STATEFULSET_NAME] ?? '',
+			clusterName:
+				statefulSet.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_CLUSTER_NAME] ?? '',
+			namespaceName:
+				statefulSet.meta?.[INFRA_MONITORING_ATTR_KEYS.K8S_NAMESPACE_NAME] ?? '',
+		},
+		start,
+		end,
+		dotMetricsEnabled,
+	);
+};

frontend/src/container/InfraMonitoringK8sV2/StatefulSets/table.config.tsx

@@ -7,7 +7,7 @@ import ColumnHeader from '../Base/ColumnHeader';
 import EntityGroupHeader from '../Base/EntityGroupHeader';
 import K8sGroupCell from '../Base/K8sGroupCell';
 import { SelectedItemParams } from '../hooks';
-import { formatBytes, getPodPhaseStatusItems } from '../commonUtils';
+import { formatBytes, getPodStatusItems } from '../commonUtils';
 import {
 	CellValueTooltip,
 	EntityProgressBar,
@@ -88,11 +88,7 @@ export const k8sStatefulSetsColumnsConfig: TableColumnDef<InframonitoringtypesSt
 			visibilityBehavior: 'hidden-on-expand',
 			cell: ({ value }): React.ReactNode => {
 				const statefulsetName = value as string;
-				return (
-					<CellValueTooltip value={statefulsetName}>
-						<TanStackTable.Text>{statefulsetName}</TanStackTable.Text>
-					</CellValueTooltip>
-				);
+				return <CellValueTooltip value={statefulsetName} />;
 			},
 		},
 		{
@@ -109,35 +105,29 @@ export const k8sStatefulSetsColumnsConfig: TableColumnDef<InframonitoringtypesSt
 			enableResize: true,
 			cell: ({ value }): React.ReactNode => {
 				const namespaceName = value as string;
-				return (
-					<CellValueTooltip value={namespaceName}>
-						<TanStackTable.Text>{namespaceName}</TanStackTable.Text>
-					</CellValueTooltip>
-				);
+				return <CellValueTooltip value={namespaceName} />;
 			},
 		},
 		{
-			id: 'pod_counts_by_phase',
+			id: 'pod_counts_by_status',
 			header: (): React.ReactNode => (
-				<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/statefulsets#pod-counts-by-phase">
-					Pod Phases
+				<ColumnHeader docPath="/infrastructure-monitoring/kubernetes/statefulsets#pod-counts-by-status">
+					Pod Status
 				</ColumnHeader>
 			),
 			accessorFn: (
 				row,
-			): InframonitoringtypesStatefulSetRecordDTO['podCountsByPhase'] =>
-				row.podCountsByPhase,
+			): InframonitoringtypesStatefulSetRecordDTO['podCountsByStatus'] =>
+				row.podCountsByStatus,
 			width: { min: 250 },
 			enableSort: false,
 			enableResize: true,
 			cell: ({ row }): React.ReactNode => {
-				const podCountsByPhase = row.podCountsByPhase;
-				if (!podCountsByPhase) {
+				const podCountsByStatus = row.podCountsByStatus;
+				if (!podCountsByStatus) {
 					return <TanStackTable.Text>-</TanStackTable.Text>;
 				}
-				return (
-					<GroupedStatusCounts items={getPodPhaseStatusItems(podCountsByPhase)} />
-				);
+				return <GroupedStatusCounts items={getPodStatusItems(podCountsByStatus)} />;
 			},
 		},
 		{

frontend/src/container/InfraMonitoringK8sV2/Volumes/table.config.tsx

@@ -81,11 +81,7 @@ export const k8sVolumesColumnsConfig: VolumeTableColumnConfig[] = [
 		visibilityBehavior: 'hidden-on-expand',
 		cell: ({ value }): React.ReactNode => {
 			const pvcName = value as string;
-			return (
-				<CellValueTooltip value={pvcName}>
-					<TanStackTable.Text>{pvcName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={pvcName} />;
 		},
 	},
 	{
@@ -101,11 +97,7 @@ export const k8sVolumesColumnsConfig: VolumeTableColumnConfig[] = [
 		enableSort: false,
 		cell: ({ value }): React.ReactNode => {
 			const namespaceName = value as string;
-			return (
-				<CellValueTooltip value={namespaceName}>
-					<TanStackTable.Text>{namespaceName}</TanStackTable.Text>
-				</CellValueTooltip>
-			);
+			return <CellValueTooltip value={namespaceName} />;
 		},
 	},
 	{

frontend/src/container/InfraMonitoringK8sV2/commonUtils.tsx

@@ -1,5 +1,9 @@
 import { Color } from '@signozhq/design-tokens';
-import { InframonitoringtypesPodCountsByPhaseDTO } from 'api/generated/services/sigNoz.schemas';
+import { BadgeColor } from '@signozhq/ui/badge';
+import {
+	InframonitoringtypesPodCountsByStatusDTO,
+	InframonitoringtypesPodStatusDTO,
+} from 'api/generated/services/sigNoz.schemas';
 
 import { StatusCountItem } from './components/GroupedStatusCounts';
 
@@ -64,17 +68,106 @@ export function getStrokeColorForLimitUtilization(value: number): string {
 	return Color.BG_SAKURA_500;
 }
 
-/**
- * Builds StatusCountItem[] for GroupedStatusCounts from pod phase counts.
- */
-export function getPodPhaseStatusItems(
-	counts: InframonitoringtypesPodCountsByPhaseDTO,
+export const POD_STATUS_COLORS: Record<
+	InframonitoringtypesPodStatusDTO,
+	BadgeColor
+> = {
+	[InframonitoringtypesPodStatusDTO.running]: 'forest',
+	[InframonitoringtypesPodStatusDTO.completed]: 'robin',
+	[InframonitoringtypesPodStatusDTO.pending]: 'amber',
+	[InframonitoringtypesPodStatusDTO.unknown]: 'vanilla',
+	[InframonitoringtypesPodStatusDTO.no_data]: 'vanilla',
+	[InframonitoringtypesPodStatusDTO.failed]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.crashloopbackoff]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.imagepullbackoff]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.errimagepull]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.createcontainerconfigerror]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.containercreating]: 'amber',
+	[InframonitoringtypesPodStatusDTO.oomkilled]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.error]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.containercannotrun]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.evicted]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.nodeaffinity]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.nodelost]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.shutdown]: 'cherry',
+	[InframonitoringtypesPodStatusDTO.unexpectedadmissionerror]: 'cherry',
+};
+
+type PodStatusCategory =
+	| 'running'
+	| 'completed'
+	| 'pending'
+	| 'unknown'
+	| 'error';
+
+const POD_STATUS_CATEGORY_MAP: Record<
+	keyof InframonitoringtypesPodCountsByStatusDTO,
+	PodStatusCategory
+> = {
+	running: 'running',
+	completed: 'completed',
+	pending: 'pending',
+	unknown: 'unknown',
+	failed: 'error',
+	crashLoopBackOff: 'error',
+	imagePullBackOff: 'error',
+	errImagePull: 'error',
+	createContainerConfigError: 'error',
+	containerCreating: 'error',
+	oomKilled: 'error',
+	error: 'error',
+	containerCannotRun: 'error',
+	evicted: 'error',
+	nodeAffinity: 'error',
+	nodeLost: 'error',
+	shutdown: 'error',
+	unexpectedAdmissionError: 'error',
+};
+
+type ErrorStatusKey = {
+	[K in keyof InframonitoringtypesPodCountsByStatusDTO]: (typeof POD_STATUS_CATEGORY_MAP)[K] extends 'error'
+		? K
+		: never;
+}[keyof InframonitoringtypesPodCountsByStatusDTO];
+
+const ERROR_STATUS_LABELS: Record<ErrorStatusKey, string> = {
+	failed: 'Failed',
+	crashLoopBackOff: 'CrashLoopBackOff',
+	imagePullBackOff: 'ImagePullBackOff',
+	errImagePull: 'ErrImagePull',
+	createContainerConfigError: 'CreateContainerConfigError',
+	containerCreating: 'ContainerCreating',
+	oomKilled: 'OOMKilled',
+	error: 'Error',
+	containerCannotRun: 'ContainerCannotRun',
+	evicted: 'Evicted',
+	nodeAffinity: 'NodeAffinity',
+	nodeLost: 'NodeLost',
+	shutdown: 'Shutdown',
+	unexpectedAdmissionError: 'UnexpectedAdmissionError',
+};
+
+export function getPodStatusItems(
+	counts: InframonitoringtypesPodCountsByStatusDTO,
 ): StatusCountItem[] {
+	const errorKeys = Object.keys(ERROR_STATUS_LABELS) as ErrorStatusKey[];
+
+	const errorTotal = errorKeys.reduce((sum, key) => sum + counts[key], 0);
+	const errorBreakdown = errorKeys.map((key) => ({
+		label: ERROR_STATUS_LABELS[key],
+		value: counts[key],
+	}));
+
 	return [
 		{ value: counts.running, label: 'Running', color: Color.BG_FOREST_500 },
+		{ value: counts.completed, label: 'Completed', color: Color.BG_ROBIN_500 },
 		{ value: counts.pending, label: 'Pending', color: Color.BG_AMBER_500 },
-		{ value: counts.succeeded, label: 'Succeeded', color: Color.BG_ROBIN_500 },
-		{ value: counts.failed, label: 'Failed', color: Color.BG_CHERRY_500 },
 		{ value: counts.unknown, label: 'Unknown', color: Color.BG_SLATE_400 },
+		{
+			value: errorTotal,
+			label: 'Error Status',
+			color: Color.BG_CHERRY_500,
+			breakdown: errorBreakdown,
+		},
 	];
 }

frontend/src/container/InfraMonitoringK8sV2/components/CellValueTooltip/CellValueTooltip.module.scss

@@ -52,3 +52,7 @@
 .divider {
 	--divider-color: rgba(255, 255, 255, 0.14);
 }
+
+.value {
+	width: fit-content;
+}

frontend/src/container/InfraMonitoringK8sV2/components/CellValueTooltip/CellValueTooltip.tsx

@@ -1,8 +1,9 @@
-import { useCallback, type ReactNode, type MouseEvent } from 'react';
+import { useCallback, type MouseEvent } from 'react';
 import { TooltipSimple } from '@signozhq/ui/tooltip';
 import { toast } from '@signozhq/ui/sonner';
 import { Copy, Minus, Plus } from '@signozhq/icons';
 import { useCopyToClipboard } from 'react-use';
+import TanStackTable from 'components/TanStackTableView';
 
 import { useInfraMonitoringCellActionsStore } from './useInfraMonitoringCellActionsStore';
 
@@ -11,12 +12,10 @@ import { Divider } from '@signozhq/ui/divider';
 
 export interface CellValueTooltipProps {
 	value: string;
-	children: ReactNode;
 }
 
 export function CellValueTooltip({
 	value,
-	children,
 }: CellValueTooltipProps): JSX.Element {
 	const [, copyToClipboard] = useCopyToClipboard();
 	const { lineClamp, increaseLineClamp, decreaseLineClamp } =
@@ -94,7 +93,7 @@ export function CellValueTooltip({
 				className: styles.tooltipContentWrapper,
 			}}
 		>
-			{children}
+			<TanStackTable.Text className={styles.value}>{value}</TanStackTable.Text>
 		</TooltipSimple>
 	);
 }

frontend/src/container/InfraMonitoringK8sV2/components/GroupedStatusCounts.module.scss

@@ -17,9 +17,40 @@
 	flex-shrink: 0;
 }
 
+.valueWrapper {
+	min-width: 4ch;
+}
+
+.valueWrapperTooltip {
+	display: block;
+	width: fit-content;
+}
+
 .value {
 	font-variant-numeric: tabular-nums;
-	min-width: 4ch;
 	text-align: left;
 	cursor: default;
+	min-width: min-content;
+}
+
+.tooltipContent {
+	display: flex;
+	flex-direction: column;
+	gap: 4px;
+	min-width: 120px;
+}
+
+.tooltipHeader {
+	font-weight: 600;
+	margin-bottom: 2px;
+}
+
+.tooltipRow {
+	display: flex;
+	justify-content: space-between;
+	gap: 12px;
+}
+
+.tooltipValue {
+	font-variant-numeric: tabular-nums;
 }

frontend/src/container/InfraMonitoringK8sV2/components/GroupedStatusCounts.tsx

@@ -2,11 +2,18 @@ import { TooltipSimple } from '@signozhq/ui/tooltip';
 
 import styles from './GroupedStatusCounts.module.scss';
 import TanStackTable from 'components/TanStackTableView';
+import { Typography } from '@signozhq/ui/typography';
+
+export interface StatusBreakdownItem {
+	label: string;
+	value: number;
+}
 
 export interface StatusCountItem {
 	value: number;
 	label: string;
 	color: string;
+	breakdown?: StatusBreakdownItem[];
 }
 
 interface GroupedStatusCountsProps {
@@ -14,6 +21,45 @@ interface GroupedStatusCountsProps {
 	showZeroValues?: boolean;
 }
 
+function buildTooltipContent(item: StatusCountItem): React.ReactNode {
+	if (!item.breakdown || item.breakdown.length === 0) {
+		return (
+			<Typography.Text>
+				{item.label}: {item.value}
+			</Typography.Text>
+		);
+	}
+
+	const nonZeroBreakdown = item.breakdown.filter((b) => b.value > 0);
+	if (nonZeroBreakdown.length === 0) {
+		return (
+			<div className={styles.tooltipContent}>
+				<Typography.Text className={styles.tooltipHeader}>
+					{item.label}
+				</Typography.Text>
+
+				<Typography.Text>No errors</Typography.Text>
+			</div>
+		);
+	}
+
+	return (
+		<div className={styles.tooltipContent}>
+			<Typography.Text className={styles.tooltipHeader}>
+				{item.label}
+			</Typography.Text>
+			{nonZeroBreakdown.map((b) => (
+				<div key={b.label} className={styles.tooltipRow}>
+					<Typography.Text>{b.label}</Typography.Text>
+					<Typography.Text className={styles.tooltipValue}>
+						{b.value}
+					</Typography.Text>
+				</div>
+			))}
+		</div>
+	);
+}
+
 export function GroupedStatusCounts({
 	items,
 	showZeroValues = true,
@@ -33,13 +79,15 @@ export function GroupedStatusCounts({
 						className={styles.separator}
 						style={{ backgroundColor: item.color }}
 					/>
-					<TooltipSimple title={`${item.label}: ${item.value}`}>
-						<span>
-							<TanStackTable.Text className={styles.value}>
-								{item.value || '-'}
-							</TanStackTable.Text>
-						</span>
-					</TooltipSimple>
+					<div className={styles.valueWrapper}>
+						<TooltipSimple title={buildTooltipContent(item)} arrow align="start">
+							<span className={styles.valueWrapperTooltip}>
+								<TanStackTable.Text className={styles.value}>
+									{item.value || '-'}
+								</TanStackTable.Text>
+							</span>
+						</TooltipSimple>
+					</div>
 				</div>
 			))}
 		</div>

frontend/src/container/InfraMonitoringK8sV2/constants.ts

@@ -2,8 +2,12 @@ import {
 	FiltersType,
 	IQuickFiltersConfig,
 } from 'components/QuickFilters/types';
+import { PANEL_TYPES } from 'constants/queryBuilder';
+import { GetQueryResultsProps } from 'lib/dashboard/getQueryResults';
 import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
-import { DataSource } from 'types/common/queryBuilder';
+import { EQueryType } from 'types/common/dashboard';
+import { DataSource, ReduceOperators } from 'types/common/queryBuilder';
+import { v4 } from 'uuid';
 
 // TODO(backend): Find a way to generate this via openapi
 export const INFRA_MONITORING_ATTR_KEYS = {
@@ -130,6 +134,7 @@ export enum VIEWS {
 	CONTAINERS = 'containers',
 	PROCESSES = 'processes',
 	EVENTS = 'events',
+	POD_METRICS = 'pod_metrics',
 }
 
 export const VIEW_TYPES = {
@@ -137,6 +142,7 @@ export const VIEW_TYPES = {
 	LOGS: VIEWS.LOGS,
 	TRACES: VIEWS.TRACES,
 	EVENTS: VIEWS.EVENTS,
+	POD_METRICS: VIEWS.POD_METRICS,
 };
 
 export const K8sCategories = {
@@ -916,3 +922,261 @@ export const METRIC_NAMESPACE_BY_ENTITY: Record<InfraMonitoringEntity, string> =
 		[InfraMonitoringEntity.JOBS]: 'k8s.',
 		[InfraMonitoringEntity.VOLUMES]: 'k8s.volume.',
 	};
+
+export interface WorkloadFilterContext {
+	workloadNameKey: string;
+	workloadNameValue: string;
+	clusterName: string;
+	namespaceName?: string;
+}
+
+export const podUtilizationByPodWidgetInfo = [
+	{
+		title: 'CPU Limit Utilization By Pod Name',
+		yAxisUnit: 'percentunit',
+	},
+	{
+		title: 'CPU Request Utilization By Pod Name',
+		yAxisUnit: 'percentunit',
+	},
+	{
+		title: 'Memory Limit Utilization By Pod Name',
+		yAxisUnit: 'percentunit',
+	},
+	{
+		title: 'Memory Request Utilization By Pod Name',
+		yAxisUnit: 'percentunit',
+	},
+	{
+		title: 'FileSystem Usage Percentage By Pod Name',
+		yAxisUnit: 'percentunit',
+	},
+];
+
+export function getPodUtilizationByPodQueryPayloads(
+	context: WorkloadFilterContext,
+	start: number,
+	end: number,
+	dotMetricsEnabled: boolean,
+): GetQueryResultsProps[] {
+	const getKey = (dotKey: string, underscoreKey: string): string =>
+		dotMetricsEnabled ? dotKey : underscoreKey;
+
+	const k8sPodCpuLimitUtilKey = getKey(
+		'k8s.pod.cpu_limit_utilization',
+		'k8s_pod_cpu_limit_utilization',
+	);
+	const k8sPodCpuRequestUtilKey = getKey(
+		'k8s.pod.cpu_request_utilization',
+		'k8s_pod_cpu_request_utilization',
+	);
+	const k8sPodMemLimitUtilKey = getKey(
+		'k8s.pod.memory_limit_utilization',
+		'k8s_pod_memory_limit_utilization',
+	);
+	const k8sPodMemRequestUtilKey = getKey(
+		'k8s.pod.memory_request_utilization',
+		'k8s_pod_memory_request_utilization',
+	);
+	const k8sPodFsUsageKey = getKey(
+		'k8s.pod.filesystem.usage',
+		'k8s_pod_filesystem_usage',
+	);
+	const k8sPodFsCapacityKey = getKey(
+		'k8s.pod.filesystem.capacity',
+		'k8s_pod_filesystem_capacity',
+	);
+	const k8sPodNameKey = getKey('k8s.pod.name', 'k8s_pod_name');
+	const k8sClusterNameKey = getKey('k8s.cluster.name', 'k8s_cluster_name');
+	const k8sNamespaceNameKey = getKey('k8s.namespace.name', 'k8s_namespace_name');
+
+	const baseFilters = [
+		{
+			id: 'workload',
+			key: {
+				dataType: DataTypes.String,
+				id: `${context.workloadNameKey}--string--tag--false`,
+				key: context.workloadNameKey,
+				type: 'tag',
+			},
+			op: '=',
+			value: context.workloadNameValue,
+		},
+		{
+			id: 'cluster',
+			key: {
+				dataType: DataTypes.String,
+				id: `${k8sClusterNameKey}--string--tag--false`,
+				key: k8sClusterNameKey,
+				type: 'tag',
+			},
+			op: '=',
+			value: context.clusterName,
+		},
+		...(context.namespaceName
+			? [
+					{
+						id: 'namespace',
+						key: {
+							dataType: DataTypes.String,
+							id: `${k8sNamespaceNameKey}--string--tag--false`,
+							key: k8sNamespaceNameKey,
+							type: 'tag',
+						},
+						op: '=',
+						value: context.namespaceName,
+					},
+				]
+			: []),
+	];
+
+	const podNameGroupBy = [
+		{
+			dataType: DataTypes.String,
+			id: `${k8sPodNameKey}--string--tag--false`,
+			key: k8sPodNameKey,
+			type: 'tag',
+		},
+	];
+
+	const buildSingleMetricQuery = (
+		metricKey: string,
+		metricId: string,
+	): GetQueryResultsProps => ({
+		selectedTime: 'GLOBAL_TIME',
+		graphType: PANEL_TYPES.TIME_SERIES,
+		query: {
+			builder: {
+				queryData: [
+					{
+						aggregateAttribute: {
+							dataType: DataTypes.Float64,
+							id: metricId,
+							key: metricKey,
+							type: 'Gauge',
+						},
+						aggregateOperator: 'avg',
+						dataSource: DataSource.METRICS,
+						disabled: false,
+						expression: 'A',
+						filters: {
+							items: [...baseFilters],
+							op: 'AND',
+						},
+						functions: [],
+						groupBy: podNameGroupBy,
+						having: [],
+						legend: `{{${k8sPodNameKey}}}`,
+						limit: null,
+						orderBy: [],
+						queryName: 'A',
+						reduceTo: ReduceOperators.AVG,
+						spaceAggregation: 'sum',
+						stepInterval: 60,
+						timeAggregation: 'avg',
+					},
+				],
+				queryFormulas: [],
+				queryTraceOperator: [],
+			},
+			clickhouse_sql: [{ disabled: false, legend: '', name: 'A', query: '' }],
+			id: v4(),
+			promql: [{ disabled: false, legend: '', name: 'A', query: '' }],
+			queryType: EQueryType.QUERY_BUILDER,
+		},
+		variables: {},
+		formatForWeb: false,
+		start,
+		end,
+	});
+
+	const filesystemUsagePercentQuery: GetQueryResultsProps = {
+		selectedTime: 'GLOBAL_TIME',
+		graphType: PANEL_TYPES.TIME_SERIES,
+		query: {
+			builder: {
+				queryData: [
+					{
+						aggregateAttribute: {
+							dataType: DataTypes.Float64,
+							id: 'fs_usage',
+							key: k8sPodFsUsageKey,
+							type: 'Gauge',
+						},
+						aggregateOperator: 'avg',
+						dataSource: DataSource.METRICS,
+						disabled: true,
+						expression: 'A',
+						filters: {
+							items: [...baseFilters],
+							op: 'AND',
+						},
+						functions: [],
+						groupBy: podNameGroupBy,
+						having: [],
+						legend: `{{${k8sPodNameKey}}}`,
+						limit: null,
+						orderBy: [],
+						queryName: 'A',
+						reduceTo: ReduceOperators.AVG,
+						spaceAggregation: 'sum',
+						stepInterval: 60,
+						timeAggregation: 'avg',
+					},
+					{
+						aggregateAttribute: {
+							dataType: DataTypes.Float64,
+							id: 'fs_capacity',
+							key: k8sPodFsCapacityKey,
+							type: 'Gauge',
+						},
+						aggregateOperator: 'avg',
+						dataSource: DataSource.METRICS,
+						disabled: true,
+						expression: 'B',
+						filters: {
+							items: [...baseFilters],
+							op: 'AND',
+						},
+						functions: [],
+						groupBy: podNameGroupBy,
+						having: [],
+						legend: `{{${k8sPodNameKey}}}`,
+						limit: null,
+						orderBy: [],
+						queryName: 'B',
+						reduceTo: ReduceOperators.AVG,
+						spaceAggregation: 'sum',
+						stepInterval: 60,
+						timeAggregation: 'avg',
+					},
+				],
+				queryFormulas: [
+					{
+						disabled: false,
+						expression: 'A/B',
+						legend: `{{${k8sPodNameKey}}}`,
+						queryName: 'F1',
+					},
+				],
+				queryTraceOperator: [],
+			},
+			clickhouse_sql: [{ disabled: false, legend: '', name: 'A', query: '' }],
+			id: v4(),
+			promql: [{ disabled: false, legend: '', name: 'A', query: '' }],
+			queryType: EQueryType.QUERY_BUILDER,
+		},
+		variables: {},
+		formatForWeb: false,
+		start,
+		end,
+	};
+
+	return [
+		buildSingleMetricQuery(k8sPodCpuLimitUtilKey, 'cpu_limit_util'),
+		buildSingleMetricQuery(k8sPodCpuRequestUtilKey, 'cpu_request_util'),
+		buildSingleMetricQuery(k8sPodMemLimitUtilKey, 'mem_limit_util'),
+		buildSingleMetricQuery(k8sPodMemRequestUtilKey, 'mem_request_util'),
+		filesystemUsagePercentQuery,
+	];
+}

pkg/alertmanager/alertmanagerserver/distpatcher_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"reflect"
 	"sort"
 	"sync"
 	"testing"
@@ -27,7 +28,6 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -110,44 +110,42 @@ func TestAggrGroup(t *testing.T) {
 		}
 	)
 
-	type notification struct {
-		alerts     alertmanagertypes.AlertSlice
-		notifiedAt time.Time
-	}
-	alertsCh := make(chan notification)
+	var (
+		last       = time.Now()
+		current    = time.Now()
+		lastCurMtx = &sync.Mutex{}
+		alertsCh   = make(chan alertmanagertypes.AlertSlice)
+	)
 
 	ntfy := func(ctx context.Context, alerts ...*alertmanagertypes.Alert) bool {
 		// Validate that the context is properly populated.
-		notifiedAt, ok := notify.Now(ctx)
-		assert.True(t, ok, "now missing")
-		_, ok = notify.GroupKey(ctx)
-		assert.True(t, ok, "group key missing")
-		lbls, ok := notify.GroupLabels(ctx)
-		if assert.True(t, ok, "group labels missing") {
-			assert.Equal(t, lset, lbls, "wrong group labels")
+		if _, ok := notify.Now(ctx); !ok {
+			t.Errorf("now missing")
 		}
-		rcv, ok := notify.ReceiverName(ctx)
-		if assert.True(t, ok, "receiver missing") {
-			assert.Equal(t, opts.Receiver, rcv, "wrong receiver")
+		if _, ok := notify.GroupKey(ctx); !ok {
+			t.Errorf("group key missing")
 		}
-		ri, ok := notify.RepeatInterval(ctx)
-		if assert.True(t, ok, "repeat interval missing") {
-			assert.Equal(t, notificationConfig.Renotify.RenotifyInterval, ri, "wrong repeat interval")
+		if lbls, ok := notify.GroupLabels(ctx); !ok || !reflect.DeepEqual(lbls, lset) {
+			t.Errorf("wrong group labels: %q", lbls)
+		}
+		if rcv, ok := notify.ReceiverName(ctx); !ok || rcv != opts.Receiver {
+			t.Errorf("wrong receiver: %q", rcv)
+		}
+		if ri, ok := notify.RepeatInterval(ctx); !ok || ri != notificationConfig.Renotify.RenotifyInterval {
+			t.Errorf("wrong repeat interval: %q", ri)
 		}
 
-		alertsCh <- notification{
-			alerts:     alertmanagertypes.AlertSlice(alerts),
-			notifiedAt: notifiedAt,
-		}
+		lastCurMtx.Lock()
+		last = current
+		// Subtract a millisecond to allow for races.
+		current = time.Now().Add(-time.Millisecond)
+		lastCurMtx.Unlock()
+
+		alertsCh <- alertmanagertypes.AlertSlice(alerts)
 
 		return true
 	}
 
-	assertNotifiedAfter := func(previous, current time.Time, interval time.Duration) {
-		t.Helper()
-		require.GreaterOrEqual(t, current.Sub(previous), interval, "received batch too early")
-	}
-
 	removeEndsAt := func(as alertmanagertypes.AlertSlice) alertmanagertypes.AlertSlice {
 		for i, a := range as {
 			ac := *a
@@ -158,26 +156,29 @@ func TestAggrGroup(t *testing.T) {
 	}
 
 	// Test regular situation where we wait for group_wait to send out alerts.
-	groupStartedAt := time.Now()
 	ag := newAggrGroup(context.Background(), lset, route, nil, promslog.NewNopLogger(), notificationConfig.Renotify.RenotifyInterval)
 
 	go ag.run(ntfy)
 
 	ag.insert(a1)
-	var lastNotificationAt time.Time
 
 	select {
 	case <-time.After(2 * opts.GroupWait):
-		require.FailNow(t, "expected initial batch after group_wait")
+		t.Fatalf("expected initial batch after group_wait")
 
-	case notification := <-alertsCh:
-		assertNotifiedAfter(groupStartedAt, notification.notifiedAt, opts.GroupWait)
-		lastNotificationAt = notification.notifiedAt
-		batch := notification.alerts
+	case batch := <-alertsCh:
+		lastCurMtx.Lock()
+		s := time.Since(last)
+		lastCurMtx.Unlock()
+		if s < opts.GroupWait {
+			t.Fatalf("received batch too early after %v", s)
+		}
 		exp := removeEndsAt(alertmanagertypes.AlertSlice{a1})
 		sort.Sort(batch)
 
-		require.Equal(t, exp, batch)
+		if !reflect.DeepEqual(batch, exp) {
+			t.Fatalf("expected alerts %v but got %v", exp, batch)
+		}
 	}
 
 	for i := 0; i < 3; i++ {
@@ -186,16 +187,21 @@ func TestAggrGroup(t *testing.T) {
 
 		select {
 		case <-time.After(2 * opts.GroupInterval):
-			require.FailNow(t, "expected new batch after group interval but received none")
+			t.Fatalf("expected new batch after group interval but received none")
 
-		case notification := <-alertsCh:
-			assertNotifiedAfter(lastNotificationAt, notification.notifiedAt, opts.GroupInterval)
-			lastNotificationAt = notification.notifiedAt
-			batch := notification.alerts
+		case batch := <-alertsCh:
+			lastCurMtx.Lock()
+			s := time.Since(last)
+			lastCurMtx.Unlock()
+			if s < opts.GroupInterval {
+				t.Fatalf("received batch too early after %v", s)
+			}
 			exp := removeEndsAt(alertmanagertypes.AlertSlice{a1, a3})
 			sort.Sort(batch)
 
-			require.Equal(t, exp, batch)
+			if !reflect.DeepEqual(batch, exp) {
+				t.Fatalf("expected alerts %v but got %v", exp, batch)
+			}
 		}
 	}
 
@@ -214,15 +220,15 @@ func TestAggrGroup(t *testing.T) {
 	// a2 lies way in the past so the initial group_wait should be skipped.
 	select {
 	case <-time.After(opts.GroupWait / 2):
-		require.FailNow(t, "expected immediate alert but received none")
+		t.Fatalf("expected immediate alert but received none")
 
-	case notification := <-alertsCh:
-		lastNotificationAt = notification.notifiedAt
-		batch := notification.alerts
+	case batch := <-alertsCh:
 		exp := removeEndsAt(alertmanagertypes.AlertSlice{a1, a2})
 		sort.Sort(batch)
 
-		require.Equal(t, exp, batch)
+		if !reflect.DeepEqual(batch, exp) {
+			t.Fatalf("expected alerts %v but got %v", exp, batch)
+		}
 	}
 
 	for i := 0; i < 3; i++ {
@@ -231,16 +237,21 @@ func TestAggrGroup(t *testing.T) {
 
 		select {
 		case <-time.After(2 * opts.GroupInterval):
-			require.FailNow(t, "expected new batch after group interval but received none")
+			t.Fatalf("expected new batch after group interval but received none")
 
-		case notification := <-alertsCh:
-			assertNotifiedAfter(lastNotificationAt, notification.notifiedAt, opts.GroupInterval)
-			lastNotificationAt = notification.notifiedAt
-			batch := notification.alerts
+		case batch := <-alertsCh:
+			lastCurMtx.Lock()
+			s := time.Since(last)
+			lastCurMtx.Unlock()
+			if s < opts.GroupInterval {
+				t.Fatalf("received batch too early after %v", s)
+			}
 			exp := removeEndsAt(alertmanagertypes.AlertSlice{a1, a2, a3})
 			sort.Sort(batch)
 
-			require.Equal(t, exp, batch)
+			if !reflect.DeepEqual(batch, exp) {
+				t.Fatalf("expected alerts %v but got %v", exp, batch)
+			}
 		}
 	}
 
@@ -252,14 +263,19 @@ func TestAggrGroup(t *testing.T) {
 
 	select {
 	case <-time.After(2 * opts.GroupInterval):
-		require.FailNow(t, "expected new batch after group interval but received none")
-	case notification := <-alertsCh:
-		assertNotifiedAfter(lastNotificationAt, notification.notifiedAt, opts.GroupInterval)
-		lastNotificationAt = notification.notifiedAt
-		batch := notification.alerts
+		t.Fatalf("expected new batch after group interval but received none")
+	case batch := <-alertsCh:
+		lastCurMtx.Lock()
+		s := time.Since(last)
+		lastCurMtx.Unlock()
+		if s < opts.GroupInterval {
+			t.Fatalf("received batch too early after %v", s)
+		}
 		sort.Sort(batch)
 
-		require.Equal(t, exp, batch)
+		if !reflect.DeepEqual(batch, exp) {
+			t.Fatalf("expected alerts %v but got %v", exp, batch)
+		}
 	}
 
 	// Resolve all remaining alerts, they should be removed after the next batch was sent.
@@ -273,16 +289,24 @@ func TestAggrGroup(t *testing.T) {
 
 	select {
 	case <-time.After(2 * opts.GroupInterval):
-		require.FailNow(t, "expected new batch after group interval but received none")
+		t.Fatalf("expected new batch after group interval but received none")
 
-	case notification := <-alertsCh:
-		assertNotifiedAfter(lastNotificationAt, notification.notifiedAt, opts.GroupInterval)
-		batch := notification.alerts
+	case batch := <-alertsCh:
+		lastCurMtx.Lock()
+		s := time.Since(last)
+		lastCurMtx.Unlock()
+		if s < opts.GroupInterval {
+			t.Fatalf("received batch too early after %v", s)
+		}
 		sort.Sort(batch)
 
-		require.Equal(t, resolved, batch)
+		if !reflect.DeepEqual(batch, resolved) {
+			t.Fatalf("expected alerts %v but got %v", resolved, batch)
+		}
 
-		require.Eventually(t, ag.empty, 2*opts.GroupInterval, 10*time.Millisecond, "expected aggregation group to be empty after resolving alerts: %v", ag)
+		if !ag.empty() {
+			t.Fatalf("Expected aggregation group to be empty after resolving alerts: %v", ag)
+		}
 	}
 
 	ag.stop()
@@ -316,7 +340,9 @@ func TestGroupLabels(t *testing.T) {
 
 	ls := getGroupLabels(a, route.RouteOpts.GroupBy, false)
 
-	require.Equal(t, expLs, ls)
+	if !reflect.DeepEqual(ls, expLs) {
+		t.Fatalf("expected labels are %v, but got %v", expLs, ls)
+	}
 }
 
 func TestAggrRouteMap(t *testing.T) {
@@ -332,13 +358,17 @@ route:
   group_interval: 1m
   receiver: 'slack'`
 	conf, err := config.Load(confData)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	providerSettings := createTestProviderSettings()
 	logger := providerSettings.Logger
 	route := dispatch.NewRoute(conf.Route, nil)
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
 	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer alerts.Close()
 
 	timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
@@ -347,7 +377,9 @@ route:
 	store := nfroutingstoretest.NewMockSQLRouteStore()
 	store.MatchExpectationsInOrder(false)
 	nfManager, err := rulebasednotification.New(context.Background(), providerSettings, nfmanager.Config{}, store)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	orgId := "test-org"
 
 	ctx := context.Background()
@@ -465,7 +497,9 @@ route:
 		require.NoError(t, err)
 	}
 	err = alerts.Put(ctx, inputAlerts...)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// Let alerts get processed.
 	for i := 0; len(recorder.Alerts()) != 4; i++ {
@@ -597,13 +631,17 @@ route:
   group_interval: 10ms
   receiver: 'slack'`
 	conf, err := config.Load(confData)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	providerSettings := createTestProviderSettings()
 	logger := providerSettings.Logger
 	route := dispatch.NewRoute(conf.Route, nil)
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
 	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer alerts.Close()
 
 	timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
@@ -612,7 +650,9 @@ route:
 	store := nfroutingstoretest.NewMockSQLRouteStore()
 	store.MatchExpectationsInOrder(false)
 	nfManager, err := rulebasednotification.New(context.Background(), providerSettings, nfmanager.Config{}, store)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	orgId := "test-org"
 
 	ctx := context.Background()
@@ -759,7 +799,9 @@ route:
 		require.NoError(t, err)
 	}
 	err = alerts.Put(ctx, inputAlerts...)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	for i := 0; len(recorder.Alerts()) != 9; i++ {
 		time.Sleep(400 * time.Millisecond)
@@ -848,13 +890,17 @@ route:
   group_interval: 10ms
   receiver: 'slack'`
 	conf, err := config.Load(confData)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	providerSettings := createTestProviderSettings()
 	logger := providerSettings.Logger
 	route := dispatch.NewRoute(conf.Route, nil)
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
 	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer alerts.Close()
 
 	timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
@@ -863,7 +909,9 @@ route:
 	store := nfroutingstoretest.NewMockSQLRouteStore()
 	store.MatchExpectationsInOrder(false)
 	nfManager, err := rulebasednotification.New(context.Background(), providerSettings, nfmanager.Config{}, store)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	orgId := "test-org"
 
 	ctx := context.Background()
@@ -981,7 +1029,9 @@ route:
 		require.NoError(t, err)
 	}
 	err = alerts.Put(ctx, inputAlerts...)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	for i := 0; len(recorder.Alerts()) != 3 && i < 15; i++ {
 		time.Sleep(400 * time.Millisecond)
@@ -1110,7 +1160,9 @@ func TestDispatcherRace(t *testing.T) {
 	logger := promslog.NewNopLogger()
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
 	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer alerts.Close()
 
 	timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
@@ -1136,13 +1188,17 @@ route:
   group_interval: 5m
   receiver: 'slack'`
 	conf, err := config.Load(confData)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	route := dispatch.NewRoute(conf.Route, nil)
 	providerSettings := createTestProviderSettings()
 	logger := providerSettings.Logger
 	marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
 	alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	defer alerts.Close()
 	timeout := func(d time.Duration) time.Duration { return d }
 	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*alertmanagertypes.Alert)}
@@ -1150,7 +1206,9 @@ route:
 	store := nfroutingstoretest.NewMockSQLRouteStore()
 	store.MatchExpectationsInOrder(false)
 	nfManager, err := rulebasednotification.New(context.Background(), providerSettings, nfmanager.Config{}, store)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 	orgId := "test-org"
 
 	for i := 0; i < numAlerts; i++ {
@@ -1209,7 +1267,9 @@ func TestDispatcher_DoMaintenance(t *testing.T) {
 	marker := alertmanagertypes.NewMarker(r)
 
 	alerts, err := mem.NewAlerts(context.Background(), marker, time.Minute, 0, nil, promslog.NewNopLogger(), prometheus.NewRegistry(), nil)
-	require.NoError(t, err)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	route := &dispatch.Route{
 		RouteOpts: dispatch.RouteOpts{
@@ -1303,14 +1363,18 @@ route:
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			conf, err := config.Load(tc.confData)
-			require.NoError(t, err)
+			if err != nil {
+				t.Fatal(err)
+			}
 
 			providerSettings := createTestProviderSettings()
 			logger := providerSettings.Logger
 			route := dispatch.NewRoute(conf.Route, nil)
 			marker := alertmanagertypes.NewMarker(prometheus.NewRegistry())
 			alerts, err := mem.NewAlerts(context.Background(), marker, time.Hour, 0, nil, logger, prometheus.NewRegistry(), nil)
-			require.NoError(t, err)
+			if err != nil {
+				t.Fatal(err)
+			}
 			defer alerts.Close()
 
 			timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
@@ -1319,7 +1383,9 @@ route:
 			store := nfroutingstoretest.NewMockSQLRouteStore()
 			store.MatchExpectationsInOrder(false)
 			nfManager, err := rulebasednotification.New(context.Background(), providerSettings, nfmanager.Config{}, store)
-			require.NoError(t, err)
+			if err != nil {
+				t.Fatal(err)
+			}
 			d := NewDispatcher(alerts, route, recorder, marker, timeout, nil, logger, metrics, nfManager, "test-org")
 			// setup the dispatcher for tests
 			d.receiverRoutes = map[string]*dispatch.Route{}

pkg/modules/llmpricingrule/impllmpricingrule/module.go

@@ -213,18 +213,18 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
 					Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
 						Name:   "A",
 						Signal: telemetrytypes.SignalTraces,
-						Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", llmpricingruletypes.GenAIRequestModel)},
+						Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", telemetrytypes.GenAIRequestModel)},
 						Aggregations: []qbtypes.TraceAggregation{
 							{Expression: "count()", Alias: "spanCount"},
 						},
 						GroupBy: []qbtypes.GroupByKey{
 							{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-								Name:          llmpricingruletypes.GenAIRequestModel,
+								Name:          telemetrytypes.GenAIRequestModel,
 								FieldContext:  telemetrytypes.FieldContextSpan,
 								FieldDataType: telemetrytypes.FieldDataTypeString,
 							}},
 							{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-								Name:          llmpricingruletypes.GenAIProviderName,
+								Name:          telemetrytypes.GenAIProviderName,
 								FieldContext:  telemetrytypes.FieldContextSpan,
 								FieldDataType: telemetrytypes.FieldDataTypeString,
 							}},
@@ -254,9 +254,9 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
 		switch c.Type {
 		case qbtypes.ColumnTypeGroup:
 			switch c.Name {
-			case llmpricingruletypes.GenAIRequestModel:
+			case telemetrytypes.GenAIRequestModel:
 				modelIdx = i
-			case llmpricingruletypes.GenAIProviderName:
+			case telemetrytypes.GenAIProviderName:
 				providerIdx = i
 			}
 		case qbtypes.ColumnTypeAggregation:

pkg/prometheus/clickhouseprometheus/client.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
-	"github.com/SigNoz/signoz/pkg/query-service/constants"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
 	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
 	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
@@ -143,10 +142,6 @@ func (client *client) queryToClickhouseQuery(_ context.Context, query *prompb.Qu
 	conditions = append(conditions, "temporality IN ['Cumulative', 'Unspecified']")
 	conditions = append(conditions, fmt.Sprintf("unix_milli >= %d AND unix_milli < %d", start, end))
 
-	normalized := !constants.IsDotMetricsEnabled
-
-	conditions = append(conditions, fmt.Sprintf("__normalized = %v", normalized))
-
 	args = append(args, metricName)
 	for _, m := range query.Matchers {
 		switch m.Type {

pkg/querier/querier.go

@@ -42,6 +42,7 @@ type querier struct {
 	metadataStore            telemetrytypes.MetadataStore
 	promEngine               prometheus.Prometheus
 	traceStmtBuilder         qbtypes.StatementBuilder[qbtypes.TraceAggregation]
+	aiTraceStmtBuilder       qbtypes.StatementBuilder[qbtypes.TraceAggregation]
 	logStmtBuilder           qbtypes.StatementBuilder[qbtypes.LogAggregation]
 	auditStmtBuilder         qbtypes.StatementBuilder[qbtypes.LogAggregation]
 	metricStmtBuilder        qbtypes.StatementBuilder[qbtypes.MetricAggregation]
@@ -61,6 +62,7 @@ func New(
 	metadataStore telemetrytypes.MetadataStore,
 	promEngine prometheus.Prometheus,
 	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
+	aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
@@ -82,6 +84,7 @@ func New(
 		metadataStore:            metadataStore,
 		promEngine:               promEngine,
 		traceStmtBuilder:         traceStmtBuilder,
+		aiTraceStmtBuilder:       aiTraceStmtBuilder,
 		logStmtBuilder:           logStmtBuilder,
 		auditStmtBuilder:         auditStmtBuilder,
 		metricStmtBuilder:        metricStmtBuilder,
@@ -237,7 +240,12 @@ func (q *querier) buildQueries(
 			case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
 				spec.ShiftBy = extractShiftFromBuilderQuery(spec)
 				timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
-				bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
+				stmtBuilder := q.traceStmtBuilder
+				if spec.Source == telemetrytypes.SourceAI {
+					event.Source = telemetrytypes.SourceAI.StringValue()
+					stmtBuilder = q.aiTraceStmtBuilder
+				}
+				bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
 				queries[spec.Name] = bq
 				steps[spec.Name] = spec.StepInterval
 			case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
@@ -860,7 +868,11 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
 		specCopy := qt.spec.Copy()
 		specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
 		adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
-		return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.traceStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
+		shiftStmtBuilder := q.traceStmtBuilder
+		if qt.spec.Source == telemetrytypes.SourceAI {
+			shiftStmtBuilder = q.aiTraceStmtBuilder
+		}
+		return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, shiftStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
 
 	case *builderQuery[qbtypes.LogAggregation]:
 		specCopy := qt.spec.Copy()

pkg/querier/querier_test.go

@@ -49,6 +49,7 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
 		metadataStore,
 		nil,                // prometheus
 		nil,                // traceStmtBuilder
+		nil,                // aiTraceStmtBuilder
 		nil,                // logStmtBuilder
 		nil,                // auditStmtBuilder
 		nil,                // metricStmtBuilder
@@ -121,6 +122,7 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
 		metadataStore,
 		nil,                      // prometheus
 		nil,                      // traceStmtBuilder
+		nil,                      // aiTraceStmtBuilder
 		nil,                      // logStmtBuilder
 		nil,                      // auditStmtBuilder
 		&mockMetricStmtBuilder{}, // metricStmtBuilder

pkg/querier/signozquerier/provider.go

@@ -9,6 +9,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/prometheus"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
+	"github.com/SigNoz/signoz/pkg/telemetryai"
 	"github.com/SigNoz/signoz/pkg/telemetryaudit"
 	"github.com/SigNoz/signoz/pkg/telemetrylogs"
 	"github.com/SigNoz/signoz/pkg/telemetrymetadata"
@@ -92,6 +93,17 @@ func newProvider(
 		cfg.SkipResourceFingerprint.Threshold,
 	)
 
+	// AI trace statement builder (source=ai). The gen_ai gate/column keys are
+	// surfaced by the metadata store itself (enrichWithGenAIKeys), so queries work
+	// before any gen_ai metadata is ingested — no per-builder decoration needed.
+	// The standard trace builder doubles as the delegate for the span-list path.
+	aiTraceStmtBuilder := telemetryai.NewAITraceStatementBuilder(
+		settings,
+		telemetryMetadataStore,
+		traceStmtBuilder,
+		flagger,
+	)
+
 	// Create trace operator statement builder
 	traceOperatorStmtBuilder := telemetrytraces.NewTraceOperatorStatementBuilder(
 		settings,
@@ -185,6 +197,7 @@ func newProvider(
 		telemetryMetadataStore,
 		prometheus,
 		traceStmtBuilder,
+		aiTraceStmtBuilder,
 		logStmtBuilder,
 		auditStmtBuilder,
 		metricStmtBuilder,

pkg/query-service/rules/setups_test.go

@@ -48,6 +48,7 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
 		metadataStore,
 		nil, // prometheus
 		nil, // traceStmtBuilder
+		nil, // aiTraceStmtBuilder
 		nil, // logStmtBuilder
 		nil, // auditStmtBuilder
 		metricStmtBuilder,
@@ -103,6 +104,7 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
 		metadataStore,
 		nil,            // prometheus
 		nil,            // traceStmtBuilder
+		nil,            // aiTraceStmtBuilder
 		logStmtBuilder, // logStmtBuilder
 		nil,            // auditStmtBuilder
 		nil,            // metricStmtBuilder
@@ -152,6 +154,7 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
 		metadataStore,
 		nil,              // prometheus
 		traceStmtBuilder, // traceStmtBuilder
+		nil,              // aiTraceStmtBuilder
 		nil,              // logStmtBuilder
 		nil,              // auditStmtBuilder
 		nil,              // metricStmtBuilder

pkg/query-service/rules/threshold_rule_test.go

@@ -628,7 +628,7 @@ func TestThresholdRuleUnitCombinations(t *testing.T) {
 		queryString := "SELECT any"
 		telemetryStore.Mock().
 			ExpectQuery(queryString).
-			WithArgs(nil, nil, nil, nil, nil, nil, nil, nil, nil).
+			WithArgs(nil, nil, nil, nil, nil, nil, nil, nil).
 			WillReturnRows(rows)
 		postableRule.RuleCondition.CompareOperator = c.compareOperator
 		postableRule.RuleCondition.MatchType = c.matchType
@@ -737,7 +737,7 @@ func TestThresholdRuleNoData(t *testing.T) {
 		queryString := "SELECT any"
 		telemetryStore.Mock().
 			ExpectQuery(queryString).
-			WithArgs(nil, nil, nil, nil, nil, nil, nil, nil).
+			WithArgs(nil, nil, nil, nil, nil, nil, nil).
 			WillReturnRows(rows)
 
 		querier, mockMetadataStore := prepareQuerierForMetrics(t, telemetryStore)
@@ -1129,7 +1129,7 @@ func TestMultipleThresholdRule(t *testing.T) {
 		queryString := "SELECT any"
 		telemetryStore.Mock().
 			ExpectQuery(queryString).
-			WithArgs(nil, nil, nil, nil, nil, nil, nil, nil).
+			WithArgs(nil, nil, nil, nil, nil, nil, nil).
 			WillReturnRows(rows)
 
 		querier, mockMetadataStore := prepareQuerierForMetrics(t, telemetryStore)
@@ -1922,7 +1922,7 @@ func TestThresholdEval_RequireMinPoints(t *testing.T) {
 		queryString := "SELECT any"
 		telemetryStore.Mock().
 			ExpectQuery(queryString).
-			WithArgs(nil, nil, nil, nil, nil, nil, nil, nil).
+			WithArgs(nil, nil, nil, nil, nil, nil, nil).
 			WillReturnRows(rows)
 
 		querier, mockMetadataStore := prepareQuerierForMetrics(t, telemetryStore)

pkg/query-service/rules/utils.go

@@ -57,9 +57,6 @@ func GenerateMetricQueryCHArgs(
 		queryArgs = append(queryArgs, temporality.StringValue())
 	}
 
-	// Add normalized flag
-	queryArgs = append(queryArgs, false)
-
 	// Step2: Add temporal aggregation args
 	// build args for filtering signoz_metrics.distributed_samples_v4 table
 	temporalAggArgs := []interface{}{

pkg/querybuilder/filter_split.go

@@ -0,0 +1,154 @@
+package querybuilder
+
+import (
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/antlr4-go/antlr/v4"
+)
+
+// SplitFilterForAggregates partitions a single filter expression into a span-level
+// part (a WHERE over spans) and a trace-level part (a HAVING over per-trace
+// aggregates), splitting on the top-level AND.
+//
+// A key is trace-level when it carries the trace field context (`trace.completion_tokens`)
+// or, with no context, its bare name is in aggregateNames. Any other explicit context
+// (`span.`, `resource.`, …) is span-level. Trace-level and span-level keys may be
+// AND-combined (they run at different query stages) but not OR-combined; an OR that
+// mixes the two is an error.
+//
+// Syntax errors are ignored here — each part is re-parsed downstream (PrepareWhereClause
+// for the span part, the HAVING rewriter for the trace part), which surface them.
+func SplitFilterForAggregates(query string, aggregateNames map[string]struct{}) (spanExpr string, havingExpr string, err error) {
+	if strings.TrimSpace(query) == "" {
+		return "", "", nil
+	}
+
+	s := filterSplitter{query: []rune(query), aggregateNames: aggregateNames}
+	s.visit(parseFilterQuery(query))
+
+	if s.mixed {
+		return "", "", errors.NewInvalidInputf(errors.CodeInvalidInput,
+			"trace-level and span-level filters cannot be combined within an OR/NOT group; separate them with a top-level AND")
+	}
+	return strings.Join(s.span, " AND "), strings.Join(s.having, " AND "), nil
+}
+
+func parseFilterQuery(query string) antlr.Tree {
+	lexer := grammar.NewFilterQueryLexer(antlr.NewInputStream(query))
+	lexer.RemoveErrorListeners()
+	parser := grammar.NewFilterQueryParser(antlr.NewCommonTokenStream(lexer, 0))
+	parser.RemoveErrorListeners()
+	return parser.Query()
+}
+
+// filterSplitter walks the parse tree once, flattening the top-level AND chain and
+// routing each atom (a comparison, a NOT expression, or a whole multi-branch OR group)
+// to the span or having bucket by the class of the keys it references.
+type filterSplitter struct {
+	query          []rune
+	aggregateNames map[string]struct{}
+	span           []string
+	having         []string
+	mixed          bool
+}
+
+func (s *filterSplitter) visit(node antlr.Tree) {
+	switch n := node.(type) {
+	case *grammar.QueryContext:
+		if n.Expression() != nil {
+			s.visit(n.Expression())
+		}
+	case *grammar.ExpressionContext:
+		if n.OrExpression() != nil {
+			s.visit(n.OrExpression())
+		}
+	case *grammar.OrExpressionContext:
+		// a single branch is just an AND chain; multiple branches are a real OR, kept
+		// whole so a class-mixing OR can be rejected.
+		if ands := n.AllAndExpression(); len(ands) == 1 {
+			s.visit(ands[0])
+		} else {
+			s.route(n)
+		}
+	case *grammar.AndExpressionContext:
+		for _, u := range n.AllUnaryExpression() {
+			s.visit(u)
+		}
+	case *grammar.UnaryExpressionContext:
+		if n.NOT() != nil {
+			s.route(n)
+		} else if n.Primary() != nil {
+			s.visit(n.Primary())
+		}
+	case *grammar.PrimaryContext:
+		if n.OrExpression() != nil { // parenthesized sub-expression
+			s.visit(n.OrExpression())
+		} else {
+			s.route(n)
+		}
+	}
+}
+
+// route classifies an atom and appends its original source text to the right bucket.
+func (s *filterSplitter) route(atom antlr.ParserRuleContext) {
+	isTrace, isSpan := classifyKeys(atom, s.aggregateNames)
+	if isTrace && isSpan {
+		s.mixed = true
+		return
+	}
+	text := atomSourceText(s.query, atom)
+	// A multi-branch OR group's source slice excludes its enclosing parens (they belong
+	// to the parent Primary). Re-wrap it so rejoining a bucket with " AND " cannot invert
+	// OR/AND precedence, e.g. `a AND (b OR c)` must not flatten to `a AND b OR c`.
+	if or, ok := atom.(*grammar.OrExpressionContext); ok && len(or.AllAndExpression()) > 1 {
+		text = "(" + text + ")"
+	}
+	if isTrace {
+		s.having = append(s.having, text)
+	} else {
+		s.span = append(s.span, text)
+	}
+}
+
+// classifyKeys reports whether a subtree references trace-level and/or span-level keys.
+// A key is trace-level when it carries the trace field context or, with no context,
+// its name is a known aggregate; an unknown name under the trace context stays
+// trace-level so the aggregate validation rejects it with a targeted error. Any other
+// explicit context (`span.`, `resource.`, …) is span-level.
+func classifyKeys(node antlr.Tree, aggregateNames map[string]struct{}) (isTrace, isSpan bool) {
+	kc, ok := node.(*grammar.KeyContext)
+	if ok {
+		key := telemetrytypes.GetFieldKeyFromKeyText(kc.GetText())
+		switch key.FieldContext {
+		case telemetrytypes.FieldContextTrace:
+			isTrace = true
+		case telemetrytypes.FieldContextUnspecified:
+			_, isTrace = aggregateNames[key.Name]
+			isSpan = !isTrace
+		default:
+			isSpan = true
+		}
+		return
+	}
+	for i := 0; i < node.GetChildCount(); i++ {
+		t, s := classifyKeys(node.GetChild(i), aggregateNames)
+		isTrace = isTrace || t
+		isSpan = isSpan || s
+	}
+	return
+}
+
+// atomSourceText returns the original source substring for an atom, preserving
+// whitespace. The token stream drops skipped whitespace, which would glue word
+// operators (OR/AND/NOT) to their operands, so slice the input by token offsets.
+// ANTLR offsets are rune indices (InputStream holds []rune), hence the rune slice.
+func atomSourceText(query []rune, atom antlr.ParserRuleContext) string {
+	start, stop := atom.GetStart(), atom.GetStop()
+	if start == nil || stop == nil || start.GetStart() < 0 || stop.GetStop() >= len(query) || stop.GetStop() < start.GetStart() {
+		return atom.GetText()
+	}
+	return string(query[start.GetStart() : stop.GetStop()+1])
+}

pkg/querybuilder/filter_split_test.go

@@ -0,0 +1,167 @@
+package querybuilder
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSplitFilterForAggregates(t *testing.T) {
+	agg := map[string]struct{}{"completion_tokens": {}, "span_count": {}, "prompt_tokens": {}}
+
+	type tc struct {
+		name    string
+		query   string
+		span    string // expected span-level (WHERE) part; "" => empty
+		having  string // expected trace-level (HAVING) part; "" => empty
+		wantErr bool
+	}
+
+	cases := []tc{
+		// --- empty input ---------------------------------------------------------
+		{
+			name: "empty",
+		},
+		{
+			name:  "whitespace only",
+			query: "   ",
+		},
+
+		// --- single class --------------------------------------------------------
+		{
+			name:  "span only",
+			query: "service.name = 'x'",
+			span:  "service.name = 'x'",
+		},
+		{
+			name:   "agg only bare",
+			query:  "completion_tokens > 1000",
+			having: "completion_tokens > 1000",
+		},
+		{
+			// the user-facing `trace.` prefix marks a trace-level aggregate.
+			name:   "agg only trace prefix",
+			query:  "trace.completion_tokens > 1000",
+			having: "trace.completion_tokens > 1000",
+		},
+		{
+			// an unknown name under the trace context still routes trace-level, so the
+			// aggregate validation rejects it with a targeted error instead of the span
+			// path failing on an unknown field.
+			name:   "unknown aggregate under trace context stays trace-level",
+			query:  "trace.not_an_aggregate > 1000",
+			having: "trace.not_an_aggregate > 1000",
+		},
+
+		{
+			// ANTLR token offsets are rune indices; slicing must not shift after a
+			// multi-byte char (this used to truncate 1000 → 100).
+			name:   "unicode value before the split",
+			query:  "service.name = 'héllo' AND completion_tokens > 1000",
+			span:   "service.name = 'héllo'",
+			having: "completion_tokens > 1000",
+		},
+
+		// --- top-level AND splits across the two buckets -------------------------
+		{
+			name:   "span AND agg",
+			query:  "service.name = 'x' AND completion_tokens > 1000",
+			span:   "service.name = 'x'",
+			having: "completion_tokens > 1000",
+		},
+		{
+			// order within a bucket is preserved; the two span atoms join with AND.
+			name:   "span AND span AND agg",
+			query:  "service.name = 'x' AND kind_string = 'Internal' AND completion_tokens > 1000",
+			span:   "service.name = 'x' AND kind_string = 'Internal'",
+			having: "completion_tokens > 1000",
+		},
+		{
+			// a parenthesized top-level AND still splits across the two buckets.
+			name:   "parenthesized span AND agg",
+			query:  "(service.name = 'x' AND completion_tokens > 1000)",
+			span:   "service.name = 'x'",
+			having: "completion_tokens > 1000",
+		},
+
+		// --- OR groups are re-wrapped in parens so a later AND-join can't invert
+		//     precedence (`a AND (b OR c)` must not flatten to `a AND b OR c`) ------
+		{
+			name:   "agg OR agg",
+			query:  "completion_tokens > 1000 OR span_count > 3",
+			having: "(completion_tokens > 1000 OR span_count > 3)",
+		},
+		{
+			name:  "span OR span",
+			query: "service.name = 'x' OR kind_string = 'Internal'",
+			span:  "(service.name = 'x' OR kind_string = 'Internal')",
+		},
+		{
+			name:  "span AND (span OR span)",
+			query: "service.name = 'x' AND (kind_string = 'Internal' OR kind_string = 'Client')",
+			span:  "service.name = 'x' AND (kind_string = 'Internal' OR kind_string = 'Client')",
+		},
+		{
+			name:   "agg AND (agg OR agg)",
+			query:  "prompt_tokens > 5 AND (completion_tokens > 1000 OR span_count > 3)",
+			having: "prompt_tokens > 5 AND (completion_tokens > 1000 OR span_count > 3)",
+		},
+		{
+			// the OR group routes to span, the trailing aggregate to having.
+			name:   "span AND (span OR span) AND agg",
+			query:  "a.b = 'x' AND (c.d = 'y' OR e.f = 'z') AND completion_tokens > 1000",
+			span:   "a.b = 'x' AND (c.d = 'y' OR e.f = 'z')",
+			having: "completion_tokens > 1000",
+		},
+
+		// --- a nested AND group flattens across the buckets (no spurious parens) --
+		{
+			name:   "(span AND agg) AND agg",
+			query:  "(service.name = 'x' AND completion_tokens > 1000) AND prompt_tokens > 5",
+			span:   "service.name = 'x'",
+			having: "completion_tokens > 1000 AND prompt_tokens > 5",
+		},
+
+		// --- NOT wrapping a single-class group is routed whole to that class ------
+		{
+			name:   "not agg",
+			query:  "NOT (completion_tokens > 1000)",
+			having: "NOT (completion_tokens > 1000)",
+		},
+		{
+			name:  "not span",
+			query: "NOT (service.name = 'x')",
+			span:  "NOT (service.name = 'x')",
+		},
+
+		// --- class-mixing is rejected in an OR group, a NOT group, or a nested OR -
+		{
+			name:    "agg OR span rejected",
+			query:   "completion_tokens > 1000 OR service.name = 'x'",
+			wantErr: true,
+		},
+		{
+			name:    "not mixed rejected",
+			query:   "NOT (completion_tokens > 1000 AND service.name = 'x')",
+			wantErr: true,
+		},
+		{
+			name:    "span AND (agg OR span) rejected",
+			query:   "service.name = 'x' AND (completion_tokens > 1000 OR kind_string = 'Client')",
+			wantErr: true,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			span, having, err := SplitFilterForAggregates(c.query, agg)
+			if c.wantErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, c.span, span, "span part")
+			require.Equal(t, c.having, having, "having part")
+		})
+	}
+}

pkg/querybuilder/having_rewriter.go

@@ -18,6 +18,19 @@ func NewHavingExpressionRewriter() *HavingExpressionRewriter {
 	}
 }
 
+// Rewrite rewrites and validates a HAVING expression against a caller-supplied
+// column map (user-facing name -> SQL identifier/expression). Values are inlined, so
+// the result is a bare SQL boolean expression with no bound args. Used by callers
+// that project their own aggregate columns (e.g. the AI trace list) rather than the
+// query's Aggregations.
+func (r *HavingExpressionRewriter) Rewrite(expression string, columnMap map[string]string) (string, error) {
+	if len(strings.TrimSpace(expression)) == 0 {
+		return "", nil
+	}
+	r.columnMap = columnMap
+	return r.rewriteAndValidate(expression)
+}
+
 // RewriteForTraces rewrites and validates the HAVING expression for a traces query.
 func (r *HavingExpressionRewriter) RewriteForTraces(expression string, aggregations []qbtypes.TraceAggregation) (string, error) {
 	if len(strings.TrimSpace(expression)) == 0 {

pkg/telemetryai/field_mapper.go

@@ -0,0 +1,99 @@
+package telemetryai
+
+import (
+	"strings"
+
+	scopedtraces "github.com/SigNoz/signoz/pkg/telemetryscopedtraces"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// genAIBaseConditionProvider: an AI trace has >=1 gen_ai LLM, tool, or agent span.
+type genAIBaseConditionProvider struct {
+	keys []string
+}
+
+var _ scopedtraces.BaseConditionProvider = (*genAIBaseConditionProvider)(nil)
+
+func newGenAIBaseConditionProvider() scopedtraces.BaseConditionProvider {
+	return &genAIBaseConditionProvider{
+		keys: []string{telemetrytypes.GenAIRequestModel, telemetrytypes.GenAIToolName, telemetrytypes.GenAIAgentName},
+	}
+}
+
+func (p *genAIBaseConditionProvider) FilterExpression() string {
+	parts := make([]string, 0, len(p.keys))
+	for _, k := range p.keys {
+		parts = append(parts, k+" EXISTS")
+	}
+	return strings.Join(parts, " OR ")
+}
+
+func (p *genAIBaseConditionProvider) FieldKeys() []*telemetrytypes.TelemetryFieldKey {
+	// Definitions come from GenAIFieldDefinitions so they can't drift from the
+	// canonical semconv keys; copy to take the address.
+	keys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(p.keys))
+	for _, k := range p.keys {
+		def := telemetrytypes.GenAIFieldDefinitions[k]
+		keys = append(keys, &def)
+	}
+	return keys
+}
+
+// genAIColumnProvider adds AI/LLM per-trace metrics on top of the common columns.
+type genAIColumnProvider struct{}
+
+var _ scopedtraces.ColumnProvider = (*genAIColumnProvider)(nil)
+
+func newGenAIColumnProvider() scopedtraces.ColumnProvider {
+	return &genAIColumnProvider{}
+}
+
+func (genAIColumnProvider) Columns() []scopedtraces.TraceColumn {
+	defs := telemetrytypes.GenAIFieldDefinitions
+	reqModel := defs[telemetrytypes.GenAIRequestModel]
+	toolName := defs[telemetrytypes.GenAIToolName]
+	inTok := defs[telemetrytypes.GenAIUsageInputTokens]
+	outTok := defs[telemetrytypes.GenAIUsageOutputTokens]
+	cost := defs[telemetrytypes.SignozGenAITotalCost]
+	inMsg := defs[telemetrytypes.GenAIInputMessages]
+	outMsg := defs[telemetrytypes.GenAIOutputMessages]
+
+	str := telemetrytypes.FieldDataTypeString
+	return append(scopedtraces.CommonTraceColumns(),
+		// LLM calls only (request model present), not the full gate.
+		scopedtraces.TraceColumn{Alias: "llm_call_count", Orderable: true, Expr: scopedtraces.CountExists(&reqModel)},
+		scopedtraces.TraceColumn{Alias: "tool_call_count", Orderable: true, Expr: scopedtraces.CountExists(&toolName)},
+		scopedtraces.TraceColumn{Alias: "distinct_tool_count", Orderable: true, Expr: scopedtraces.UniqCount(&toolName, str)},
+		// tokens live only on LLM spans, so a plain sum needs no gate scoping.
+		scopedtraces.TraceColumn{Alias: "input_tokens", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &inTok)},
+		scopedtraces.TraceColumn{Alias: "output_tokens", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &outTok)},
+		scopedtraces.TraceColumn{Alias: "total_tokens", Orderable: true, Expr: scopedtraces.SumOfKeys(telemetrytypes.FieldDataTypeFloat64, &inTok, &outTok)},
+		// per-span cost attached by the SigNoz LLM pricing processor.
+		scopedtraces.TraceColumn{Alias: "estimated_cost_usd", Orderable: true, Expr: scopedtraces.Reduce(scopedtraces.AggSum, &cost)},
+		// slowest single LLM call in the trace.
+		scopedtraces.TraceColumn{Alias: "max_llm_latency_ns", Orderable: true, Expr: scopedtraces.ScopedToKeyColumn(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("duration_nano"), &reqModel)},
+		// errors across the whole trace (any span), so display-only.
+		scopedtraces.TraceColumn{Alias: "error_count", Expr: scopedtraces.CondCount(scopedtraces.IntrinsicSpanKey("has_error"), qbtypes.FilterOperatorEqual, true)},
+		// timestamp of the last gen_ai span (LLM/tool/agent), hence gate-scoped.
+		scopedtraces.TraceColumn{Alias: "last_activity_time", Orderable: true, Expr: scopedtraces.ScopedReduce(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("timestamp"))},
+		// previews: first call's input (the prompt), last call's output (the answer).
+		scopedtraces.TraceColumn{Alias: "input", SpanLevel: true, Expr: scopedtraces.PickBy(&inMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickEarliest)},
+		scopedtraces.TraceColumn{Alias: "output", SpanLevel: true, Expr: scopedtraces.PickBy(&outMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickLatest)},
+	)
+}
+
+func (genAIColumnProvider) DefaultOrderAlias() string { return "last_activity_time" }
+
+func (p genAIColumnProvider) AggregateAliases() []string {
+	// Derived from Columns() so a new column can't be forgotten; SpanLevel columns
+	// are filtered span-level, so skip them.
+	cols := p.Columns()
+	aliases := make([]string, 0, len(cols))
+	for _, c := range cols {
+		if !c.SpanLevel {
+			aliases = append(aliases, c.Alias)
+		}
+	}
+	return aliases
+}

pkg/telemetryai/statement_builder.go

@@ -0,0 +1,21 @@
+package telemetryai
+
+import (
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/flagger"
+	scopedtraces "github.com/SigNoz/signoz/pkg/telemetryscopedtraces"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// NewAITraceStatementBuilder wires the generic scoped-trace builder with the gen_ai
+// gate and AI columns. This package holds only gen_ai domain knowledge; the query
+// topology lives in telemetryscopedtraces.
+func NewAITraceStatementBuilder(
+	settings factory.ProviderSettings,
+	metadataStore telemetrytypes.MetadataStore,
+	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
+	fl flagger.Flagger,
+) qbtypes.StatementBuilder[qbtypes.TraceAggregation] {
+	return scopedtraces.NewScopedTraceStatementBuilder(settings, metadataStore, newGenAIBaseConditionProvider(), newGenAIColumnProvider(), traceStmtBuilder, fl)
+}

pkg/telemetryai/statement_builder_test.go

@@ -0,0 +1,992 @@
+package telemetryai
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
+	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
+	"github.com/SigNoz/signoz/pkg/querybuilder"
+	scopedtraces "github.com/SigNoz/signoz/pkg/telemetryscopedtraces"
+	"github.com/SigNoz/signoz/pkg/telemetrytraces"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
+	"github.com/SigNoz/signoz/pkg/valuer"
+	"github.com/stretchr/testify/require"
+)
+
+// otelKeysMap seeds the OpenTelemetry gen_ai semantic-convention keys the AI
+// queries reference, so the metadata-backed field resolution succeeds in tests.
+func otelKeysMap() map[string][]*telemetrytypes.TelemetryFieldKey {
+	strKey := func(name string) *telemetrytypes.TelemetryFieldKey {
+		return &telemetrytypes.TelemetryFieldKey{
+			Name:          name,
+			Signal:        telemetrytypes.SignalTraces,
+			FieldContext:  telemetrytypes.FieldContextAttribute,
+			FieldDataType: telemetrytypes.FieldDataTypeString,
+		}
+	}
+	numKey := func(name string) *telemetrytypes.TelemetryFieldKey {
+		return &telemetrytypes.TelemetryFieldKey{
+			Name:          name,
+			Signal:        telemetrytypes.SignalTraces,
+			FieldContext:  telemetrytypes.FieldContextAttribute,
+			FieldDataType: telemetrytypes.FieldDataTypeFloat64,
+		}
+	}
+
+	m := make(map[string][]*telemetrytypes.TelemetryFieldKey)
+
+	// gen_ai semconv keys sourced from the single source of truth, mirroring what the
+	// production metadata store surfaces via enrichWithGenAIKeys.
+	for name, def := range telemetrytypes.GenAIFieldDefinitions {
+		keyCopy := def
+		m[name] = []*telemetrytypes.TelemetryFieldKey{&keyCopy}
+	}
+
+	// Extra keys these tests reference that aren't gen_ai semconv definitions.
+	m["gen_ai.user.id"] = []*telemetrytypes.TelemetryFieldKey{strKey("gen_ai.user.id")}
+	m["_signoz.gen_ai.total_cost"] = []*telemetrytypes.TelemetryFieldKey{numKey("_signoz.gen_ai.total_cost")}
+	m["gen_ai.usage.cached_input_tokens"] = []*telemetrytypes.TelemetryFieldKey{numKey("gen_ai.usage.cached_input_tokens")}
+	m["has_error"] = []*telemetrytypes.TelemetryFieldKey{{
+		Name:          "has_error",
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextSpan,
+		FieldDataType: telemetrytypes.FieldDataTypeBool,
+	}}
+	// service.name carries the resource-column evolutions like production metadata, so
+	// the rendered value expression prefers the JSON resource column over the legacy
+	// map (matching the standard traces builder tests).
+	m["service.name"] = []*telemetrytypes.TelemetryFieldKey{{
+		Name:          "service.name",
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextResource,
+		FieldDataType: telemetrytypes.FieldDataTypeString,
+		Evolutions:    resourceEvolutions(),
+	}}
+	return m
+}
+
+// resourceEvolutions is the canonical resource-column timeline: the legacy
+// resources_string map at epoch 0 and the JSON resource column released inside the
+// test window (mirrors telemetrytraces' mockEvolutionData).
+func resourceEvolutions() []*telemetrytypes.EvolutionEntry {
+	return []*telemetrytypes.EvolutionEntry{
+		{
+			Signal:       telemetrytypes.SignalTraces,
+			ColumnName:   "resources_string",
+			ColumnType:   "Map(LowCardinality(String), String)",
+			FieldContext: telemetrytypes.FieldContextResource,
+			FieldName:    "__all__",
+			ReleaseTime:  time.Unix(0, 0),
+		},
+		{
+			Signal:       telemetrytypes.SignalTraces,
+			ColumnName:   "resource",
+			ColumnType:   "JSON()",
+			FieldContext: telemetrytypes.FieldContextResource,
+			FieldName:    "__all__",
+			ReleaseTime:  time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC),
+		},
+	}
+}
+
+// standard test window (ms), matching the traces builder tests.
+const (
+	testStartMs = uint64(1747947419000)
+	testEndMs   = uint64(1747983448000)
+)
+
+func newTestBuilder(t *testing.T) qbtypes.StatementBuilder[qbtypes.TraceAggregation] {
+	return newTestBuilderWithKeys(t, otelKeysMap())
+}
+
+// newTestBuilderWithKeys mirrors the production wiring in signozquerier's provider.
+// The gen_ai keys are seeded via keysMap here; in production the metadata store
+// surfaces them itself (enrichWithGenAIKeys).
+func newTestBuilderWithKeys(t *testing.T, keysMap map[string][]*telemetrytypes.TelemetryFieldKey) qbtypes.StatementBuilder[qbtypes.TraceAggregation] {
+	t.Helper()
+	settings := instrumentationtest.New().ToProviderSettings()
+	fm := telemetrytraces.NewFieldMapper()
+	cb := telemetrytraces.NewConditionBuilder(fm)
+	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
+	mockMetadataStore.KeysMap = keysMap
+	fl := flaggertest.New(t)
+	// In production the metadata store enriches gen_ai keys (enrichWithGenAIKeys);
+	// here the mock is seeded directly via keysMap.
+	metadataStore := telemetrytypes.MetadataStore(mockMetadataStore)
+	rewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, nil, fl)
+	traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
+		settings,
+		metadataStore,
+		fm,
+		cb,
+		rewriter,
+		nil,
+		fl,
+		false,
+		100000,
+	)
+	return NewAITraceStatementBuilder(
+		settings,
+		metadataStore,
+		traceStmtBuilder,
+		fl,
+	)
+}
+
+// ---------------------------------------------------------------------------
+// Full-query golden tests
+//
+// Each pins the WHOLE generated statement, with bound args inlined into the `?`
+// placeholders, as ONE self-contained literal — so a failure diff shows the entire
+// query and the expected SQL can be copied straight into a ClickHouse client. The
+// `want` strings are formatted for readability; the comparison is whitespace- and
+// backtick-insensitive (see normalizeSQL), so only the SQL tokens themselves matter.
+//
+// The four trace-list goldens cover the corners of how `matched` is assembled —
+// {no span filter, span filter} × {no aggregate filter, aggregate filter} — plus a
+// mixed filter + multi-key order, plus the delegated span list. Note `matched` selects
+// only the aggregates ORDER BY / HAVING reference; the rest appear only in enrichment.
+//
+// Run `go test ./pkg/telemetryai/ -run TestBuild_FullSQL -v` to also print each query.
+// ---------------------------------------------------------------------------
+
+// renderSQL substitutes bound args into the `?` placeholders so the whole statement
+// reads as one literal SQL string.
+func renderSQL(t *testing.T, stmt *qbtypes.Statement) string {
+	t.Helper()
+	var b strings.Builder
+	argi := 0
+	for i := 0; i < len(stmt.Query); i++ {
+		if stmt.Query[i] == '?' {
+			require.Less(t, argi, len(stmt.Args), "more ? than args in query")
+			b.WriteString(formatArg(stmt.Args[argi]))
+			argi++
+			continue
+		}
+		b.WriteByte(stmt.Query[i])
+	}
+	require.Equal(t, len(stmt.Args), argi, "arg count does not match number of placeholders")
+	return b.String()
+}
+
+func formatArg(a any) string {
+	if s, ok := a.(string); ok {
+		return "'" + s + "'"
+	}
+	return fmt.Sprintf("%v", a)
+}
+
+// normalizeSQL makes the comparison insensitive to formatting: it drops identifier
+// backticks, collapses whitespace runs to a single space, and removes spaces directly
+// inside parentheses. This lets the golden strings be freely indented/wrapped (and
+// written as Go raw literals, which cannot contain backticks) — only the SQL tokens
+// and their order matter.
+func normalizeSQL(s string) string {
+	s = strings.Join(strings.Fields(strings.ReplaceAll(s, "`", "")), " ")
+	s = strings.ReplaceAll(s, "( ", "(")
+	s = strings.ReplaceAll(s, " )", ")")
+	return s
+}
+
+func requireSQLEqual(t *testing.T, want string, stmt *qbtypes.Statement) {
+	t.Helper()
+	got := renderSQL(t, stmt)
+	t.Logf("\n%s", got)
+	require.Equal(t, normalizeSQL(want), normalizeSQL(got))
+}
+
+// No filter: matched selects only the default order key (last_activity_time), WHERE is
+// just window + gate mask, no HAVING.
+func TestBuild_FullSQL_TraceList_NoFilter(t *testing.T) {
+	b := newTestBuilder(t)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI, Limit: 20,
+		}, nil)
+	require.NoError(t, err)
+
+	requireSQLEqual(t, `
+WITH matched AS (
+    SELECT trace_id,
+        maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time
+    FROM signoz_traces.distributed_signoz_index_v3
+    WHERE timestamp >= '1747947419000000000'
+      AND timestamp < '1747983448000000000'
+      AND ts_bucket_start >= 1747945619
+      AND ts_bucket_start <= 1747983448
+      AND ((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true))
+    GROUP BY trace_id
+    ORDER BY last_activity_time DESC, trace_id DESC
+    LIMIT 20
+),
+ranked AS (
+    SELECT trace_id, min(start) AS t_start, max(end) AS t_end
+    FROM signoz_traces.distributed_trace_summary
+    WHERE trace_id GLOBAL IN (SELECT trace_id FROM matched)
+      AND end >= fromUnixTimestamp64Nano(1747947419000000000)
+      AND start < fromUnixTimestamp64Nano(1747983448000000000)
+    GROUP BY trace_id
+),
+buckets AS (
+    SELECT DISTINCT b AS ts_bucket
+    FROM ranked
+    ARRAY JOIN range(toUInt64(intDiv(toUnixTimestamp(t_start), 1800) * 1800 - 1800), toUInt64(intDiv(toUnixTimestamp(t_end), 1800) * 1800 + 1800), 1800) AS b
+)
+SELECT trace_id,
+    min(timestamp) AS start_time,
+    max(timestamp) AS end_time,
+    (max(toUnixTimestamp64Nano(timestamp) + duration_nano) - min(toUnixTimestamp64Nano(timestamp))) AS trace_duration_nano,
+    count() AS span_count,
+    anyIf(name, parent_span_id = '') AS root_span_name,
+    any(multiIf(multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS service.name,
+    countIf(mapContains(attributes_string, 'gen_ai.request.model') = true) AS llm_call_count,
+    countIf(mapContains(attributes_string, 'gen_ai.tool.name') = true) AS tool_call_count,
+    uniqIf(multiIf(mapContains(attributes_string, 'gen_ai.tool.name') = true, attributes_string['gen_ai.tool.name'], NULL), mapContains(attributes_string, 'gen_ai.tool.name') = true) AS distinct_tool_count,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)) AS input_tokens,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+    coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)), 0) + coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)), 0) AS total_tokens,
+    sum(multiIf(mapContains(attributes_number, '_signoz.gen_ai.total_cost') = true, toFloat64(attributes_number['_signoz.gen_ai.total_cost']), NULL)) AS estimated_cost_usd,
+    maxIf(duration_nano, mapContains(attributes_string, 'gen_ai.request.model') = true) AS max_llm_latency_ns,
+    countIf(has_error = true) AS error_count,
+    maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time,
+    argMinIf(multiIf(mapContains(attributes_string, 'gen_ai.input.messages') = true, attributes_string['gen_ai.input.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.input.messages') = true) AS input,
+    argMaxIf(multiIf(mapContains(attributes_string, 'gen_ai.output.messages') = true, attributes_string['gen_ai.output.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.output.messages') = true) AS output
+FROM signoz_traces.distributed_signoz_index_v3
+WHERE ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)
+  AND trace_id GLOBAL IN (SELECT trace_id FROM ranked)
+GROUP BY trace_id
+ORDER BY last_activity_time DESC, trace_id DESC
+SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000
+`, stmt)
+}
+
+// Promotion: a materialized gen_ai attribute must resolve to its materialized column
+// everywhere it appears — gate mask, countIf/scoped existence, and value columns —
+// while un-promoted attributes stay in the attributes map, so one query mixes both
+// forms. Here gen_ai.request.model and gen_ai.usage.input_tokens are materialized:
+// the gate/llm_call_count/max_llm_latency use `..._exists`, input_tokens/total_tokens
+// use the materialized value column, and tool/output_tokens/cost/messages stay in the map.
+func TestBuild_FullSQL_TraceList_MaterializedColumns(t *testing.T) {
+	keys := otelKeysMap()
+	for _, name := range []string{"gen_ai.request.model", "gen_ai.usage.input_tokens"} {
+		for _, k := range keys[name] {
+			k.Materialized = true
+		}
+	}
+	b := newTestBuilderWithKeys(t, keys)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI, Limit: 20,
+		}, nil)
+	require.NoError(t, err)
+
+	requireSQLEqual(t, `
+WITH matched AS (
+    SELECT trace_id,
+        maxIf(timestamp, (attribute_string_gen_ai$$request$$model_exists = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time
+    FROM signoz_traces.distributed_signoz_index_v3
+    WHERE timestamp >= '1747947419000000000'
+      AND timestamp < '1747983448000000000'
+      AND ts_bucket_start >= 1747945619
+      AND ts_bucket_start <= 1747983448
+      AND ((attribute_string_gen_ai$$request$$model_exists = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true))
+    GROUP BY trace_id
+    ORDER BY last_activity_time DESC, trace_id DESC
+    LIMIT 20
+),
+ranked AS (
+    SELECT trace_id, min(start) AS t_start, max(end) AS t_end
+    FROM signoz_traces.distributed_trace_summary
+    WHERE trace_id GLOBAL IN (SELECT trace_id FROM matched)
+      AND end >= fromUnixTimestamp64Nano(1747947419000000000)
+      AND start < fromUnixTimestamp64Nano(1747983448000000000)
+    GROUP BY trace_id
+),
+buckets AS (
+    SELECT DISTINCT b AS ts_bucket
+    FROM ranked
+    ARRAY JOIN range(toUInt64(intDiv(toUnixTimestamp(t_start), 1800) * 1800 - 1800), toUInt64(intDiv(toUnixTimestamp(t_end), 1800) * 1800 + 1800), 1800) AS b
+)
+SELECT trace_id,
+    min(timestamp) AS start_time,
+    max(timestamp) AS end_time,
+    (max(toUnixTimestamp64Nano(timestamp) + duration_nano) - min(toUnixTimestamp64Nano(timestamp))) AS trace_duration_nano,
+    count() AS span_count,
+    anyIf(name, parent_span_id = '') AS root_span_name,
+    any(multiIf(multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS service.name,
+    countIf(attribute_string_gen_ai$$request$$model_exists = true) AS llm_call_count,
+    countIf(mapContains(attributes_string, 'gen_ai.tool.name') = true) AS tool_call_count,
+    uniqIf(multiIf(mapContains(attributes_string, 'gen_ai.tool.name') = true, attributes_string['gen_ai.tool.name'], NULL), mapContains(attributes_string, 'gen_ai.tool.name') = true) AS distinct_tool_count,
+    sum(multiIf(attribute_number_gen_ai$$usage$$input_tokens_exists = true, toFloat64(attribute_number_gen_ai$$usage$$input_tokens), NULL)) AS input_tokens,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+    coalesce(sum(multiIf(attribute_number_gen_ai$$usage$$input_tokens_exists = true, toFloat64(attribute_number_gen_ai$$usage$$input_tokens), NULL)), 0) + coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)), 0) AS total_tokens,
+    sum(multiIf(mapContains(attributes_number, '_signoz.gen_ai.total_cost') = true, toFloat64(attributes_number['_signoz.gen_ai.total_cost']), NULL)) AS estimated_cost_usd,
+    maxIf(duration_nano, attribute_string_gen_ai$$request$$model_exists = true) AS max_llm_latency_ns,
+    countIf(has_error = true) AS error_count,
+    maxIf(timestamp, (attribute_string_gen_ai$$request$$model_exists = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time,
+    argMinIf(multiIf(mapContains(attributes_string, 'gen_ai.input.messages') = true, attributes_string['gen_ai.input.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.input.messages') = true) AS input,
+    argMaxIf(multiIf(mapContains(attributes_string, 'gen_ai.output.messages') = true, attributes_string['gen_ai.output.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.output.messages') = true) AS output
+FROM signoz_traces.distributed_signoz_index_v3
+WHERE ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)
+  AND trace_id GLOBAL IN (SELECT trace_id FROM ranked)
+GROUP BY trace_id
+ORDER BY last_activity_time DESC, trace_id DESC
+SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000
+`, stmt)
+}
+
+// Span-level AND trace-level filter, order by the aggregate, pagination. matched selects
+// only output_tokens (the sole aggregate referenced by both ORDER BY and HAVING) — not
+// input_tokens/llm_call_count/last_activity_time. The span predicate widens the WHERE
+// prune and becomes a countIf(...) > 0 existence check alongside the gate countIf.
+func TestBuild_FullSQL_TraceList_SpanAndTraceFilter(t *testing.T) {
+	b := newTestBuilder(t)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "gen_ai.request.model = 'gpt-4o-mini' AND output_tokens > 1000"},
+			Order:  []qbtypes.OrderBy{{Key: qbtypes.OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "output_tokens"}}, Direction: qbtypes.OrderDirectionDesc}},
+			Limit:  10, Offset: 30,
+		}, nil)
+	require.NoError(t, err)
+
+	requireSQLEqual(t, `
+WITH matched AS (
+    SELECT trace_id,
+        sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens
+    FROM signoz_traces.distributed_signoz_index_v3
+    WHERE timestamp >= '1747947419000000000'
+      AND timestamp < '1747983448000000000'
+      AND ts_bucket_start >= 1747945619
+      AND ts_bucket_start <= 1747983448
+      AND ((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)
+        OR (attributes_string['gen_ai.request.model'] = 'gpt-4o-mini' AND mapContains(attributes_string, 'gen_ai.request.model') = true))
+    GROUP BY trace_id
+    HAVING countIf((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) > 0
+        AND countIf((attributes_string['gen_ai.request.model'] = 'gpt-4o-mini' AND mapContains(attributes_string, 'gen_ai.request.model') = true)) > 0
+        AND output_tokens > 1000
+    ORDER BY output_tokens DESC, trace_id DESC
+    LIMIT 10 OFFSET 30
+),
+ranked AS (
+    SELECT trace_id, min(start) AS t_start, max(end) AS t_end
+    FROM signoz_traces.distributed_trace_summary
+    WHERE trace_id GLOBAL IN (SELECT trace_id FROM matched)
+      AND end >= fromUnixTimestamp64Nano(1747947419000000000)
+      AND start < fromUnixTimestamp64Nano(1747983448000000000)
+    GROUP BY trace_id
+),
+buckets AS (
+    SELECT DISTINCT b AS ts_bucket
+    FROM ranked
+    ARRAY JOIN range(toUInt64(intDiv(toUnixTimestamp(t_start), 1800) * 1800 - 1800), toUInt64(intDiv(toUnixTimestamp(t_end), 1800) * 1800 + 1800), 1800) AS b
+)
+SELECT trace_id,
+    min(timestamp) AS start_time,
+    max(timestamp) AS end_time,
+    (max(toUnixTimestamp64Nano(timestamp) + duration_nano) - min(toUnixTimestamp64Nano(timestamp))) AS trace_duration_nano,
+    count() AS span_count,
+    anyIf(name, parent_span_id = '') AS root_span_name,
+    any(multiIf(multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS service.name,
+    countIf(mapContains(attributes_string, 'gen_ai.request.model') = true) AS llm_call_count,
+    countIf(mapContains(attributes_string, 'gen_ai.tool.name') = true) AS tool_call_count,
+    uniqIf(multiIf(mapContains(attributes_string, 'gen_ai.tool.name') = true, attributes_string['gen_ai.tool.name'], NULL), mapContains(attributes_string, 'gen_ai.tool.name') = true) AS distinct_tool_count,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)) AS input_tokens,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+    coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)), 0) + coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)), 0) AS total_tokens,
+    sum(multiIf(mapContains(attributes_number, '_signoz.gen_ai.total_cost') = true, toFloat64(attributes_number['_signoz.gen_ai.total_cost']), NULL)) AS estimated_cost_usd,
+    maxIf(duration_nano, mapContains(attributes_string, 'gen_ai.request.model') = true) AS max_llm_latency_ns,
+    countIf(has_error = true) AS error_count,
+    maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time,
+    argMinIf(multiIf(mapContains(attributes_string, 'gen_ai.input.messages') = true, attributes_string['gen_ai.input.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.input.messages') = true) AS input,
+    argMaxIf(multiIf(mapContains(attributes_string, 'gen_ai.output.messages') = true, attributes_string['gen_ai.output.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.output.messages') = true) AS output
+FROM signoz_traces.distributed_signoz_index_v3
+WHERE ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)
+  AND trace_id GLOBAL IN (SELECT trace_id FROM ranked)
+GROUP BY trace_id
+ORDER BY output_tokens DESC, trace_id DESC
+SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000
+`, stmt)
+}
+
+// Aggregate-only filter (no span filter). WHERE prune is NOT widened, there is no
+// gate/span countIf, just the aggregate HAVING. `trace.output_tokens` rewrites to the
+// output_tokens alias. matched selects output_tokens (HAVING) + last_activity_time (default order).
+func TestBuild_FullSQL_TraceList_AggregateFilterOnly(t *testing.T) {
+	b := newTestBuilder(t)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "trace.output_tokens > 1000"},
+			Limit:  20,
+		}, nil)
+	require.NoError(t, err)
+
+	requireSQLEqual(t, `
+WITH matched AS (
+    SELECT trace_id,
+        sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+        maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time
+    FROM signoz_traces.distributed_signoz_index_v3
+    WHERE timestamp >= '1747947419000000000'
+      AND timestamp < '1747983448000000000'
+      AND ts_bucket_start >= 1747945619
+      AND ts_bucket_start <= 1747983448
+      AND ((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true))
+    GROUP BY trace_id
+    HAVING output_tokens > 1000
+    ORDER BY last_activity_time DESC, trace_id DESC
+    LIMIT 20
+),
+ranked AS (
+    SELECT trace_id, min(start) AS t_start, max(end) AS t_end
+    FROM signoz_traces.distributed_trace_summary
+    WHERE trace_id GLOBAL IN (SELECT trace_id FROM matched)
+      AND end >= fromUnixTimestamp64Nano(1747947419000000000)
+      AND start < fromUnixTimestamp64Nano(1747983448000000000)
+    GROUP BY trace_id
+),
+buckets AS (
+    SELECT DISTINCT b AS ts_bucket
+    FROM ranked
+    ARRAY JOIN range(toUInt64(intDiv(toUnixTimestamp(t_start), 1800) * 1800 - 1800), toUInt64(intDiv(toUnixTimestamp(t_end), 1800) * 1800 + 1800), 1800) AS b
+)
+SELECT trace_id,
+    min(timestamp) AS start_time,
+    max(timestamp) AS end_time,
+    (max(toUnixTimestamp64Nano(timestamp) + duration_nano) - min(toUnixTimestamp64Nano(timestamp))) AS trace_duration_nano,
+    count() AS span_count,
+    anyIf(name, parent_span_id = '') AS root_span_name,
+    any(multiIf(multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS service.name,
+    countIf(mapContains(attributes_string, 'gen_ai.request.model') = true) AS llm_call_count,
+    countIf(mapContains(attributes_string, 'gen_ai.tool.name') = true) AS tool_call_count,
+    uniqIf(multiIf(mapContains(attributes_string, 'gen_ai.tool.name') = true, attributes_string['gen_ai.tool.name'], NULL), mapContains(attributes_string, 'gen_ai.tool.name') = true) AS distinct_tool_count,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)) AS input_tokens,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+    coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)), 0) + coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)), 0) AS total_tokens,
+    sum(multiIf(mapContains(attributes_number, '_signoz.gen_ai.total_cost') = true, toFloat64(attributes_number['_signoz.gen_ai.total_cost']), NULL)) AS estimated_cost_usd,
+    maxIf(duration_nano, mapContains(attributes_string, 'gen_ai.request.model') = true) AS max_llm_latency_ns,
+    countIf(has_error = true) AS error_count,
+    maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time,
+    argMinIf(multiIf(mapContains(attributes_string, 'gen_ai.input.messages') = true, attributes_string['gen_ai.input.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.input.messages') = true) AS input,
+    argMaxIf(multiIf(mapContains(attributes_string, 'gen_ai.output.messages') = true, attributes_string['gen_ai.output.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.output.messages') = true) AS output
+FROM signoz_traces.distributed_signoz_index_v3
+WHERE ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)
+  AND trace_id GLOBAL IN (SELECT trace_id FROM ranked)
+GROUP BY trace_id
+ORDER BY last_activity_time DESC, trace_id DESC
+SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000
+`, stmt)
+}
+
+// Span-only filter (no aggregate filter). WHERE is widened; HAVING has the gate + span
+// countIf pair but no trailing aggregate. `has_error = true` resolves to a
+// materialized-column predicate (not a map access). matched selects only the default order key.
+func TestBuild_FullSQL_TraceList_SpanFilterOnly(t *testing.T) {
+	b := newTestBuilder(t)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "has_error = true"},
+			Limit:  20,
+		}, nil)
+	require.NoError(t, err)
+
+	requireSQLEqual(t, `
+WITH matched AS (
+    SELECT trace_id,
+        maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time
+    FROM signoz_traces.distributed_signoz_index_v3
+    WHERE timestamp >= '1747947419000000000'
+      AND timestamp < '1747983448000000000'
+      AND ts_bucket_start >= 1747945619
+      AND ts_bucket_start <= 1747983448
+      AND ((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)
+        OR has_error = true)
+    GROUP BY trace_id
+    HAVING countIf((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) > 0
+        AND countIf(has_error = true) > 0
+    ORDER BY last_activity_time DESC, trace_id DESC
+    LIMIT 20
+),
+ranked AS (
+    SELECT trace_id, min(start) AS t_start, max(end) AS t_end
+    FROM signoz_traces.distributed_trace_summary
+    WHERE trace_id GLOBAL IN (SELECT trace_id FROM matched)
+      AND end >= fromUnixTimestamp64Nano(1747947419000000000)
+      AND start < fromUnixTimestamp64Nano(1747983448000000000)
+    GROUP BY trace_id
+),
+buckets AS (
+    SELECT DISTINCT b AS ts_bucket
+    FROM ranked
+    ARRAY JOIN range(toUInt64(intDiv(toUnixTimestamp(t_start), 1800) * 1800 - 1800), toUInt64(intDiv(toUnixTimestamp(t_end), 1800) * 1800 + 1800), 1800) AS b
+)
+SELECT trace_id,
+    min(timestamp) AS start_time,
+    max(timestamp) AS end_time,
+    (max(toUnixTimestamp64Nano(timestamp) + duration_nano) - min(toUnixTimestamp64Nano(timestamp))) AS trace_duration_nano,
+    count() AS span_count,
+    anyIf(name, parent_span_id = '') AS root_span_name,
+    any(multiIf(multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS service.name,
+    countIf(mapContains(attributes_string, 'gen_ai.request.model') = true) AS llm_call_count,
+    countIf(mapContains(attributes_string, 'gen_ai.tool.name') = true) AS tool_call_count,
+    uniqIf(multiIf(mapContains(attributes_string, 'gen_ai.tool.name') = true, attributes_string['gen_ai.tool.name'], NULL), mapContains(attributes_string, 'gen_ai.tool.name') = true) AS distinct_tool_count,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)) AS input_tokens,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+    coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)), 0) + coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)), 0) AS total_tokens,
+    sum(multiIf(mapContains(attributes_number, '_signoz.gen_ai.total_cost') = true, toFloat64(attributes_number['_signoz.gen_ai.total_cost']), NULL)) AS estimated_cost_usd,
+    maxIf(duration_nano, mapContains(attributes_string, 'gen_ai.request.model') = true) AS max_llm_latency_ns,
+    countIf(has_error = true) AS error_count,
+    maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time,
+    argMinIf(multiIf(mapContains(attributes_string, 'gen_ai.input.messages') = true, attributes_string['gen_ai.input.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.input.messages') = true) AS input,
+    argMaxIf(multiIf(mapContains(attributes_string, 'gen_ai.output.messages') = true, attributes_string['gen_ai.output.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.output.messages') = true) AS output
+FROM signoz_traces.distributed_signoz_index_v3
+WHERE ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)
+  AND trace_id GLOBAL IN (SELECT trace_id FROM ranked)
+GROUP BY trace_id
+ORDER BY last_activity_time DESC, trace_id DESC
+SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000
+`, stmt)
+}
+
+// Resource filter: a resource attribute in the filter is pulled into a __resource_filter
+// CTE (fingerprints matching the resource condition), and the `matched` scan is narrowed
+// by `resource_fingerprint GLOBAL IN (…)`. The resource key is dropped from the span
+// predicate (skipResourceFilter), so here there is no span-level existence check — the
+// prune stays the gate mask and the whole match is scoped to the resource fingerprints.
+func TestBuild_FullSQL_TraceList_ResourceFilter(t *testing.T) {
+	b := newTestBuilder(t)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "resource.service.name = 'checkout'"},
+			Limit:  20,
+		}, nil)
+	require.NoError(t, err)
+
+	requireSQLEqual(t, `
+WITH __resource_filter AS (
+    SELECT fingerprint
+    FROM signoz_traces.distributed_traces_v3_resource
+    WHERE (simpleJSONExtractString(labels, 'service.name') = 'checkout' AND labels LIKE '%service.name%' AND labels LIKE '%service.name":"checkout%')
+      AND seen_at_ts_bucket_start >= 1747945619
+      AND seen_at_ts_bucket_start <= 1747983448
+    GROUP BY fingerprint
+),
+matched AS (
+    SELECT trace_id,
+        maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time
+    FROM signoz_traces.distributed_signoz_index_v3
+    WHERE timestamp >= '1747947419000000000'
+      AND timestamp < '1747983448000000000'
+      AND ts_bucket_start >= 1747945619
+      AND ts_bucket_start <= 1747983448
+      AND ((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true))
+      AND resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)
+    GROUP BY trace_id
+    ORDER BY last_activity_time DESC, trace_id DESC
+    LIMIT 20
+),
+ranked AS (
+    SELECT trace_id, min(start) AS t_start, max(end) AS t_end
+    FROM signoz_traces.distributed_trace_summary
+    WHERE trace_id GLOBAL IN (SELECT trace_id FROM matched)
+      AND end >= fromUnixTimestamp64Nano(1747947419000000000)
+      AND start < fromUnixTimestamp64Nano(1747983448000000000)
+    GROUP BY trace_id
+),
+buckets AS (
+    SELECT DISTINCT b AS ts_bucket
+    FROM ranked
+    ARRAY JOIN range(toUInt64(intDiv(toUnixTimestamp(t_start), 1800) * 1800 - 1800), toUInt64(intDiv(toUnixTimestamp(t_end), 1800) * 1800 + 1800), 1800) AS b
+)
+SELECT trace_id,
+    min(timestamp) AS start_time,
+    max(timestamp) AS end_time,
+    (max(toUnixTimestamp64Nano(timestamp) + duration_nano) - min(toUnixTimestamp64Nano(timestamp))) AS trace_duration_nano,
+    count() AS span_count,
+    anyIf(name, parent_span_id = '') AS root_span_name,
+    any(multiIf(multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS service.name,
+    countIf(mapContains(attributes_string, 'gen_ai.request.model') = true) AS llm_call_count,
+    countIf(mapContains(attributes_string, 'gen_ai.tool.name') = true) AS tool_call_count,
+    uniqIf(multiIf(mapContains(attributes_string, 'gen_ai.tool.name') = true, attributes_string['gen_ai.tool.name'], NULL), mapContains(attributes_string, 'gen_ai.tool.name') = true) AS distinct_tool_count,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)) AS input_tokens,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+    coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)), 0) + coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)), 0) AS total_tokens,
+    sum(multiIf(mapContains(attributes_number, '_signoz.gen_ai.total_cost') = true, toFloat64(attributes_number['_signoz.gen_ai.total_cost']), NULL)) AS estimated_cost_usd,
+    maxIf(duration_nano, mapContains(attributes_string, 'gen_ai.request.model') = true) AS max_llm_latency_ns,
+    countIf(has_error = true) AS error_count,
+    maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time,
+    argMinIf(multiIf(mapContains(attributes_string, 'gen_ai.input.messages') = true, attributes_string['gen_ai.input.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.input.messages') = true) AS input,
+    argMaxIf(multiIf(mapContains(attributes_string, 'gen_ai.output.messages') = true, attributes_string['gen_ai.output.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.output.messages') = true) AS output
+FROM signoz_traces.distributed_signoz_index_v3
+WHERE ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)
+  AND trace_id GLOBAL IN (SELECT trace_id FROM ranked)
+GROUP BY trace_id
+ORDER BY last_activity_time DESC, trace_id DESC
+SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000
+`, stmt)
+}
+
+// Mixed filter (two span predicates AND'd into one existence check + an aggregate) with
+// a two-key order on different aggregates than the filter. matched selects input_tokens
+// + last_activity_time (ORDER BY) and output_tokens (HAVING) — three of four; llm_call_count is not.
+func TestBuild_FullSQL_TraceList_MixedFiltersMultiOrder(t *testing.T) {
+	b := newTestBuilder(t)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "gen_ai.request.model = 'gpt-4o' AND has_error = true AND output_tokens > 500"},
+			Order: []qbtypes.OrderBy{
+				{Key: qbtypes.OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "input_tokens"}}, Direction: qbtypes.OrderDirectionDesc},
+				{Key: qbtypes.OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "last_activity_time"}}, Direction: qbtypes.OrderDirectionAsc},
+			},
+			Limit: 15,
+		}, nil)
+	require.NoError(t, err)
+
+	requireSQLEqual(t, `
+WITH matched AS (
+    SELECT trace_id,
+        sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)) AS input_tokens,
+        sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+        maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time
+    FROM signoz_traces.distributed_signoz_index_v3
+    WHERE timestamp >= '1747947419000000000'
+      AND timestamp < '1747983448000000000'
+      AND ts_bucket_start >= 1747945619
+      AND ts_bucket_start <= 1747983448
+      AND ((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)
+        OR ((attributes_string['gen_ai.request.model'] = 'gpt-4o' AND mapContains(attributes_string, 'gen_ai.request.model') = true) AND has_error = true))
+    GROUP BY trace_id
+    HAVING countIf((mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) > 0
+        AND countIf(((attributes_string['gen_ai.request.model'] = 'gpt-4o' AND mapContains(attributes_string, 'gen_ai.request.model') = true) AND has_error = true)) > 0
+        AND output_tokens > 500
+    ORDER BY input_tokens DESC, last_activity_time ASC, trace_id DESC
+    LIMIT 15
+),
+ranked AS (
+    SELECT trace_id, min(start) AS t_start, max(end) AS t_end
+    FROM signoz_traces.distributed_trace_summary
+    WHERE trace_id GLOBAL IN (SELECT trace_id FROM matched)
+      AND end >= fromUnixTimestamp64Nano(1747947419000000000)
+      AND start < fromUnixTimestamp64Nano(1747983448000000000)
+    GROUP BY trace_id
+),
+buckets AS (
+    SELECT DISTINCT b AS ts_bucket
+    FROM ranked
+    ARRAY JOIN range(toUInt64(intDiv(toUnixTimestamp(t_start), 1800) * 1800 - 1800), toUInt64(intDiv(toUnixTimestamp(t_end), 1800) * 1800 + 1800), 1800) AS b
+)
+SELECT trace_id,
+    min(timestamp) AS start_time,
+    max(timestamp) AS end_time,
+    (max(toUnixTimestamp64Nano(timestamp) + duration_nano) - min(toUnixTimestamp64Nano(timestamp))) AS trace_duration_nano,
+    count() AS span_count,
+    anyIf(name, parent_span_id = '') AS root_span_name,
+    any(multiIf(multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.service.name IS NOT NULL, resource.service.name::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS service.name,
+    countIf(mapContains(attributes_string, 'gen_ai.request.model') = true) AS llm_call_count,
+    countIf(mapContains(attributes_string, 'gen_ai.tool.name') = true) AS tool_call_count,
+    uniqIf(multiIf(mapContains(attributes_string, 'gen_ai.tool.name') = true, attributes_string['gen_ai.tool.name'], NULL), mapContains(attributes_string, 'gen_ai.tool.name') = true) AS distinct_tool_count,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)) AS input_tokens,
+    sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)) AS output_tokens,
+    coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.input_tokens') = true, toFloat64(attributes_number['gen_ai.usage.input_tokens']), NULL)), 0) + coalesce(sum(multiIf(mapContains(attributes_number, 'gen_ai.usage.output_tokens') = true, toFloat64(attributes_number['gen_ai.usage.output_tokens']), NULL)), 0) AS total_tokens,
+    sum(multiIf(mapContains(attributes_number, '_signoz.gen_ai.total_cost') = true, toFloat64(attributes_number['_signoz.gen_ai.total_cost']), NULL)) AS estimated_cost_usd,
+    maxIf(duration_nano, mapContains(attributes_string, 'gen_ai.request.model') = true) AS max_llm_latency_ns,
+    countIf(has_error = true) AS error_count,
+    maxIf(timestamp, (mapContains(attributes_string, 'gen_ai.request.model') = true OR mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_string, 'gen_ai.agent.name') = true)) AS last_activity_time,
+    argMinIf(multiIf(mapContains(attributes_string, 'gen_ai.input.messages') = true, attributes_string['gen_ai.input.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.input.messages') = true) AS input,
+    argMaxIf(multiIf(mapContains(attributes_string, 'gen_ai.output.messages') = true, attributes_string['gen_ai.output.messages'], NULL), timestamp, mapContains(attributes_string, 'gen_ai.output.messages') = true) AS output
+FROM signoz_traces.distributed_signoz_index_v3
+WHERE ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)
+  AND trace_id GLOBAL IN (SELECT trace_id FROM ranked)
+GROUP BY trace_id
+ORDER BY input_tokens DESC, last_activity_time ASC, trace_id DESC
+SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000
+`, stmt)
+}
+
+// Span list (requestType raw): delegated to the traces builder with the gate ANDed
+// into the user filter, so only gen_ai spans matching the filter come back. Standard
+// span columns, single SELECT (no CTE pipeline).
+func TestBuild_FullSQL_SpanList_Raw(t *testing.T) {
+	b := newTestBuilder(t)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeRaw,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "gen_ai.request.model = 'gpt-4o-mini'"},
+			Limit:  10,
+		}, nil)
+	require.NoError(t, err)
+
+	requireSQLEqual(t, `
+SELECT timestamp AS __SELECT_KEY_0_timestamp, trace_id AS __SELECT_KEY_1_trace_id, span_id AS __SELECT_KEY_2_span_id,
+    trace_state AS __SELECT_KEY_3_trace_state, parent_span_id AS __SELECT_KEY_4_parent_span_id, flags AS __SELECT_KEY_5_flags,
+    name AS __SELECT_KEY_6_name, kind AS __SELECT_KEY_7_kind, kind_string AS __SELECT_KEY_8_kind_string, duration_nano AS __SELECT_KEY_9_duration_nano,
+    status_code AS __SELECT_KEY_10_status_code, status_message AS __SELECT_KEY_11_status_message,
+    status_code_string AS __SELECT_KEY_12_status_code_string, events AS __SELECT_KEY_13_events, links AS __SELECT_KEY_14_links,
+    response_status_code AS __SELECT_KEY_15_response_status_code, external_http_url AS __SELECT_KEY_16_external_http_url,
+    http_url AS __SELECT_KEY_17_http_url, external_http_method AS __SELECT_KEY_18_external_http_method,
+    http_method AS __SELECT_KEY_19_http_method, http_host AS __SELECT_KEY_20_http_host, db_name AS __SELECT_KEY_21_db_name,
+    db_operation AS __SELECT_KEY_22_db_operation, has_error AS __SELECT_KEY_23_has_error, is_remote AS __SELECT_KEY_24_is_remote,
+    attributes_string, attributes_number, attributes_bool, resources_string
+FROM signoz_traces.distributed_signoz_index_v3
+WHERE (((mapContains(attributes_string, 'gen_ai.request.model') = true
+        OR mapContains(attributes_string, 'gen_ai.tool.name') = true
+        OR mapContains(attributes_string, 'gen_ai.agent.name') = true))
+    AND ((attributes_string['gen_ai.request.model'] = 'gpt-4o-mini'
+        AND mapContains(attributes_string, 'gen_ai.request.model') = true)))
+  AND timestamp >= '1747947419000000000'
+  AND timestamp < '1747983448000000000'
+  AND ts_bucket_start >= 1747945619
+  AND ts_bucket_start <= 1747983448
+LIMIT 10
+`, stmt)
+}
+
+// ---------------------------------------------------------------------------
+// Behavior / branch tests not covered by the goldens above
+// ---------------------------------------------------------------------------
+
+// A filter mixing a resource attribute with a span-level and an aggregate condition:
+// the resource key routes into __resource_filter (fingerprint prune), the span key stays
+// as a countIf existence check, and the aggregate becomes a HAVING — all AND-combined.
+// service.name (resource context) comes from otelKeysMap.
+func TestBuild_TraceList_ResourcePlusSpanPlusAggregateFilter(t *testing.T) {
+	b := newTestBuilder(t)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "resource.service.name = 'checkout' AND has_error = true AND output_tokens > 1000"},
+			Limit:  10,
+		}, nil)
+	require.NoError(t, err)
+
+	got := renderSQL(t, stmt)
+	// resource condition -> fingerprint CTE + prune, not filtered on the span index
+	// (the service.name output column still reads the resource map, hence the = form).
+	require.Contains(t, got, "__resource_filter AS (")
+	require.Contains(t, got, "resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)")
+	require.NotContains(t, got, "resources_string['service.name'] = 'checkout'")
+	// span condition -> existence check in matched HAVING.
+	require.Contains(t, got, "countIf(has_error = true) > 0")
+	// aggregate condition -> HAVING on the matched aggregate alias.
+	require.Contains(t, got, "output_tokens")
+}
+
+// The resolver-unset (nil) fallback is covered in pkg/telemetryscopedtraces, which
+// can construct that builder state directly.
+
+// Trace-level and span-level predicates may not be OR-combined.
+func TestBuild_TraceList_TraceOrSpanMixRejected(t *testing.T) {
+	b := newTestBuilder(t)
+	query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Signal: telemetrytypes.SignalTraces,
+		Source: telemetrytypes.SourceAI,
+		Filter: &qbtypes.Filter{Expression: "trace.output_tokens > 1000 OR gen_ai.request.model = 'x'"},
+		Limit:  10,
+	}
+	_, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace, query, nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "cannot be combined")
+}
+
+// An output-only aggregate (span_count / trace_duration_nano) can be displayed but not
+// used in the aggregate filter or ORDER BY — it is not computable in the matched pass.
+func TestBuild_TraceList_OutputOnlyAggregateRejected(t *testing.T) {
+	b := newTestBuilder(t)
+
+	// filter by span_count -> rejected
+	_, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "span_count > 3"},
+		}, nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "span_count")
+
+	// order by trace_duration_nano -> rejected
+	_, err = b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Order: []qbtypes.OrderBy{{Key: qbtypes.OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "trace_duration_nano"}}, Direction: qbtypes.OrderDirectionDesc}},
+		}, nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "unsupported order key")
+}
+
+// duration_nano no longer names an aggregate (the trace column is trace_duration_nano),
+// so a bare filter on it is span-level like everywhere else in the product: the trace
+// matches when any span exceeds the duration.
+func TestBuild_TraceList_SpanDurationFilterIsSpanLevel(t *testing.T) {
+	keys := otelKeysMap()
+	keys["duration_nano"] = []*telemetrytypes.TelemetryFieldKey{{
+		Name:          "duration_nano",
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextSpan,
+		FieldDataType: telemetrytypes.FieldDataTypeNumber,
+	}}
+	b := newTestBuilderWithKeys(t, keys)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+			Filter: &qbtypes.Filter{Expression: "duration_nano > 1000000"},
+			Limit:  10,
+		}, nil)
+	require.NoError(t, err)
+
+	got := renderSQL(t, stmt)
+	require.Contains(t, got, "countIf(duration_nano > 1e+06) > 0")
+	require.NotContains(t, got, "HAVING trace_duration_nano")
+}
+
+// A HAVING referencing a non-aggregate column is rejected.
+func TestBuild_TraceList_Having_UnknownColumn(t *testing.T) {
+	b := newTestBuilder(t)
+	query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Signal: telemetrytypes.SignalTraces,
+		Source: telemetrytypes.SourceAI,
+		Having: &qbtypes.Having{Expression: "service.name > 1"}, // not an aggregate column
+		Limit:  10,
+	}
+	_, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace, query, nil)
+	require.Error(t, err)
+}
+
+// Ordering by an unknown key is rejected.
+func TestBuild_TraceList_UnsupportedOrderKey(t *testing.T) {
+	b := newTestBuilder(t)
+	query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Signal: telemetrytypes.SignalTraces,
+		Source: telemetrytypes.SourceAI,
+		Order: []qbtypes.OrderBy{
+			{Key: qbtypes.OrderByKey{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "http.request.method"}}, Direction: qbtypes.OrderDirectionDesc},
+		},
+	}
+	_, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace, query, nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "unsupported order key")
+}
+
+// With no limit set, the builder applies the default of 100.
+func TestBuild_TraceList_DefaultLimit(t *testing.T) {
+	b := newTestBuilder(t)
+	query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Signal: telemetrytypes.SignalTraces,
+		Source: telemetrytypes.SourceAI,
+	}
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace, query, nil)
+	require.NoError(t, err)
+	require.Contains(t, stmt.Query, "LIMIT ?")
+	require.Contains(t, stmt.Args, 100)
+}
+
+// Only trace list and span list (raw) are supported; distribution is not.
+func TestBuild_UnsupportedRequestType(t *testing.T) {
+	b := newTestBuilder(t)
+	query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Signal: telemetrytypes.SignalTraces,
+		Source: telemetrytypes.SourceAI,
+		Aggregations: []qbtypes.TraceAggregation{
+			{Expression: "count()"},
+		},
+	}
+	_, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeDistribution, query, nil)
+	require.ErrorIs(t, err, scopedtraces.ErrUnsupportedRequestType)
+}
+
+// A gate key ingested under several data types (e.g. string + number from a
+// misbehaving SDK) contributes ALL variants to the mask, OR-combined — not just
+// the first — matching the standard visitor's EXISTS handling.
+func TestBuild_TraceList_MultiVariantGateKey(t *testing.T) {
+	keys := otelKeysMap()
+	keys[telemetrytypes.GenAIToolName] = append(keys[telemetrytypes.GenAIToolName], &telemetrytypes.TelemetryFieldKey{
+		Name:          telemetrytypes.GenAIToolName,
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextAttribute,
+		FieldDataType: telemetrytypes.FieldDataTypeFloat64,
+	})
+	b := newTestBuilderWithKeys(t, keys)
+	stmt, err := b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+		qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+			Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI, Limit: 10,
+		}, nil)
+	require.NoError(t, err)
+
+	got := renderSQL(t, stmt)
+	require.Contains(t, got, "mapContains(attributes_string, 'gen_ai.tool.name') = true OR mapContains(attributes_number, 'gen_ai.tool.name') = true")
+}
+
+// `trace.` parses as the trace field context and marks a trace-level aggregate; the
+// legacy `tracefield.` spelling is explicitly rejected (filter and having alike), and
+// an output-only aggregate under the context gets the targeted rejection rather than
+// an unknown-span-field failure.
+func TestBuild_TraceList_TraceContextPrefix(t *testing.T) {
+	b := newTestBuilder(t)
+	build := func(q qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]) (*qbtypes.Statement, error) {
+		q.Signal, q.Source, q.Limit = telemetrytypes.SignalTraces, telemetrytypes.SourceAI, 20
+		return b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace, q, nil)
+	}
+
+	_, err := build(qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Filter: &qbtypes.Filter{Expression: "trace.output_tokens > 1000"}})
+	require.NoError(t, err)
+
+	_, err = build(qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Filter: &qbtypes.Filter{Expression: "tracefield.output_tokens > 1000"}})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), `use the "trace." prefix`)
+
+	_, err = build(qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Having: &qbtypes.Having{Expression: "tracefield.output_tokens > 1000"}})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), `use the "trace." prefix`)
+
+	_, err = build(qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Filter: &qbtypes.Filter{Expression: "trace.span_count > 3"}})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "cannot be used")
+}
+
+// Query variables in a trace-level condition are substituted into the HAVING (the
+// span path binds them via PrepareWhereClause; the HAVING is a text rewrite).
+func TestBuild_TraceList_VariableInAggregateFilter(t *testing.T) {
+	b := newTestBuilder(t)
+	build := func(expr string, vars map[string]qbtypes.VariableItem) (*qbtypes.Statement, error) {
+		return b.Build(context.Background(), valuer.UUID{}, testStartMs, testEndMs, qbtypes.RequestTypeTrace,
+			qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+				Signal: telemetrytypes.SignalTraces, Source: telemetrytypes.SourceAI,
+				Filter: &qbtypes.Filter{Expression: expr},
+				Limit:  20,
+			}, vars)
+	}
+
+	// scalar variable -> literal in HAVING
+	stmt, err := build("trace.output_tokens > $threshold",
+		map[string]qbtypes.VariableItem{"threshold": {Value: 700}})
+	require.NoError(t, err)
+	require.Contains(t, stmt.Query, "HAVING output_tokens > 700")
+
+	// list variable with IN
+	stmt, err = build("trace.llm_call_count IN $counts",
+		map[string]qbtypes.VariableItem{"counts": {Value: []any{1, 2}}})
+	require.NoError(t, err)
+	require.Contains(t, stmt.Query, "HAVING llm_call_count IN")
+
+	// dynamic __all__ -> condition dropped, no HAVING at all
+	stmt, err = build("trace.output_tokens > $threshold",
+		map[string]qbtypes.VariableItem{"threshold": {Type: qbtypes.DynamicVariableType, Value: "__all__"}})
+	require.NoError(t, err)
+	require.NotContains(t, stmt.Query, "HAVING")
+
+	// unresolved variable -> rejected, not compared as a literal
+	_, err = build("trace.output_tokens > $missing", map[string]qbtypes.VariableItem{"other": {Value: 1}})
+	require.Error(t, err)
+}

pkg/telemetrymetadata/metadata.go

@@ -1188,6 +1188,27 @@ func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFi
 	return keys
 }
 
+// enrichWithGenAIKeys adds keys that can be queried for GenAI signals, even though they have not been ingested yet.
+func enrichWithGenAIKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
+	for _, selector := range selectors {
+		if selector.Signal != telemetrytypes.SignalTraces && selector.Signal != telemetrytypes.SignalUnspecified {
+			continue
+		}
+		for name, def := range telemetrytypes.GenAIFieldDefinitions {
+			if len(keys[name]) > 0 {
+				continue // already resolved from ingested data
+			}
+			if !selectorMatchesIntrinsicField(selector, def) {
+				continue
+			}
+			keyCopy := def
+			keys[name] = []*telemetrytypes.TelemetryFieldKey{&keyCopy}
+		}
+	}
+
+	return keys
+}
+
 func selectorMatchesIntrinsicField(selector *telemetrytypes.FieldKeySelector, definition telemetrytypes.TelemetryFieldKey) bool {
 	if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != definition.FieldContext {
 		return false
@@ -1273,6 +1294,9 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, orgID valuer.UUID, fie
 
 	applyBackwardCompatibleKeys(mapOfKeys)
 	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, selectors)
+	if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
+		mapOfKeys = enrichWithGenAIKeys(mapOfKeys, selectors)
+	}
 
 	return mapOfKeys, complete, nil
 }
@@ -1351,6 +1375,9 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, orgID valuer.UUID
 
 	applyBackwardCompatibleKeys(mapOfKeys)
 	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, fieldKeySelectors)
+	if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
+		mapOfKeys = enrichWithGenAIKeys(mapOfKeys, fieldKeySelectors)
+	}
 
 	return mapOfKeys, complete, nil
 }

pkg/telemetrymetrics/const.go

@@ -3,7 +3,6 @@ package telemetrymetrics
 import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 
 var IntrinsicFields = []string{
-	"__normalized",
 	"temporality",
 	"metric_name",
 	"type",

pkg/telemetrymetrics/reduced_test.go

@@ -40,80 +40,80 @@ func TestReducedStatementBuilder(t *testing.T) {
 			name:  "gauge_sum_latest",
 			query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationLatest, metrictypes.SpaceAggregationSum),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, anyLast(last) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, argMax(value, unix_milli) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
-				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, anyLast(last) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT points.reduced_fingerprint AS fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, argMax(`sum_last`, unix_milli) AS per_series_value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points FINAL INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746997200000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000)},
 			},
 		},
 		{
 			name:  "gauge_avg_avg",
 			query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
-				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT points.reduced_fingerprint AS fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(`sum_last`) AS per_series_value, avg(`count_series`) AS per_series_weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points FINAL INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746997200000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000)},
 			},
 		},
 		{
 			name:  "gauge_min_min",
 			query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMin, metrictypes.SpaceAggregationMin),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(min) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(value) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`min`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
-				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(min) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(`min`) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points FINAL INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746997200000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000)},
 			},
 		},
 		{
 			name:  "gauge_max_max",
 			query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMax, metrictypes.SpaceAggregationMax),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(value) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`max`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
-				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(`max`) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points FINAL INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746997200000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000)},
 			},
 		},
 		{
 			name:  "counter_sum_rate",
 			query: reducedQuery("test.metric.sum", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationRate, metrictypes.SpaceAggregationSum),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) / 300 AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
-				Args:  []any{"test.metric.sum", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), 0, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), "test.metric.sum", uint64(1746999600000), uint64(1747172760000), false},
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(`sum`) / 300 AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points FINAL INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Args:  []any{"test.metric.sum", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric.sum", uint64(1746999600000), uint64(1747172760000), 0, "test.metric.sum", uint64(1746997200000), uint64(1747172760000), "test.metric.sum", uint64(1746999600000), uint64(1747172760000)},
 			},
 		},
 		{
 			name:  "counter_avg_increase",
 			query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationIncrease, metrictypes.SpaceAggregationAvg),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
-				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0, "test.metric", uint64(1746999600000), uint64(1747172760000), "test.metric", uint64(1746999600000), uint64(1747172760000), false},
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT points.reduced_fingerprint AS fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(`sum`) AS per_series_value, avg(`count_series`) AS per_series_weight FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points FINAL INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric", uint64(1746999600000), uint64(1747172760000), 0, "test.metric", uint64(1746997200000), uint64(1747172760000), "test.metric", uint64(1746999600000), uint64(1747172760000)},
 			},
 		},
 		{
 			name:  "counter_min_omitted",
 			query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationRate, metrictypes.SpaceAggregationMin),
 			expected: qbtypes.Statement{
-				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts",
-				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0},
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts",
+				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric", uint64(1746999600000), uint64(1747172760000), 0},
 			},
 		},
 		{
 			name:  "counter_max_omitted",
 			query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationRate, metrictypes.SpaceAggregationMax),
 			expected: qbtypes.Statement{
-				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts",
-				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0},
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts",
+				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric", uint64(1746999600000), uint64(1747172760000), 0},
 			},
 		},
 		{
 			name:  "histogram_p99",
 			query: reducedQuery("test.metric.bucket", metrictypes.HistogramType, metrictypes.Cumulative, metrictypes.TimeAggregationUnspecified, metrictypes.SpaceAggregationPercentile99),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(value) / 300 AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts, `le`), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts",
-				Args:  []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), false},
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(`sum`) / 300 AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points FINAL INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Args:  []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746997200000), uint64(1747172760000), "test.metric.bucket", uint64(1746999900000), uint64(1747172760000)},
 			},
 		},
 		{
 			name:  "summary_avg",
 			query: reducedQuery("test.metric", metrictypes.SummaryType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
-				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT points.reduced_fingerprint AS fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(`sum_last`) AS per_series_value, avg(`count_series`) AS per_series_weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points FINAL INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Args:  []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746997200000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000)},
 			},
 		},
 	}

pkg/telemetrymetrics/statement_builder.go

@@ -231,10 +231,18 @@ func (b *MetricQueryStatementBuilder) buildPipelineStatement(
 	if agg.Reduced && !useBuffer {
 		var tsCTE string
 		var tsArgs []any
-		if tsCTE, tsArgs, err = b.buildReducedTimeSeriesCTE(ctx, orgID, start, end, query, keys, variables); err != nil {
+		// time series rows are written on hour boundaries
+		tsStart := start - (start % oneHourInMilliseconds)
+		if tsCTE, tsArgs, err = b.buildReducedTimeSeriesCTE(ctx, orgID, tsStart, end, query, keys, variables); err != nil {
 			return nil, err
 		}
-		if temporalFrag, temporalArgs, ok := b.buildReducedTemporalAggregationCTE(start, end, query, tsCTE, tsArgs); ok {
+		if qbtypes.CanShortCircuitReduced(agg) {
+			// spatial_aggregation_cte directly, no per-series level
+			if spatialFrag, spatialArgs, ok := b.buildReducedSpatialAggFastPath(start, end, query, tsCTE, tsArgs); ok {
+				reducedFragments = []string{spatialFrag}
+				reducedArgs = [][]any{spatialArgs}
+			}
+		} else if temporalFrag, temporalArgs, ok := b.buildReducedTemporalAggregationCTE(start, end, query, tsCTE, tsArgs); ok {
 			spatialFrag, spatialArgs := b.buildReducedSpatialAggregationCTE(query)
 			reducedFragments = []string{temporalFrag, spatialFrag}
 			reducedArgs = [][]any{temporalArgs, spatialArgs}
@@ -265,7 +273,10 @@ func unionStatements(main, reduced *qbtypes.Statement, query qbtypes.QueryBuilde
 	for _, g := range query.GroupBy {
 		orderBy = fmt.Sprintf("`%s`, ", g.Name) + orderBy
 	}
-	q := fmt.Sprintf("SELECT * FROM (%s) UNION ALL SELECT * FROM (%s) ORDER BY %s", main.Query, reduced.Query, orderBy)
+	q := fmt.Sprintf(
+		"SELECT * FROM (%s) UNION ALL SELECT * FROM (%s) ORDER BY %s SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+		main.Query, reduced.Query, orderBy,
+	)
 	args := append(append([]any{}, main.Args...), reduced.Args...)
 	warnings := append(append([]string{}, main.Warnings...), reduced.Warnings...)
 	return &qbtypes.Statement{Query: q, Args: args, Warnings: warnings}, nil
@@ -314,7 +325,6 @@ func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE(
 		sb.In("metric_name", query.Aggregations[0].MetricName),
 		sb.GTE("unix_milli", start),
 		sb.LTE("unix_milli", end),
-		sb.EQ("__normalized", false),
 	)
 
 	if !preparedWhereClause.IsEmpty() {
@@ -327,6 +337,46 @@ func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE(
 	return fmt.Sprintf("(%s) AS filtered_time_series", q), args, nil
 }
 
+// buildReducedSpatialAggFastPath is the reduced analog of
+// buildTemporalAggDeltaFastPath: for combinations where the temporal and
+// spatial aggregations collapse (CanShortCircuitReduced), it emits the
+// spatial_aggregation_cte in one level with no per-series grouping, so shards
+// send one state per (step, group) instead of per (series, step, group).
+// FINAL still dedups recomputed 60s buckets at scan time.
+func (b *MetricQueryStatementBuilder) buildReducedSpatialAggFastPath(
+	start, end uint64,
+	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
+	timeSeriesCTE string,
+	timeSeriesCTEArgs []any,
+) (string, []any, bool) {
+	agg := query.Aggregations[0]
+	stepSec := int64(query.StepInterval.Seconds())
+
+	value, _, ok := ReducedValueColumn(agg.Type, agg.SpaceAggregation)
+	if !ok {
+		return "", nil, false
+	}
+
+	sb := sqlbuilder.NewSelectBuilder()
+	sb.Select(fmt.Sprintf("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts", stepSec))
+	for _, g := range query.GroupBy {
+		sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
+	}
+	sb.SelectMore(fmt.Sprintf("%s AS value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
+	sb.From(fmt.Sprintf("%s.%s AS points FINAL", DBName, WhichReducedSamplesTableToUse(agg.Type)))
+	sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
+	sb.Where(
+		sb.In("metric_name", agg.MetricName),
+		sb.GTE("unix_milli", start),
+		sb.LT("unix_milli", end),
+	)
+	sb.GroupBy("ts")
+	sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
+
+	q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)
+	return fmt.Sprintf("__spatial_aggregation_cte AS (%s)", q), args, true
+}
+
 func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
 	start, end uint64,
 	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
@@ -341,41 +391,31 @@ func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
 		return "", nil, false
 	}
 
-	// dedup recomputed buckets: latest computed_at wins per (series, 60s bucket)
-	dedup := sqlbuilder.NewSelectBuilder()
-	dedup.Select("reduced_fingerprint AS fingerprint", "unix_milli")
-	dedup.SelectMore(fmt.Sprintf("argMax(%s, computed_at) AS value", value))
-	if weight != "" {
-		dedup.SelectMore(fmt.Sprintf("argMax(%s, computed_at) AS weight", weight))
-	}
-	dedup.From(fmt.Sprintf("%s.%s", DBName, WhichReducedSamplesTableToUse(agg.Type)))
-	dedup.Where(
-		dedup.In("metric_name", agg.MetricName),
-		dedup.GTE("unix_milli", start),
-		dedup.LT("unix_milli", end),
-	)
-	dedup.GroupBy("reduced_fingerprint", "unix_milli")
-	dedupQuery, dedupArgs := dedup.BuildWithFlavor(sqlbuilder.ClickHouse)
-
+	// TODO(srikanthccv): add _5m/_30m tables similar to samples_v4
+	// and wire them up in querier before GA
 	sb := sqlbuilder.NewSelectBuilder()
-	sb.Select("fingerprint")
+	sb.Select("points.reduced_fingerprint AS fingerprint")
 	sb.SelectMore(fmt.Sprintf("toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(%d)) AS ts", stepSec))
 	for _, g := range query.GroupBy {
 		sb.SelectMore(fmt.Sprintf("`%s`", g.Name))
 	}
-	sb.SelectMore(fmt.Sprintf("%s AS per_series_value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec)))
+	sb.SelectMore(fmt.Sprintf("%s AS per_series_value", ReducedTimeAggregationColumn(agg.TimeAggregation, stepSec, value)))
 	if weight != "" {
 		// count_series is a series count, not additive over time, so the avg
 		// denominator is reduced with avg
-		sb.SelectMore("avg(weight) AS per_series_weight")
+		sb.SelectMore(fmt.Sprintf("avg(%s) AS per_series_weight", weight))
 	}
-	sb.From(fmt.Sprintf("(%s) AS points", dedupQuery))
-	sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
+	sb.From(fmt.Sprintf("%s.%s AS points FINAL", DBName, WhichReducedSamplesTableToUse(agg.Type)))
+	sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
+	sb.Where(
+		sb.In("metric_name", agg.MetricName),
+		sb.GTE("unix_milli", start),
+		sb.LT("unix_milli", end),
+	)
 	sb.GroupBy("fingerprint", "ts")
 	sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
 
-	initArgs := append(append([]any{}, dedupArgs...), timeSeriesCTEArgs...)
-	q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, initArgs...)
+	q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)
 	return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, true
 }
 
@@ -510,11 +550,6 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
 		sb.Where(sb.ILike("temporality", query.Aggregations[0].Temporality.StringValue()))
 	}
 
-	// TODO configurable if we don't rollout the new un-normalized metrics
-	sb.Where(
-		sb.EQ("__normalized", false),
-	)
-
 	// the buffer holds both raw rows and the reduced catalog rows; the raw read
 	// only wants the original series
 	if tsTable == TimeseriesV4BufferLocalTableName {

pkg/telemetrymetrics/stmt_builder_test.go

@@ -51,8 +51,8 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",
-				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", false, "cartservice", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",
+				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", "cartservice", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},
 			},
 			expectedErr: nil,
 		},
@@ -84,8 +84,8 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND (match(JSONExtractString(labels, 'materialized.key.name'), ?) OR JSONExtractString(labels, 'service.name') = ?) GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",
-				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", false, "cartservice", "cartservice", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND (match(JSONExtractString(labels, 'materialized.key.name'), ?) OR JSONExtractString(labels, 'service.name') = ?) GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",
+				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", "cartservice", "cartservice", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},
 			},
 			expectedErr: nil,
 		},
@@ -117,8 +117,8 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",
-				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "delta", false, "cartservice", "signoz_calls_total", uint64(1747947390000), uint64(1747983420000)},
+				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `service.name`, ts",
+				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "delta", "cartservice", "signoz_calls_total", uint64(1747947390000), uint64(1747983420000)},
 			},
 			expectedErr: nil,
 		},
@@ -150,8 +150,8 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts ORDER BY `service.name`, ts",
-				Args:  []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", false, "cartservice", "signoz_latency", uint64(1747947390000), uint64(1747983420000)},
+				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts ORDER BY `service.name`, ts",
+				Args:  []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "cartservice", "signoz_latency", uint64(1747947390000), uint64(1747983420000)},
 			},
 			expectedErr: nil,
 		},
@@ -183,8 +183,8 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint, `host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `host.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `host.name`, ts",
-				Args:  []any{"system.memory.usage", uint64(1747936800000), uint64(1747983420000), "unspecified", false, "big-data-node-1", "system.memory.usage", uint64(1747947390000), uint64(1747983420000), 0},
+				Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint, `host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `host.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `host.name`, ts",
+				Args:  []any{"system.memory.usage", uint64(1747936800000), uint64(1747983420000), "unspecified", "big-data-node-1", "system.memory.usage", uint64(1747947390000), uint64(1747983420000), 0},
 			},
 			expectedErr: nil,
 		},
@@ -213,8 +213,8 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts ORDER BY `service.name`, ts",
-				Args:  []any{"http_server_duration_bucket", uint64(1747936800000), uint64(1747983420000), "cumulative", false, "http_server_duration_bucket", uint64(1747947360000), uint64(1747983420000), 0},
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `service.name`, `le`) SELECT ts, `service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `service.name`, ts ORDER BY `service.name`, ts",
+				Args:  []any{"http_server_duration_bucket", uint64(1747936800000), uint64(1747983420000), "cumulative", "http_server_duration_bucket", uint64(1747947360000), uint64(1747983420000), 0},
 			},
 			expectedErr: nil,
 		},
@@ -245,8 +245,8 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `k8s.statefulset.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `k8s.statefulset.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'k8s.statefulset.name') AS `k8s.statefulset.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? AND JSONExtractString(labels, 'k8s.statefulset.name') = ? GROUP BY fingerprint, `k8s.statefulset.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `k8s.statefulset.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `k8s.statefulset.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `k8s.statefulset.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `k8s.statefulset.name`, ts",
-				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", false, "my-statefulset", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `k8s.statefulset.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `k8s.statefulset.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'k8s.statefulset.name') AS `k8s.statefulset.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'k8s.statefulset.name') = ? GROUP BY fingerprint, `k8s.statefulset.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `k8s.statefulset.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `k8s.statefulset.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `k8s.statefulset.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `k8s.statefulset.name`, ts",
+				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", "my-statefulset", "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},
 			},
 			expectedErr: nil,
 		},

pkg/telemetrymetrics/tables.go

@@ -393,25 +393,24 @@ func ReducedValueColumn(metricType metrictypes.Type, space metrictypes.SpaceAggr
 	return "", "", false
 }
 
-// ReducedTimeAggregationColumn applies the time aggregation to the reduced `value`
-// column over the step's 60s buckets. latest uses argMax over the bucket timestamp
-// (the buckets have no read order); rate divides the per-step sum by the step.
-func ReducedTimeAggregationColumn(timeAggregation metrictypes.TimeAggregation, stepSec int64) string {
+// ReducedTimeAggregationColumn applies the time aggregation to the reduced value
+// column over the step's 60s buckets.
+func ReducedTimeAggregationColumn(timeAggregation metrictypes.TimeAggregation, stepSec int64, value string) string {
 	switch timeAggregation {
 	case metrictypes.TimeAggregationLatest:
-		return "argMax(value, unix_milli)"
+		return fmt.Sprintf("argMax(%s, unix_milli)", value)
 	case metrictypes.TimeAggregationAvg:
-		return "avg(value)"
+		return fmt.Sprintf("avg(%s)", value)
 	case metrictypes.TimeAggregationMin:
-		return "min(value)"
+		return fmt.Sprintf("min(%s)", value)
 	case metrictypes.TimeAggregationMax:
-		return "max(value)"
+		return fmt.Sprintf("max(%s)", value)
 	case metrictypes.TimeAggregationCount:
-		return "count(value)"
+		return fmt.Sprintf("count(%s)", value)
 	case metrictypes.TimeAggregationRate:
-		return fmt.Sprintf("sum(value) / %d", stepSec)
+		return fmt.Sprintf("sum(%s) / %d", value, stepSec)
 	default: // sum, increase
-		return "sum(value)"
+		return fmt.Sprintf("sum(%s)", value)
 	}
 }
 

pkg/telemetryscopedtraces/field_mapper.go

@@ -0,0 +1,107 @@
+package telemetryscopedtraces
+
+import (
+	"context"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/querybuilder"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+// CommonTraceColumns are domain-neutral columns any trace list can reuse. All
+// aggregate over every span, so none is Orderable.
+func CommonTraceColumns() []TraceColumn {
+	ts := IntrinsicSpanKey("timestamp")
+	duration := IntrinsicSpanKey("duration_nano")
+	name := IntrinsicSpanKey("name")
+	parentSpanID := IntrinsicSpanKey("parent_span_id")
+	serviceName := &telemetrytypes.TelemetryFieldKey{
+		Name:          "service.name",
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextResource,
+		FieldDataType: telemetrytypes.FieldDataTypeString,
+	}
+	return []TraceColumn{
+		{Alias: "start_time", Expr: FieldReduce(AggMin, ts)},
+		{Alias: "end_time", Expr: FieldReduce(AggMax, ts)},
+		// Not plain "duration_nano": that name is the intrinsic span field, and an
+		// alias would shadow it — both in ClickHouse identifier resolution and in
+		// bare-name filter classification.
+		{Alias: "trace_duration_nano", Expr: TraceDuration(ts, duration)},
+		{Alias: "span_count", Expr: CountAll()},
+		{Alias: "root_span_name", Expr: FieldAnyWhere(name, parentSpanID, qbtypes.FilterOperatorEqual, "")},
+		{Alias: "service.name", SpanLevel: true, Expr: AnyValue(serviceName, telemetrytypes.FieldDataTypeString)},
+	}
+}
+
+// fieldMapper resolves aggregate-column SQL through the shared field mapper and
+// condition builder, following their method shapes (FieldFor / ConditionFor / …) so
+// column resolution reads like the other statement builders. keys is the fetched
+// metadata for the keys the columns reference; the gate mask is set by the builder
+// after resolveMask (Scoped* aggregates embed it). All returned expressions are
+// escaped once, ready to embed in an outer builder.
+type fieldMapper struct {
+	fm       qbtypes.FieldMapper
+	cb       qbtypes.ConditionBuilder
+	keys     map[string][]*telemetrytypes.TelemetryFieldKey
+	maskExpr string
+	maskArgs []any
+}
+
+func newFieldMapper(fm qbtypes.FieldMapper, cb qbtypes.ConditionBuilder, keys map[string][]*telemetrytypes.TelemetryFieldKey) *fieldMapper {
+	return &fieldMapper{fm: fm, cb: cb, keys: keys}
+}
+
+// FieldFor returns the column expression for key via the field mapper.
+func (r *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
+	expr, err := r.fm.FieldFor(ctx, orgID, startNs, endNs, key)
+	if err != nil {
+		return "", err
+	}
+	return sqlbuilder.Escape(expr), nil
+}
+
+// ConditionFor returns a boolean predicate for key via the condition builder
+// (materialized column when present, else map access).
+func (r *fieldMapper) ConditionFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) (string, []any, error) {
+	resolvedKey := key
+	cands := r.keys[key.Name]
+	if len(cands) == 0 {
+		cands = []*telemetrytypes.TelemetryFieldKey{key}
+	} else {
+		resolvedKey = cands[0]
+	}
+	sb := sqlbuilder.NewSelectBuilder()
+	conds, _, err := r.cb.ConditionFor(ctx, orgID, startNs, endNs, resolvedKey, cands, op, value, sb)
+	if err != nil {
+		return "", nil, err
+	}
+	// One condition per candidate variant (a key can be ingested under several data
+	// types); OR them all, like the visitor does for EXISTS.
+	if len(conds) == 1 {
+		sb.Where(conds[0])
+	} else {
+		sb.Where(sb.Or(conds...))
+	}
+	expr, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	expr = strings.TrimPrefix(expr, "WHERE ")
+	return sqlbuilder.Escape(expr), args, nil
+}
+
+// ExistsFor returns the EXISTS predicate for key.
+func (r *fieldMapper) ExistsFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, []any, error) {
+	return r.ConditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorExists, nil)
+}
+
+// ValueFor returns the value expression for an attribute key. The metadata variant
+// is preferred because it carries Materialized — a provider's static definition
+// never does, so a promoted attribute would otherwise fall back to map access.
+func (r *fieldMapper) ValueFor(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) (string, []any, error) {
+	if cands := r.keys[key.Name]; len(cands) > 0 {
+		key = cands[0]
+	}
+	return querybuilder.CollisionHandledFinalExpr(ctx, orgID, startNs, endNs, key, r.fm, r.cb, r.keys, dt, nil, false)
+}

pkg/telemetryscopedtraces/provider.go

@@ -0,0 +1,247 @@
+package telemetryscopedtraces
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+// This file is the extension surface of the scoped trace builder: the two contracts a
+// span category implements (base condition + columns) and the Aggregate constructors
+// the columns are declared with. All SQL rendering goes through the fieldMapper.
+
+// BaseConditionProvider defines which spans are in scope. It only declares the gate
+// (a filter expression + its field keys); the builder resolves the keys through the
+// field mapper, so attribute access stays materialization-aware.
+type BaseConditionProvider interface {
+	// FilterExpression is the grammar-level (EXISTS) gate, used on the delegated
+	// span-list path.
+	FilterExpression() string
+	// FieldKeys are the gate's keys, used to build the per-span mask
+	// (OR of resolved EXISTS conditions).
+	FieldKeys() []*telemetrytypes.TelemetryFieldKey
+}
+
+// ColumnProvider supplies the columns a trace list computes.
+type ColumnProvider interface {
+	Columns() []TraceColumn
+	// DefaultOrderAlias is sorted by (desc) when the query gives no order.
+	DefaultOrderAlias() string
+	// AggregateAliases are the computed per-trace column names, used to classify a
+	// filter key as trace-level vs span-level. Excludes SpanLevel columns.
+	AggregateAliases() []string
+}
+
+// TraceColumn is one per-trace output column.
+type TraceColumn struct {
+	// Alias must not reuse a physical span-index column name (e.g. duration_nano):
+	// ClickHouse resolves bare identifiers to same-SELECT aliases first, so any
+	// expression referencing that column would silently bind to the alias.
+	Alias string
+	// Orderable columns can be used in ORDER BY and the aggregate filter. All-span
+	// aggregates (span_count, trace_duration_nano, …) are display-only and set false.
+	Orderable bool
+	// SpanLevel columns surface a real span/resource attribute (service.name,
+	// input/output messages); a filter on them is applied span-level, so they are
+	// excluded from AggregateAliases.
+	SpanLevel bool
+	Expr      Aggregate
+}
+
+// Aggregate renders one column's SQL through the fieldMapper and lists the attribute
+// keys it references so the builder can pre-fetch their metadata. Build one with the
+// constructors below; the zero value is not usable.
+type Aggregate struct {
+	keys   []*telemetrytypes.TelemetryFieldKey
+	render func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (expr string, args []any, err error)
+}
+
+// IntrinsicSpanKey references an intrinsic span-index field (timestamp, name, …) by
+// its canonical name; the field mapper resolves it to the physical column.
+func IntrinsicSpanKey(name string) *telemetrytypes.TelemetryFieldKey {
+	return &telemetrytypes.TelemetryFieldKey{
+		Name:         name,
+		Signal:       telemetrytypes.SignalTraces,
+		FieldContext: telemetrytypes.FieldContextSpan,
+	}
+}
+
+// AggFunc is a ClickHouse aggregate function name.
+type AggFunc string
+
+const (
+	AggSum AggFunc = "sum"
+	AggMax AggFunc = "max"
+	AggMin AggFunc = "min"
+)
+
+// PickDirection selects the earliest (argMin) or latest (argMax) span by ordering.
+type PickDirection int
+
+const (
+	PickLatest PickDirection = iota
+	PickEarliest
+)
+
+// CountAll renders count().
+func CountAll() Aggregate {
+	return Aggregate{render: func(context.Context, valuer.UUID, uint64, uint64, *fieldMapper) (string, []any, error) {
+		return "count()", nil, nil
+	}}
+}
+
+// FieldReduce renders <fn>(<field>) over a field-mapper-resolved column.
+func FieldReduce(fn AggFunc, key *telemetrytypes.TelemetryFieldKey) Aggregate {
+	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		f, err := m.FieldFor(ctx, orgID, startNs, endNs, key)
+		if err != nil {
+			return "", nil, err
+		}
+		return fmt.Sprintf("%s(%s)", fn, f), nil, nil
+	}}
+}
+
+// TraceDuration renders the full-trace wall duration: last span end minus first
+// span start.
+func TraceDuration(tsKey, durationKey *telemetrytypes.TelemetryFieldKey) Aggregate {
+	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		ts, err := m.FieldFor(ctx, orgID, startNs, endNs, tsKey)
+		if err != nil {
+			return "", nil, err
+		}
+		dur, err := m.FieldFor(ctx, orgID, startNs, endNs, durationKey)
+		if err != nil {
+			return "", nil, err
+		}
+		return fmt.Sprintf("(max(toUnixTimestamp64Nano(%s) + %s) - min(toUnixTimestamp64Nano(%s)))", ts, dur, ts), nil, nil
+	}}
+}
+
+// FieldAnyWhere renders anyIf(<field>, <cond>) — the field value from any span
+// matching the condition.
+func FieldAnyWhere(valueKey, condKey *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, condValue any) Aggregate {
+	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		v, err := m.FieldFor(ctx, orgID, startNs, endNs, valueKey)
+		if err != nil {
+			return "", nil, err
+		}
+		cond, args, err := m.ConditionFor(ctx, orgID, startNs, endNs, condKey, op, condValue)
+		return fmt.Sprintf("anyIf(%s, %s)", v, cond), args, err
+	}}
+}
+
+// AnyValue renders any(<value>) over a metadata-resolved attribute value.
+func AnyValue(key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) Aggregate {
+	return Aggregate{keys: keysOf(key), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		v, args, err := m.ValueFor(ctx, orgID, startNs, endNs, key, dt)
+		return fmt.Sprintf("any(%s)", v), args, err
+	}}
+}
+
+// CountExists renders countIf(<key> EXISTS) — counts spans carrying key.
+func CountExists(key *telemetrytypes.TelemetryFieldKey) Aggregate {
+	return Aggregate{keys: keysOf(key), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		cond, args, err := m.ExistsFor(ctx, orgID, startNs, endNs, key)
+		return fmt.Sprintf("countIf(%s)", cond), args, err
+	}}
+}
+
+// CondCount renders countIf(<cond>) over a condition-builder-resolved predicate.
+func CondCount(key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) Aggregate {
+	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		cond, args, err := m.ConditionFor(ctx, orgID, startNs, endNs, key, op, value)
+		return fmt.Sprintf("countIf(%s)", cond), args, err
+	}}
+}
+
+// Reduce renders <fn>(<value>) over a resolved numeric attribute value.
+func Reduce(fn AggFunc, valueKey *telemetrytypes.TelemetryFieldKey) Aggregate {
+	return Aggregate{keys: keysOf(valueKey), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		v, args, err := m.ValueFor(ctx, orgID, startNs, endNs, valueKey, telemetrytypes.FieldDataTypeFloat64)
+		return fmt.Sprintf("%s(%s)", fn, v), args, err
+	}}
+}
+
+// ScopedReduce renders <fn>If(<field>, <gate mask>) over a field-mapper-resolved column.
+func ScopedReduce(fn AggFunc, key *telemetrytypes.TelemetryFieldKey) Aggregate {
+	return Aggregate{render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		f, err := m.FieldFor(ctx, orgID, startNs, endNs, key)
+		if err != nil {
+			return "", nil, err
+		}
+		return fmt.Sprintf("%sIf(%s, %s)", fn, f, m.maskExpr), append([]any{}, m.maskArgs...), nil
+	}}
+}
+
+// ScopedToKeyColumn renders <fn>If(<field>, <scopeKey> EXISTS) — a span-index field
+// aggregated over spans carrying scopeKey (e.g. max LLM latency).
+func ScopedToKeyColumn(fn AggFunc, columnKey, scopeKey *telemetrytypes.TelemetryFieldKey) Aggregate {
+	return Aggregate{keys: keysOf(scopeKey), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		col, err := m.FieldFor(ctx, orgID, startNs, endNs, columnKey)
+		if err != nil {
+			return "", nil, err
+		}
+		cond, args, err := m.ExistsFor(ctx, orgID, startNs, endNs, scopeKey)
+		return fmt.Sprintf("%sIf(%s, %s)", fn, col, cond), args, err
+	}}
+}
+
+// PickBy renders argMinIf/argMaxIf(<value>, <orderField>, <value> EXISTS) — the value
+// from the earliest/latest span that carries it.
+func PickBy(valueKey *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType, orderKey *telemetrytypes.TelemetryFieldKey, dir PickDirection) Aggregate {
+	fn := "argMaxIf"
+	if dir == PickEarliest {
+		fn = "argMinIf"
+	}
+	return Aggregate{keys: keysOf(valueKey), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		v, vargs, err := m.ValueFor(ctx, orgID, startNs, endNs, valueKey, dt)
+		if err != nil {
+			return "", nil, err
+		}
+		order, err := m.FieldFor(ctx, orgID, startNs, endNs, orderKey)
+		if err != nil {
+			return "", nil, err
+		}
+		cond, cargs, err := m.ExistsFor(ctx, orgID, startNs, endNs, valueKey)
+		return fmt.Sprintf("%s(%s, %s, %s)", fn, v, order, cond), append(vargs, cargs...), err
+	}}
+}
+
+// UniqCount renders uniqIf(<value>, <value> EXISTS) — distinct count of an attribute.
+func UniqCount(valueKey *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) Aggregate {
+	return Aggregate{keys: keysOf(valueKey), render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		v, vargs, err := m.ValueFor(ctx, orgID, startNs, endNs, valueKey, dt)
+		if err != nil {
+			return "", nil, err
+		}
+		cond, cargs, err := m.ExistsFor(ctx, orgID, startNs, endNs, valueKey)
+		return fmt.Sprintf("uniqIf(%s, %s)", v, cond), append(vargs, cargs...), err
+	}}
+}
+
+// SumOfKeys renders coalesce(sum(<v1>), 0) + coalesce(sum(<v2>), 0) + … over several
+// numeric attributes. Coalesced because a key absent from every span sums to NULL and
+// NULL + n = NULL — a trace with only output tokens would otherwise total NULL.
+func SumOfKeys(dt telemetrytypes.FieldDataType, valueKeys ...*telemetrytypes.TelemetryFieldKey) Aggregate {
+	return Aggregate{keys: valueKeys, render: func(ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, m *fieldMapper) (string, []any, error) {
+		parts := make([]string, 0, len(valueKeys))
+		var args []any
+		for _, k := range valueKeys {
+			v, vargs, err := m.ValueFor(ctx, orgID, startNs, endNs, k, dt)
+			if err != nil {
+				return "", nil, err
+			}
+			parts = append(parts, fmt.Sprintf("coalesce(sum(%s), 0)", v))
+			args = append(args, vargs...)
+		}
+		return strings.Join(parts, " + "), args, nil
+	}}
+}
+
+func keysOf(k *telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
+	return []*telemetrytypes.TelemetryFieldKey{k}
+}

pkg/telemetryscopedtraces/statement_builder.go

@@ -0,0 +1,778 @@
+package telemetryscopedtraces
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"sort"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/flagger"
+	"github.com/SigNoz/signoz/pkg/querybuilder"
+	"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
+	"github.com/SigNoz/signoz/pkg/telemetrytraces"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+	qbvariables "github.com/SigNoz/signoz/pkg/variables"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+var (
+	ErrUnsupportedRequestType = errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type for the scoped trace builder")
+)
+
+// scopedTraceStatementBuilder builds a trace list scoped to one span category
+// (e.g. gen_ai spans). The query shape is fixed; BaseConditionProvider decides which
+// spans are in scope and ColumnProvider decides the per-trace columns, so a new
+// category only needs a new pair of providers.
+type scopedTraceStatementBuilder struct {
+	logger                    *slog.Logger
+	metadataStore             telemetrytypes.MetadataStore
+	fm                        qbtypes.FieldMapper
+	cb                        qbtypes.ConditionBuilder
+	baseCond                  BaseConditionProvider
+	columnProvider            ColumnProvider
+	traceStmtBuilder          qbtypes.StatementBuilder[qbtypes.TraceAggregation]
+	resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
+}
+
+var _ qbtypes.StatementBuilder[qbtypes.TraceAggregation] = (*scopedTraceStatementBuilder)(nil)
+
+// NewScopedTraceStatementBuilder wires the generic trace-list builder. The field
+// mapper / condition builder are built here, not injected — the list always scans the
+// telemetrytraces span index. traceStmtBuilder (the delegate for the span-list path)
+// is injected because the provider already has the canonical instance.
+func NewScopedTraceStatementBuilder(
+	settings factory.ProviderSettings,
+	metadataStore telemetrytypes.MetadataStore,
+	baseCond BaseConditionProvider,
+	columnProvider ColumnProvider,
+	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
+	fl flagger.Flagger,
+) qbtypes.StatementBuilder[qbtypes.TraceAggregation] {
+	scopedSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryscopedtraces")
+
+	fm := telemetrytraces.NewFieldMapper()
+	cb := telemetrytraces.NewConditionBuilder(fm)
+
+	// Same resource-fingerprint prune as the standard trace builder — the list scans
+	// the same span index.
+	resourceFilterStmtBuilder := telemetryresourcefilter.New[qbtypes.TraceAggregation](
+		settings,
+		telemetrytraces.DBName,
+		telemetrytraces.TracesResourceV3TableName,
+		telemetrytypes.SignalTraces,
+		telemetrytypes.SourceUnspecified,
+		metadataStore,
+		nil,
+		fl,
+	)
+
+	return &scopedTraceStatementBuilder{
+		logger:                    scopedSettings.Logger(),
+		metadataStore:             metadataStore,
+		fm:                        fm,
+		cb:                        cb,
+		baseCond:                  baseCond,
+		columnProvider:            columnProvider,
+		traceStmtBuilder:          traceStmtBuilder,
+		resourceFilterStmtBuilder: resourceFilterStmtBuilder,
+	}
+}
+
+func (b *scopedTraceStatementBuilder) Build(
+	ctx context.Context,
+	orgID valuer.UUID,
+	start uint64,
+	end uint64,
+	requestType qbtypes.RequestType,
+	query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
+	variables map[string]qbtypes.VariableItem,
+) (*qbtypes.Statement, error) {
+	switch requestType {
+	case qbtypes.RequestTypeTrace:
+		return b.buildTraceListQuery(ctx, orgID, querybuilder.ToNanoSecs(start), querybuilder.ToNanoSecs(end), query, variables)
+	case qbtypes.RequestTypeRaw:
+		return b.buildDelegated(ctx, orgID, start, end, requestType, query, variables)
+	default:
+		return nil, ErrUnsupportedRequestType
+	}
+}
+
+// buildDelegated ANDs the base gate into the user filter and delegates to the
+// standard trace builder (the span-list / raw path).
+func (b *scopedTraceStatementBuilder) buildDelegated(
+	ctx context.Context,
+	orgID valuer.UUID,
+	start, end uint64,
+	requestType qbtypes.RequestType,
+	query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
+	variables map[string]qbtypes.VariableItem,
+) (*qbtypes.Statement, error) {
+	gate := b.baseCond.FilterExpression()
+	expr := gate
+	if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
+		expr = fmt.Sprintf("(%s) AND (%s)", gate, query.Filter.Expression)
+	}
+
+	// shallow copy; only Filter is replaced, caller's query untouched
+	gated := query
+	gated.Filter = &qbtypes.Filter{Expression: expr}
+
+	return b.traceStmtBuilder.Build(ctx, orgID, start, end, requestType, gated, variables)
+}
+
+// buildTraceListQuery wires the CTE pipeline: one windowed pass picks the top-N
+// traces, then a bucket-pruned pass enriches only those.
+// Helpers appear in this file in the order they run. start/end are nanoseconds.
+//
+//	RESOLVE (keys/columns → SQL via the field mapper)
+//	  fetchKeys         metadata for every key we reference
+//	  resolveMask       the "span is in scope" predicate (OR of EXISTS)
+//	  resolveColumns    per-trace column SQL
+//	  resolveListOrders which columns to ORDER BY
+//	  splitFilter       span-level predicate + trace-level HAVING
+//
+//	BUILD
+//	  matched    one windowed, mask-pruned GROUP BY trace_id scan fusing gate + span
+//	     │       filter + HAVING + ORDER BY + LIMIT/OFFSET → the top-N trace_ids
+//	     ▼
+//	  ranked     [start,end] bounds of those traces, from the small summary table
+//	     ▼
+//	  buckets    the ts_bucket_start values they touch, to prune the next scan
+//	     ▼
+//	  enrichment every per-trace column for those traces over their full extent
+//	             (not window-clipped), scanning only their buckets
+//
+// Only Orderable columns are computable in the mask-pruned matched pass, so only they
+// can be ordered or filtered on; all-span columns (span_count, …) are output-only.
+func (b *scopedTraceStatementBuilder) buildTraceListQuery(
+	ctx context.Context,
+	orgID valuer.UUID,
+	start, end uint64,
+	query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
+	variables map[string]qbtypes.VariableItem,
+) (*qbtypes.Statement, error) {
+
+	startBucket := start/querybuilder.NsToSeconds - querybuilder.BucketAdjustment
+	endBucket := end / querybuilder.NsToSeconds
+
+	limit := query.Limit
+	if limit <= 0 {
+		limit = 100
+	}
+
+	// Resolve keys and columns once; all attribute access goes through the field mapper.
+	keys, err := b.fetchKeys(ctx, orgID)
+	if err != nil {
+		return nil, err
+	}
+	mapper := newFieldMapper(b.fm, b.cb, keys)
+	maskExpr, maskArgs, err := b.resolveMask(ctx, orgID, start, end, mapper)
+	if err != nil {
+		return nil, err
+	}
+	mapper.maskExpr, mapper.maskArgs = maskExpr, maskArgs
+	resolved, err := b.resolveColumns(ctx, orgID, start, end, mapper)
+	if err != nil {
+		return nil, err
+	}
+	orders, err := b.resolveListOrders(query.Order, resolved)
+	if err != nil {
+		return nil, err
+	}
+	orderableSet := orderableAliasSet(resolved)
+
+	// If the filter references resource attributes, add a __resource_filter CTE and
+	// narrow the matched scan by resource_fingerprint; the span predicate drops those
+	// keys so they aren't applied twice.
+	resourceFrag, resourceArgs, resourcePred, err := b.maybeAttachResourceFilter(ctx, orgID, query, start, end, variables)
+	if err != nil {
+		return nil, err
+	}
+
+	// Split the user filter: span-level predicate + trace-level HAVING expression.
+	fp, err := b.splitFilter(ctx, orgID, query, b.aggregateAliasSet(), orderableSet, start, end, variables)
+	if err != nil {
+		return nil, err
+	}
+
+	// matched → ranked → buckets → enrichment
+	matchedFrag, matchedArgs, err := b.buildMatchedCTE(start, end, startBucket, endBucket, resolved, orders, orderableSet, maskExpr, maskArgs, fp, resourcePred, limit, query.Offset)
+	if err != nil {
+		return nil, err
+	}
+	rankedFrag, rankedArgs := b.buildRankedCTE(start, end)
+	bucketsFrag := buildBucketsCTE()
+	mainSQL, mainArgs := b.buildEnrichmentSelect(resolved, orders)
+
+	cteFragments := []string{matchedFrag, rankedFrag, bucketsFrag}
+	cteArgs := [][]any{matchedArgs, rankedArgs, nil}
+
+	// __resource_filter must precede `matched`, which references it.
+	if resourceFrag != "" {
+		cteFragments = append([]string{resourceFrag}, cteFragments...)
+		cteArgs = append([][]any{resourceArgs}, cteArgs...)
+	}
+
+	finalSQL := querybuilder.CombineCTEs(cteFragments) + mainSQL + " SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000"
+	finalArgs := querybuilder.PrependArgs(cteArgs, mainArgs)
+
+	return &qbtypes.Statement{
+		Query:          finalSQL,
+		Args:           finalArgs,
+		Warnings:       fp.warnings,
+		WarningsDocURL: fp.warningsURL,
+	}, nil
+}
+
+// maybeAttachResourceFilter builds the __resource_filter CTE (fingerprints matching
+// the filter's resource conditions) and the predicate narrowing the span scan by
+// resource_fingerprint; with no resource conditions it returns empty fragments.
+//
+// Unlike the standard trace builder there is deliberately no skip-fingerprint
+// fallback: falling back would leave the resource conditions inside the OR'd
+// span-filter bucket, which changes trace membership (any span from the resource +
+// any gen_ai span, instead of a gen_ai span from the resource). Resource conditions
+// always scope the whole matched scan.
+func (b *scopedTraceStatementBuilder) maybeAttachResourceFilter(
+	ctx context.Context,
+	orgID valuer.UUID,
+	query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
+	start, end uint64,
+	variables map[string]qbtypes.VariableItem,
+) (cteFrag string, cteArgs []any, fingerprintPred string, err error) {
+	stmt, err := b.resourceFilterStmtBuilder.Build(
+		ctx, orgID, start, end, qbtypes.RequestTypeRaw, query, variables,
+	)
+	if err != nil {
+		return "", nil, "", err
+	}
+	if stmt == nil {
+		return "", nil, "", nil
+	}
+	return fmt.Sprintf("__resource_filter AS (%s)", stmt.Query), stmt.Args,
+		"resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)", nil
+}
+
+// ---------------------------------------------------------------------------
+// RESOLVE — turn keys/columns into field-mapper-aware SQL
+// ---------------------------------------------------------------------------
+
+func (b *scopedTraceStatementBuilder) fetchKeys(ctx context.Context, orgID valuer.UUID) (map[string][]*telemetrytypes.TelemetryFieldKey, error) {
+	fields := b.resolverFieldKeys()
+	selectors := make([]*telemetrytypes.FieldKeySelector, 0, len(fields))
+	for _, k := range fields {
+		selectors = append(selectors, &telemetrytypes.FieldKeySelector{
+			Name:              k.Name,
+			Signal:            k.Signal,
+			FieldContext:      k.FieldContext,
+			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
+		})
+	}
+	keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
+	return keys, err
+}
+
+func (b *scopedTraceStatementBuilder) resolverFieldKeys() []*telemetrytypes.TelemetryFieldKey {
+	seen := make(map[string]struct{})
+	var out []*telemetrytypes.TelemetryFieldKey
+	add := func(k *telemetrytypes.TelemetryFieldKey) {
+		if k == nil {
+			return
+		}
+		if _, dup := seen[k.Name]; dup {
+			return
+		}
+		seen[k.Name] = struct{}{}
+		out = append(out, k)
+	}
+	for _, k := range b.baseCond.FieldKeys() {
+		add(k)
+	}
+	for _, c := range b.columnProvider.Columns() {
+		for _, k := range c.Expr.keys {
+			add(k)
+		}
+	}
+	return out
+}
+
+// resolveMask builds the per-span in-scope mask: OR of resolved EXISTS predicates
+// over the base condition's field keys.
+func (b *scopedTraceStatementBuilder) resolveMask(ctx context.Context, orgID valuer.UUID, start, end uint64, mapper *fieldMapper) (string, []any, error) {
+	fieldKeys := b.baseCond.FieldKeys()
+	parts := make([]string, 0, len(fieldKeys))
+	var args []any
+	for _, key := range fieldKeys {
+		e, a, err := mapper.ExistsFor(ctx, orgID, start, end, key)
+		if err != nil {
+			return "", nil, err
+		}
+		parts = append(parts, e)
+		args = append(args, a...)
+	}
+	return "(" + strings.Join(parts, " OR ") + ")", args, nil
+}
+
+// resolvedColumn is a column resolved to SQL via the field mapper; expr is escaped
+// once, ready to embed in an outer SELECT.
+type resolvedColumn struct {
+	alias     string
+	expr      string
+	args      []any
+	orderable bool
+}
+
+// resolveColumns turns the declarative columns into SQL through the resolver, so all
+// attribute access goes through the field mapper / condition builder.
+func (b *scopedTraceStatementBuilder) resolveColumns(ctx context.Context, orgID valuer.UUID, start, end uint64, mapper *fieldMapper) ([]resolvedColumn, error) {
+	cols := b.columnProvider.Columns()
+	out := make([]resolvedColumn, 0, len(cols))
+	for _, c := range cols {
+		expr, args, err := c.Expr.render(ctx, orgID, start, end, mapper)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, resolvedColumn{alias: c.Alias, expr: expr, args: args, orderable: c.Orderable})
+	}
+	return out, nil
+}
+
+// listOrder is a sort key resolved to a column alias + direction; both the matched
+// CTE and the enrichment ORDER BY it.
+type listOrder struct {
+	alias     string
+	direction string
+}
+
+// resolveListOrders maps order keys to the resolved orderable columns; non-orderable
+// columns are rejected. Defaults to the column provider's default order.
+func (b *scopedTraceStatementBuilder) resolveListOrders(order []qbtypes.OrderBy, resolved []resolvedColumn) ([]listOrder, error) {
+	byAlias := make(map[string]resolvedColumn, len(resolved))
+	orderable := make([]string, 0, len(resolved))
+	for _, rc := range resolved {
+		byAlias[rc.alias] = rc
+		if rc.orderable {
+			orderable = append(orderable, rc.alias)
+		}
+	}
+
+	if len(order) == 0 {
+		return []listOrder{{alias: b.columnProvider.DefaultOrderAlias(), direction: "DESC"}}, nil
+	}
+
+	orders := make([]listOrder, 0, len(order))
+	for _, o := range order {
+		direction := "DESC"
+		if o.Direction == qbtypes.OrderDirectionAsc {
+			direction = "ASC"
+		}
+		rc, ok := byAlias[o.Key.Name]
+		if !ok || !rc.orderable {
+			return nil, errors.NewInvalidInputf(errors.CodeInvalidInput,
+				"unsupported order key %q for the trace list; orderable keys: %s", o.Key.Name, strings.Join(orderable, ", "))
+		}
+		orders = append(orders, listOrder{alias: rc.alias, direction: direction})
+	}
+	return orders, nil
+}
+
+// filterParts is the user filter split into a span-level predicate (widens the
+// matched WHERE prune and becomes a countIf existence check in HAVING) and a
+// trace-level HAVING expression.
+type filterParts struct {
+	spanPred      string
+	spanArgs      []any
+	hasSpanFilter bool
+	havingExpr    string
+	warnings      []string
+	warningsURL   string
+}
+
+// splitFilter splits query.Filter into a span-level predicate and a trace-level
+// HAVING expression (an explicit query.Having is ANDed onto the latter), then
+// validates the trace-level part against the matched-pass aggregates.
+func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID valuer.UUID, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], classifySet, orderableSet map[string]struct{}, start, end uint64, variables map[string]qbtypes.VariableItem) (filterParts, error) {
+	var fp filterParts
+	// The legacy tracefield. spelling parses identically to trace.; only the
+	// user-facing form is supported here.
+	if (query.Filter != nil && strings.Contains(query.Filter.Expression, "tracefield.")) ||
+		(query.Having != nil && strings.Contains(query.Having.Expression, "tracefield.")) {
+		return fp, errors.NewInvalidInputf(errors.CodeInvalidInput, "\"tracefield.\" is not supported; use the \"trace.\" prefix")
+	}
+	if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
+		spanExpr, traceExpr, err := querybuilder.SplitFilterForAggregates(query.Filter.Expression, classifySet)
+		if err != nil {
+			return fp, err
+		}
+		fp.havingExpr = traceExpr
+		if strings.TrimSpace(spanExpr) != "" {
+			pred, args, warnings, url, err := b.resolveSpanPredicate(ctx, orgID, start, end, spanExpr, variables)
+			if err != nil {
+				return fp, err
+			}
+			// pred is empty when the span-level keys were all resource attributes
+			// already handled by the __resource_filter CTE.
+			if strings.TrimSpace(pred) != "" {
+				fp.spanPred, fp.spanArgs, fp.hasSpanFilter = pred, args, true
+			}
+			fp.warnings, fp.warningsURL = warnings, url
+		}
+	}
+	if query.Having != nil && strings.TrimSpace(query.Having.Expression) != "" {
+		if fp.havingExpr != "" {
+			fp.havingExpr = fmt.Sprintf("(%s) AND (%s)", fp.havingExpr, query.Having.Expression)
+		} else {
+			fp.havingExpr = query.Having.Expression
+		}
+	}
+	// The span predicate binds variables via PrepareWhereClause; the HAVING is a plain
+	// text rewrite, so substitute variables here (list/IN quoting, __all__ drops the
+	// condition) before validating.
+	if strings.TrimSpace(fp.havingExpr) != "" && len(variables) > 0 {
+		replaced, err := qbvariables.ReplaceVariablesInExpression(fp.havingExpr, variables)
+		if err != nil {
+			return fp, err
+		}
+		fp.havingExpr = replaced
+	}
+	if err := validateAggregateFilter(fp.havingExpr, orderableSet); err != nil {
+		return fp, err
+	}
+	return fp, nil
+}
+
+// resolveSpanPredicate resolves a span-level filter expression to a bare boolean
+// SQL predicate + args via the field mapper.
+func (b *scopedTraceStatementBuilder) resolveSpanPredicate(ctx context.Context, orgID valuer.UUID, start, end uint64, expr string, variables map[string]qbtypes.VariableItem) (string, []any, []string, string, error) {
+	selectors := querybuilder.QueryStringToKeysSelectors(expr)
+	for i := range selectors {
+		selectors[i].Signal = telemetrytypes.SignalTraces
+	}
+	keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
+	if err != nil {
+		return "", nil, nil, "", err
+	}
+	prepared, err := querybuilder.PrepareWhereClause(expr, querybuilder.FilterExprVisitorOpts{
+		Context:          ctx,
+		Logger:           b.logger,
+		FieldMapper:      b.fm,
+		ConditionBuilder: b.cb,
+		FieldKeys:        keys,
+		// resource conditions are always handled by the __resource_filter CTE
+		SkipResourceFilter: true,
+		Variables:          variables,
+		StartNs:            start,
+		EndNs:              end,
+	})
+	if err != nil {
+		return "", nil, nil, "", err
+	}
+	if prepared.IsEmpty() {
+		return "", nil, nil, "", nil
+	}
+	sb := sqlbuilder.NewSelectBuilder()
+	sb.AddWhereClause(prepared.WhereClause)
+	sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	pred := sql[strings.Index(sql, "WHERE ")+len("WHERE "):]
+	return sqlbuilder.Escape(pred), args, prepared.Warnings, prepared.WarningsDocURL, nil
+}
+
+// buildMatchedCTE builds `matched`: the single windowed GROUP BY trace_id scan that
+// fuses gate + span filter + HAVING + ORDER BY + LIMIT/OFFSET, selecting only the
+// aliases the ORDER BY / HAVING reference.
+func (b *scopedTraceStatementBuilder) buildMatchedCTE(start, end, startBucket, endBucket uint64, resolved []resolvedColumn, orders []listOrder, orderableSet map[string]struct{}, maskExpr string, maskArgs []any, fp filterParts, resourcePred string, limit, offset int) (string, []any, error) {
+	sb := sqlbuilder.NewSelectBuilder()
+
+	// SELECT trace_id + only the aggregates ORDER BY / HAVING reference (as aliases).
+	needed := neededMatchedAliases(orders, fp.havingExpr, orderableSet)
+	selects := []string{"trace_id"}
+	for _, rc := range resolved {
+		if _, ok := needed[rc.alias]; !ok {
+			continue
+		}
+		colExpr, err := embedExpr(sb, rc.expr, rc.args)
+		if err != nil {
+			return "", nil, err
+		}
+		selects = append(selects, colExpr+" AS "+quoteAlias(rc.alias))
+	}
+	sb.Select(selects...)
+	sb.From(spanTable())
+
+	// WHERE: window + prune to in-scope spans, widened by the span filter so its
+	// spans survive for the countIf existence check below.
+	win := windowWhere(sb, start, end, startBucket, endBucket)
+	mask, err := embedExpr(sb, maskExpr, maskArgs)
+	if err != nil {
+		return "", nil, err
+	}
+	prune := "(" + mask
+	if fp.hasSpanFilter {
+		spanPred, err := embedExpr(sb, fp.spanPred, fp.spanArgs)
+		if err != nil {
+			return "", nil, err
+		}
+		prune += " OR " + spanPred
+	}
+	prune += ")"
+	where := append(win, prune)
+	if resourcePred != "" {
+		where = append(where, resourcePred)
+	}
+	sb.Where(where...)
+	sb.GroupBy("trace_id")
+
+	// HAVING: the gate/span existence checks are only needed when the WHERE was
+	// widened by a span filter; otherwise the mask alone already enforces the gate.
+	var having []string
+	if fp.hasSpanFilter {
+		havingMask, err := embedExpr(sb, maskExpr, maskArgs)
+		if err != nil {
+			return "", nil, err
+		}
+		havingPred, err := embedExpr(sb, fp.spanPred, fp.spanArgs)
+		if err != nil {
+			return "", nil, err
+		}
+		having = append(having, "countIf("+havingMask+") > 0")
+		having = append(having, "countIf("+havingPred+") > 0")
+	}
+	if strings.TrimSpace(fp.havingExpr) != "" {
+		hv, err := b.buildHaving(fp.havingExpr, orderableSet)
+		if err != nil {
+			return "", nil, err
+		}
+		if hv != "" {
+			having = append(having, hv)
+		}
+	}
+	if len(having) > 0 {
+		sb.Having(strings.Join(having, " AND "))
+	}
+
+	sb.OrderBy(orderClause(orders)...)
+	sb.Limit(limit)
+	if offset > 0 {
+		sb.Offset(offset)
+	}
+
+	sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return fmt.Sprintf("matched AS (%s)", sql), args, nil
+}
+
+// buildRankedCTE builds `ranked`: [start,end] bounds per matched trace, read from the
+// small trace-summary table.
+func (b *scopedTraceStatementBuilder) buildRankedCTE(start, end uint64) (string, []any) {
+	sb := sqlbuilder.NewSelectBuilder()
+	sb.Select("trace_id", "min(start) AS t_start", "max(end) AS t_end")
+	sb.From(summaryTable())
+	sb.Where(
+		"trace_id GLOBAL IN (SELECT trace_id FROM matched)",
+		"end >= fromUnixTimestamp64Nano("+sb.Var(start)+")",
+		"start < fromUnixTimestamp64Nano("+sb.Var(end)+")",
+	)
+	sb.GroupBy("trace_id")
+	sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return fmt.Sprintf("ranked AS (%s)", sql), args
+}
+
+// buildBucketsCTE builds `buckets`: the ts_bucket_start values the matched traces
+// span, so the enrichment scan is primary-key pruned. No args.
+func buildBucketsCTE() string {
+	adj := querybuilder.BucketAdjustment // 30-min bucket width in seconds
+	return fmt.Sprintf("buckets AS (SELECT DISTINCT b AS ts_bucket FROM ranked "+
+		"ARRAY JOIN range("+
+		"toUInt64(intDiv(toUnixTimestamp(t_start), %d) * %d - %d), "+
+		"toUInt64(intDiv(toUnixTimestamp(t_end), %d) * %d + %d), "+
+		"%d) AS b)", adj, adj, adj, adj, adj, adj, adj)
+}
+
+// buildEnrichmentSelect builds the final SELECT: every per-trace column for the
+// matched traces over their full extent, scanning only their buckets.
+//
+// Accepted discrepancy: matched ranks/paginates on window-clipped values (and, with a
+// resource filter, only over fingerprint-matching spans), while this pass recomputes
+// and ORDER BYs full-trace values — so a trace with activity outside the window or
+// resource can sort differently than it ranked. Page membership is unaffected
+// (LIMIT/OFFSET runs only in matched); rows still sort by the values the user sees.
+// Ordering by matched's values instead would re-run the matched scan (ClickHouse
+// re-executes a CTE per reference) without fixing the visible cross-page artifact.
+func (b *scopedTraceStatementBuilder) buildEnrichmentSelect(resolved []resolvedColumn, orders []listOrder) (string, []any) {
+	sb := sqlbuilder.NewSelectBuilder()
+	selects, selectArgs := selectAllColumns(resolved)
+	sb.Select(selects...)
+	sb.From(spanTable())
+	sb.Where(
+		"ts_bucket_start GLOBAL IN (SELECT ts_bucket FROM buckets)",
+		"trace_id GLOBAL IN (SELECT trace_id FROM ranked)",
+	)
+	sb.GroupBy("trace_id")
+	sb.OrderBy(orderClause(orders)...)
+	sql, builtArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return sql, append(append([]any{}, selectArgs...), builtArgs...)
+}
+
+// buildHaving rewrites a trace-level HAVING expression to the matched-pass column
+// aliases. The rewriter matches raw key text, so the trace. form is mapped alongside
+// the bare name (the legacy tracefield. spelling is rejected upfront in splitFilter).
+func (b *scopedTraceStatementBuilder) buildHaving(havingExpr string, orderableSet map[string]struct{}) (string, error) {
+	columnMap := make(map[string]string, len(orderableSet)*2)
+	for a := range orderableSet {
+		columnMap[a] = quoteAlias(a)
+		columnMap[telemetrytypes.FieldContextTrace.StringValue()+"."+a] = quoteAlias(a)
+	}
+	return querybuilder.NewHavingExpressionRewriter().Rewrite(havingExpr, columnMap)
+}
+
+// ---------------------------------------------------------------------------
+// Small shared SQL-builder utilities
+// ---------------------------------------------------------------------------
+
+// spanTable is the fully-qualified span index table.
+func spanTable() string {
+	return fmt.Sprintf("%s.%s", telemetrytraces.DBName, telemetrytraces.SpanIndexV3TableName)
+}
+
+// summaryTable is the fully-qualified trace-summary table.
+func summaryTable() string {
+	return fmt.Sprintf("%s.%s", telemetrytraces.DBName, telemetrytraces.TraceSummaryTableName)
+}
+
+// aggregateAliasSet is every trace-level column alias, used to classify filter keys
+// as trace-level vs span-level.
+func (b *scopedTraceStatementBuilder) aggregateAliasSet() map[string]struct{} {
+	set := make(map[string]struct{}, len(b.columnProvider.AggregateAliases()))
+	for _, a := range b.columnProvider.AggregateAliases() {
+		set[a] = struct{}{}
+	}
+	return set
+}
+
+// orderableAliasSet is the subset of aliases computable in the matched pass — the
+// only ones usable in ORDER BY and the aggregate filter.
+func orderableAliasSet(resolved []resolvedColumn) map[string]struct{} {
+	set := make(map[string]struct{})
+	for _, rc := range resolved {
+		if rc.orderable {
+			set[rc.alias] = struct{}{}
+		}
+	}
+	return set
+}
+
+// neededMatchedAliases is the minimal alias set the matched pass must select: those
+// in ORDER BY plus those in the aggregate HAVING. Everything else is left to the
+// enrichment scan.
+func neededMatchedAliases(orders []listOrder, havingExpr string, orderableSet map[string]struct{}) map[string]struct{} {
+	needed := make(map[string]struct{})
+	for _, o := range orders {
+		needed[o.alias] = struct{}{}
+	}
+	for _, name := range traceAggregateNames(havingExpr) {
+		if _, ok := orderableSet[name]; ok {
+			needed[name] = struct{}{}
+		}
+	}
+	return needed
+}
+
+// traceAggregateNames extracts the aggregate names a trace-level HAVING expression
+// references. QueryStringToKeysSelectors emits an extra attribute-context fallback
+// selector for context-prefixed keys (`trace.x` → attribute "trace.x"); only the
+// unspecified- and trace-context selectors name aggregates.
+func traceAggregateNames(havingExpr string) []string {
+	var names []string
+	for _, sel := range querybuilder.QueryStringToKeysSelectors(havingExpr) {
+		if sel.FieldContext == telemetrytypes.FieldContextUnspecified || sel.FieldContext == telemetrytypes.FieldContextTrace {
+			names = append(names, sel.Name)
+		}
+	}
+	return names
+}
+
+// validateAggregateFilter rejects a trace-level filter referencing an aggregate not
+// computable in the matched pass (e.g. span_count, trace_duration_nano).
+func validateAggregateFilter(havingExpr string, orderableSet map[string]struct{}) error {
+	if strings.TrimSpace(havingExpr) == "" {
+		return nil
+	}
+	allowed := make([]string, 0, len(orderableSet))
+	for a := range orderableSet {
+		allowed = append(allowed, a)
+	}
+	sort.Strings(allowed)
+	for _, name := range traceAggregateNames(havingExpr) {
+		if _, ok := orderableSet[name]; !ok {
+			return errors.NewInvalidInputf(errors.CodeInvalidInput,
+				"aggregate %q cannot be used in the trace-list filter; filterable aggregates: %s", name, strings.Join(allowed, ", "))
+		}
+	}
+	return nil
+}
+
+// embedExpr inlines a resolved expr into sb, replacing each `?` placeholder with a
+// builder Var so the args are tracked in appearance order. Resolved exprs carry
+// values only as bound args, so every `?` is a placeholder; a count mismatch would
+// silently shift args into the wrong slots — error out instead.
+func embedExpr(sb *sqlbuilder.SelectBuilder, expr string, args []any) (string, error) {
+	if n := strings.Count(expr, "?"); n != len(args) {
+		return "", errors.NewInternalf(errors.CodeInternal,
+			"scoped trace builder: %d placeholders != %d args embedding %q", n, len(args), expr)
+	}
+	var out strings.Builder
+	ai := 0
+	for i := 0; i < len(expr); i++ {
+		if expr[i] == '?' {
+			out.WriteString(sb.Var(args[ai]))
+			ai++
+			continue
+		}
+		out.WriteByte(expr[i])
+	}
+	return out.String(), nil
+}
+
+// windowWhere binds the time-window predicates to sb and returns them so the caller
+// can add its own predicates in the same Where call.
+func windowWhere(sb *sqlbuilder.SelectBuilder, start, end, startBucket, endBucket uint64) []string {
+	return []string{
+		sb.GE("timestamp", fmt.Sprintf("%d", start)),
+		sb.L("timestamp", fmt.Sprintf("%d", end)),
+		sb.GE("ts_bucket_start", startBucket),
+		sb.LE("ts_bucket_start", endBucket),
+	}
+}
+
+// orderClause renders the ORDER BY terms plus the trace_id tiebreak.
+func orderClause(orders []listOrder) []string {
+	out := make([]string, 0, len(orders)+1)
+	for _, o := range orders {
+		out = append(out, fmt.Sprintf("%s %s", quoteAlias(o.alias), o.direction))
+	}
+	return append(out, "trace_id DESC")
+}
+
+// selectAllColumns renders `expr AS alias` for every resolved column, args in select
+// order.
+func selectAllColumns(resolved []resolvedColumn) ([]string, []any) {
+	selects := []string{"trace_id"}
+	var args []any
+	for _, rc := range resolved {
+		selects = append(selects, rc.expr+" AS "+quoteAlias(rc.alias))
+		args = append(args, rc.args...)
+	}
+	return selects, args
+}
+
+// quoteAlias backticks an alias containing characters special to the SQL builder.
+func quoteAlias(alias string) string {
+	if strings.ContainsAny(alias, ".$`") {
+		return "`" + alias + "`"
+	}
+	return alias
+}

pkg/telemetryscopedtraces/statement_builder_test.go

@@ -0,0 +1,30 @@
+package telemetryscopedtraces
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/huandu/go-sqlbuilder"
+	"github.com/stretchr/testify/require"
+)
+
+// The full-pipeline golden tests live in pkg/telemetryai, which exercises this
+// builder through its production provider pair. The tests here cover only what
+// needs the package internals.
+
+// embedExpr treats every `?` byte as a placeholder; a count/args mismatch (an expr
+// carrying a literal `?`, or a dropped arg) must fail loudly instead of silently
+// shifting every subsequent arg into the wrong placeholder.
+func TestEmbedExpr_PlaceholderArgMismatch(t *testing.T) {
+	sb := sqlbuilder.NewSelectBuilder()
+
+	out, err := embedExpr(sb, "x = ? AND y = ?", []any{1, 2})
+	require.NoError(t, err)
+	require.Equal(t, 2, strings.Count(out, "$"), "both placeholders bound as builder vars")
+
+	_, err = embedExpr(sb, "x = ? AND y LIKE 'a?b'", []any{1})
+	require.Error(t, err, "literal ? in the expr must not pass as a placeholder")
+
+	_, err = embedExpr(sb, "x = ?", []any{1, 2})
+	require.Error(t, err, "extra args must not be silently dropped")
+}

pkg/types/llmpricingruletypes/pricing.go

@@ -16,18 +16,10 @@ import (
 const (
 	LLMCostFeatureType agentConf.AgentFeatureType = "llm_pricing"
 
-	GenAIRequestModel                  = "gen_ai.request.model"
-	GenAIProviderName                  = "gen_ai.provider.name"
-	GenAIUsageInputTokens              = "gen_ai.usage.input_tokens"
-	GenAIUsageOutputTokens             = "gen_ai.usage.output_tokens"
-	GenAIUsageCacheReadInputTokens     = "gen_ai.usage.cache_read.input_tokens"
-	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
-
 	SignozGenAICostInput      = "_signoz.gen_ai.cost_input"
 	SignozGenAICostOutput     = "_signoz.gen_ai.cost_output"
 	SignozGenAICostCacheRead  = "_signoz.gen_ai.cost_cache_read"
 	SignozGenAICostCacheWrite = "_signoz.gen_ai.cost_cache_write"
-	SignozGenAITotalCost      = "_signoz.gen_ai.total_cost"
 )
 
 var (

pkg/types/llmpricingruletypes/processorconfig.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"gopkg.in/yaml.v3"
 )
 
@@ -83,11 +84,11 @@ func buildProcessorConfig(rules []*LLMPricingRule) *LLMPricingRuleProcessorConfi
 
 	return &LLMPricingRuleProcessorConfig{
 		Attrs: LLMPricingRuleProcessorAttrs{
-			Model:      GenAIRequestModel,
-			In:         GenAIUsageInputTokens,
-			Out:        GenAIUsageOutputTokens,
-			CacheRead:  GenAIUsageCacheReadInputTokens,
-			CacheWrite: GenAIUsageCacheCreationInputTokens,
+			Model:      telemetrytypes.GenAIRequestModel,
+			In:         telemetrytypes.GenAIUsageInputTokens,
+			Out:        telemetrytypes.GenAIUsageOutputTokens,
+			CacheRead:  telemetrytypes.GenAIUsageCacheReadInputTokens,
+			CacheWrite: telemetrytypes.GenAIUsageCacheCreationInputTokens,
 		},
 		DefaultPricing: LLMPricingRuleProcessorDefaultPricing{
 			Rules: pricingRules,
@@ -97,7 +98,7 @@ func buildProcessorConfig(rules []*LLMPricingRule) *LLMPricingRuleProcessorConfi
 			Out:        SignozGenAICostOutput,
 			CacheRead:  SignozGenAICostCacheRead,
 			CacheWrite: SignozGenAICostCacheWrite,
-			Total:      SignozGenAITotalCost,
+			Total:      telemetrytypes.SignozGenAITotalCost,
 		},
 	}
 }

pkg/types/querybuildertypes/querybuildertypesv5/builder_query.go

@@ -256,3 +256,30 @@ func CanShortCircuitDelta(metricAgg MetricAggregation) bool {
 
 	return false
 }
+
+// CanShortCircuitReduced is like CanShortCircuitDelta but for reduced.
+func CanShortCircuitReduced(metricAgg MetricAggregation) bool {
+	if metricAgg.ValueFilter != nil {
+		return false
+	}
+
+	ta := metricAgg.TimeAggregation
+	sa := metricAgg.SpaceAggregation
+
+	if metricAgg.Type == metrictypes.SumType || metricAgg.Type == metrictypes.HistogramType {
+		return (ta == metrictypes.TimeAggregationRate || ta == metrictypes.TimeAggregationIncrease || ta == metrictypes.TimeAggregationSum) &&
+			sa == metrictypes.SpaceAggregationSum
+	}
+
+	if ta == metrictypes.TimeAggregationSum && sa == metrictypes.SpaceAggregationSum {
+		return true
+	}
+	if ta == metrictypes.TimeAggregationMin && sa == metrictypes.SpaceAggregationMin {
+		return true
+	}
+	if ta == metrictypes.TimeAggregationMax && sa == metrictypes.SpaceAggregationMax {
+		return true
+	}
+
+	return false
+}

pkg/types/querybuildertypes/querybuildertypesv5/validation.go

@@ -151,6 +151,10 @@ func (q *QueryBuilderQuery[T]) Validate(opts ...ValidationOption) error {
 		return err
 	}
 
+	if err := q.validateSource(); err != nil {
+		return err
+	}
+
 	if err := q.validateAggregations(cfg); err != nil {
 		return err
 	}
@@ -238,6 +242,18 @@ func (q *QueryBuilderQuery[T]) validateSignal() error {
 	}
 }
 
+func (q *QueryBuilderQuery[T]) validateSource() error {
+	if q.Source == telemetrytypes.SourceAI && q.Signal != telemetrytypes.SignalTraces {
+		return errors.NewInvalidInputf(
+			errors.CodeInvalidInput,
+			"source %q is only supported for the traces signal, got %q",
+			q.Source.StringValue(),
+			q.Signal.StringValue(),
+		)
+	}
+	return nil
+}
+
 func (q *QueryBuilderQuery[T]) validateAggregations(cfg validationConfig) error {
 	if cfg.skipAggregationValidation {
 		return nil

pkg/types/querybuildertypes/querybuildertypesv5/validation_test.go

@@ -1480,3 +1480,34 @@ func TestMetricAggregationValidateForType(t *testing.T) {
 		})
 	}
 }
+
+func TestQueryBuilderQuery_ValidateSource(t *testing.T) {
+	cases := []struct {
+		name    string
+		signal  telemetrytypes.Signal
+		source  telemetrytypes.Source
+		wantErr bool
+	}{
+		{name: "ai source on traces", signal: telemetrytypes.SignalTraces, source: telemetrytypes.SourceAI},
+		{name: "ai source on logs rejected", signal: telemetrytypes.SignalLogs, source: telemetrytypes.SourceAI, wantErr: true},
+		{name: "ai source on metrics rejected", signal: telemetrytypes.SignalMetrics, source: telemetrytypes.SourceAI, wantErr: true},
+		{name: "ai source on unspecified signal rejected", signal: telemetrytypes.SignalUnspecified, source: telemetrytypes.SourceAI, wantErr: true},
+		{name: "no source on logs", signal: telemetrytypes.SignalLogs, source: telemetrytypes.SourceUnspecified},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			q := QueryBuilderQuery[TraceAggregation]{Signal: tc.signal, Source: tc.source}
+			err := q.validateSource()
+			if tc.wantErr && err == nil {
+				t.Errorf("expected error, got nil")
+			}
+			if !tc.wantErr && err != nil {
+				t.Errorf("expected no error, got: %v", err)
+			}
+			if tc.wantErr && !contains(err.Error(), "only supported for the traces signal") {
+				t.Errorf("unexpected error message: %v", err)
+			}
+		})
+	}
+}

pkg/types/telemetrytypes/field_context.go

@@ -76,6 +76,7 @@ var (
 		"log":        FieldContextLog,
 		"metric":     FieldContextMetric,
 		"tracefield": FieldContextTrace,
+		"trace":      FieldContextTrace,
 	}
 )
 

pkg/types/telemetrytypes/genai_semconv.go

@@ -0,0 +1,43 @@
+package telemetrytypes
+
+// OpenTelemetry gen_ai semantic-convention attribute keys. Single source of truth
+// shared by the AI query builder and the LLM pricing pipeline.
+const (
+	GenAIRequestModel = "gen_ai.request.model"
+	GenAIToolName     = "gen_ai.tool.name"
+	GenAIAgentName    = "gen_ai.agent.name"
+	GenAIProviderName = "gen_ai.provider.name"
+
+	GenAIUsageInputTokens              = "gen_ai.usage.input_tokens"
+	GenAIUsageOutputTokens             = "gen_ai.usage.output_tokens"
+	GenAIUsageCacheReadInputTokens     = "gen_ai.usage.cache_read.input_tokens"
+	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
+
+	GenAIInputMessages  = "gen_ai.input.messages"
+	GenAIOutputMessages = "gen_ai.output.messages"
+
+	// SignozGenAITotalCost is not OTel semconv: it is the per-span total cost the
+	// SigNoz LLM pricing processor computes and attaches (see llmpricingruletypes).
+	SignozGenAITotalCost = "_signoz.gen_ai.total_cost"
+)
+
+// GenAIFieldDefinitions are the gen_ai semantic-convention span attributes the AI
+// query builder relies on. They are surfaced by the metadata store for trace
+// queries regardless of whether they have been ingested yet, so the AI gate/columns
+// resolve on a fresh install (mirrors intrinsic metric keys). String keys are the
+// gate; the usage keys are numeric.
+var GenAIFieldDefinitions = map[string]TelemetryFieldKey{
+	GenAIRequestModel: {Name: GenAIRequestModel, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
+	GenAIToolName:     {Name: GenAIToolName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
+	GenAIAgentName:    {Name: GenAIAgentName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
+	GenAIProviderName: {Name: GenAIProviderName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
+
+	GenAIUsageInputTokens:              {Name: GenAIUsageInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
+	GenAIUsageOutputTokens:             {Name: GenAIUsageOutputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
+	GenAIUsageCacheReadInputTokens:     {Name: GenAIUsageCacheReadInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
+	GenAIUsageCacheCreationInputTokens: {Name: GenAIUsageCacheCreationInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
+	SignozGenAITotalCost:               {Name: SignozGenAITotalCost, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
+
+	GenAIInputMessages:  {Name: GenAIInputMessages, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
+	GenAIOutputMessages: {Name: GenAIOutputMessages, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
+}

pkg/types/telemetrytypes/source.go

@@ -9,6 +9,7 @@ type Source struct {
 var (
 	SourceAudit       = Source{valuer.NewString("audit")}
 	SourceMeter       = Source{valuer.NewString("meter")}
+	SourceAI          = Source{valuer.NewString("ai")}
 	SourceUnspecified = Source{valuer.NewString("")}
 )
 
@@ -17,5 +18,6 @@ var (
 func (Source) Enum() []any {
 	return []any{
 		SourceMeter,
+		SourceAI,
 	}
 }

tests/conftest.py

@@ -10,7 +10,7 @@ pytest_plugins = [
     "fixtures.postgres",
     "fixtures.sql",
     "fixtures.sqlite",
-    "fixtures.zookeeper",
+    "fixtures.keeper",
     "fixtures.signoz",
     "fixtures.audit",
     "fixtures.logs",
@@ -71,18 +71,12 @@ def pytest_addoption(parser: pytest.Parser):
     parser.addoption(
         "--clickhouse-version",
         action="store",
-        default="25.5.6",
+        default="25.12.5",
         help="clickhouse version",
     )
-    parser.addoption(
-        "--zookeeper-version",
-        action="store",
-        default="3.7.1",
-        help="zookeeper version",
-    )
     parser.addoption(
         "--schema-migrator-version",
         action="store",
-        default="v0.144.3",
+        default="v0.144.6",
         help="schema migrator version",
     )

tests/fixtures/clickhouse.py

@@ -2,6 +2,7 @@ import os
 from collections.abc import Callable, Generator
 from datetime import datetime
 from typing import Any
+from uuid import uuid4
 
 import clickhouse_connect
 import clickhouse_connect.driver
@@ -17,30 +18,88 @@ from fixtures.logger import setup_logger
 
 logger = setup_logger(__name__)
 
+CLICKHOUSE_USERNAME = "signoz"
+CLICKHOUSE_PASSWORD = "password"
 
-@pytest.fixture(name="clickhouse", scope="package")
-def clickhouse(
-    tmpfs: Generator[types.LegacyPath, Any],
-    network: Network,
-    zookeeper: types.TestContainerDocker,
-    request: pytest.FixtureRequest,
-    pytestconfig: pytest.Config,
-) -> types.TestContainerClickhouse:
-    """
-    Package-scoped fixture for Clickhouse TestContainer.
+CUSTOM_FUNCTION_CONFIG = """
+<functions>
+    <function>
+        <type>executable</type>
+        <name>histogramQuantile</name>
+        <return_type>Float64</return_type>
+        <argument>
+            <type>Array(Float64)</type>
+            <name>buckets</name>
+        </argument>
+        <argument>
+            <type>Array(Float64)</type>
+            <name>counts</name>
+        </argument>
+        <argument>
+            <type>Float64</type>
+            <name>quantile</name>
+        </argument>
+        <format>CSV</format>
+        <command>./histogramQuantile</command>
+    </function>
+</functions>
+"""
+
+# Distributed inserts to a remote shard are async by default. We force
+# sycn at the profile level for deterministic tests.
+CLUSTER_USERS_CONFIG = """
+<clickhouse>
+    <profiles>
+        <default>
+            <insert_distributed_sync>1</insert_distributed_sync>
+        </default>
+    </profiles>
+</clickhouse>
+"""
+
+
+def render_remote_servers(shard_hosts: list[tuple[str, int]], secret: str | None = None) -> str:
+    """Render the <remote_servers> block for a cluster named `cluster` with one
+    single-replica shard per (host, port).
     """
+    shards = "".join(
+        f"""
+                <shard>
+                    <replica>
+                        <host>{host}</host>
+                        <port>{port}</port>
+                    </replica>
+                </shard>"""
+        for host, port in shard_hosts
+    )
 
-    def create() -> types.TestContainerClickhouse:
-        version = request.config.getoption("--clickhouse-version")
+    # Multi-node clusters need `secret` because distributed queries otherwise
+    # authenticate as the `default` user, which the docker entrypoint restricts
+    # to localhost when a custom user is configured.
+    secret_block = (
+        f"""
+                    <secret>{secret}</secret>"""
+        if secret
+        else ""
+    )
 
-        container = ClickHouseContainer(
-            image=f"clickhouse/clickhouse-server:{version}",
-            port=9000,
-            username="signoz",
-            password="password",
-        )
+    return f"""
+            <remote_servers>
+                <cluster>{secret_block}{shards}
+                </cluster>
+            </remote_servers>"""
 
-        cluster_config = f"""
+
+def render_node_config(
+    keeper_address: str,
+    keeper_port: int,
+    shard: str,
+    remote_servers: str,
+    distributed_ddl_path: str = "/clickhouse/task_queue/ddl",
+) -> str:
+    # <zookeeper> is ClickHouse's config section name for any coordination
+    # service, including ClickHouse Keeper.
+    return f"""
         <clickhouse>
             <logger>
                 <level>information</level>
@@ -55,33 +114,23 @@ def clickhouse(
             </logger>
 
             <macros>
-                <shard>01</shard>
+                <shard>{shard}</shard>
                 <replica>01</replica>
             </macros>
 
             <zookeeper>
                 <node>
-                    <host>{zookeeper.container_configs["2181"].address}</host>
-                    <port>{zookeeper.container_configs["2181"].port}</port>
+                    <host>{keeper_address}</host>
+                    <port>{keeper_port}</port>
                 </node>
             </zookeeper>
-
-            <remote_servers>
-                <cluster>
-                    <shard>
-                        <replica>
-                            <host>127.0.0.1</host>
-                            <port>9000</port>
-                        </replica>
-                    </shard>
-                </cluster>
-            </remote_servers>
+{remote_servers}
 
             <user_defined_executable_functions_config>*function.xml</user_defined_executable_functions_config>
             <user_scripts_path>/var/lib/clickhouse/user_scripts/</user_scripts_path>
 
             <distributed_ddl>
-                <path>/clickhouse/task_queue/ddl</path>
+                <path>{distributed_ddl_path}</path>
                 <profile>default</profile>
             </distributed_ddl>
 
@@ -122,38 +171,65 @@ def clickhouse(
         </clickhouse>
         """
 
-        custom_function_config = """
-        <functions>
-            <function>
-                <type>executable</type>
-                <name>histogramQuantile</name>
-                <return_type>Float64</return_type>
-                <argument>
-                    <type>Array(Float64)</type>
-                    <name>buckets</name>
-                </argument>
-                <argument>
-                    <type>Array(Float64)</type>
-                    <name>counts</name>
-                </argument>
-                <argument>
-                    <type>Float64</type>
-                    <name>quantile</name>
-                </argument>
-                <format>CSV</format>
-                <command>./histogramQuantile</command>
-            </function>
-        </functions>
-        """
 
-        tmp_dir = tmpfs("clickhouse")
+def install_histogram_quantile(container: ClickHouseContainer) -> None:
+    wrapped = container.get_wrapped_container()
+    exit_code, output = wrapped.exec_run(
+        [
+            "bash",
+            "-c",
+            (
+                'version="v0.0.1" && '
+                'node_os=$(uname -s | tr "[:upper:]" "[:lower:]") && '
+                "node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && "
+                "cd /tmp && "
+                'wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" && '
+                "tar -xzf histogram-quantile.tar.gz && "
+                "mkdir -p /var/lib/clickhouse/user_scripts && "
+                "mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile && "
+                "chmod +x /var/lib/clickhouse/user_scripts/histogramQuantile"
+            ),
+        ],
+    )
+    if exit_code != 0:
+        raise RuntimeError(f"Failed to install histogramQuantile binary: {output.decode()}")
+
+
+def create_clickhouse(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    keeper: types.TestContainerDocker,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    cache_key: str = "clickhouse",
+) -> types.TestContainerClickhouse:
+    coordinator = next(iter(keeper.container_configs.values()))
+
+    def create() -> types.TestContainerClickhouse:
+        clickhouse_version = request.config.getoption("--clickhouse-version")
+
+        container = ClickHouseContainer(
+            image=f"clickhouse/clickhouse-server:{clickhouse_version}",
+            port=9000,
+            username=CLICKHOUSE_USERNAME,
+            password=CLICKHOUSE_PASSWORD,
+        )
+
+        cluster_config = render_node_config(
+            keeper_address=coordinator.address,
+            keeper_port=coordinator.port,
+            shard="01",
+            remote_servers=render_remote_servers([("127.0.0.1", 9000)]),
+        )
+
+        tmp_dir = tmpfs(cache_key)
         cluster_config_file_path = os.path.join(tmp_dir, "cluster.xml")
         with open(cluster_config_file_path, "w", encoding="utf-8") as f:
             f.write(cluster_config)
 
         custom_function_file_path = os.path.join(tmp_dir, "custom-function.xml")
         with open(custom_function_file_path, "w", encoding="utf-8") as f:
-            f.write(custom_function_config)
+            f.write(CUSTOM_FUNCTION_CONFIG)
 
         container.with_volume_mapping(cluster_config_file_path, "/etc/clickhouse-server/config.d/cluster.xml")
         container.with_volume_mapping(
@@ -163,27 +239,7 @@ def clickhouse(
         container.with_network(network)
         container.start()
 
-        # Download and install the histogramQuantile binary
-        wrapped = container.get_wrapped_container()
-        exit_code, output = wrapped.exec_run(
-            [
-                "bash",
-                "-c",
-                (
-                    'version="v0.0.1" && '
-                    'node_os=$(uname -s | tr "[:upper:]" "[:lower:]") && '
-                    "node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && "
-                    "cd /tmp && "
-                    'wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" && '
-                    "tar -xzf histogram-quantile.tar.gz && "
-                    "mkdir -p /var/lib/clickhouse/user_scripts && "
-                    "mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile && "
-                    "chmod +x /var/lib/clickhouse/user_scripts/histogramQuantile"
-                ),
-            ],
-        )
-        if exit_code != 0:
-            raise RuntimeError(f"Failed to install histogramQuantile binary: {output.decode()}")
+        install_histogram_quantile(container)
 
         connection = clickhouse_connect.get_client(
             user=container.username,
@@ -253,7 +309,7 @@ def clickhouse(
     return reuse.wrap(
         request,
         pytestconfig,
-        "clickhouse",
+        cache_key,
         empty=lambda: types.TestContainerSQL(
             container=types.TestContainerDocker(id="", host_configs={}, container_configs={}),
             conn=None,
@@ -265,6 +321,211 @@ def clickhouse(
     )
 
 
+@pytest.fixture(name="clickhouse", scope="package")
+def clickhouse(
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    keeper: types.TestContainerDocker,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.TestContainerClickhouse:
+    """
+    Package-scoped fixture for Clickhouse TestContainer.
+    """
+    return create_clickhouse(
+        tmpfs=tmpfs,
+        network=network,
+        keeper=keeper,
+        request=request,
+        pytestconfig=pytestconfig,
+    )
+
+
+@pytest.fixture(name="clickhouse_node_conns", scope="function")
+def clickhouse_node_conns(
+    clickhouse: types.TestContainerClickhouse,
+) -> Generator[list[clickhouse_connect.driver.client.Client], Any]:
+    """Per-node clients (index 0 = the initiator) for asserting shard-local
+    state via the local, non-distributed tables. Empty for single-node
+    fixtures, which don't populate `nodes`."""
+    conns = [
+        clickhouse_connect.get_client(
+            user=clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME"],
+            password=clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD"],
+            host=node.host_configs["8123"].address,
+            port=node.host_configs["8123"].port,
+        )
+        for node in clickhouse.nodes
+    ]
+    yield conns
+    for conn in conns:
+        conn.close()
+
+
+def create_clickhouse_cluster(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    keeper: types.TestContainerDocker,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    cache_key: str = "clickhouse_cluster",
+    shards: int = 2,
+) -> types.TestContainerClickhouse:
+    """
+    To some extent, taken inspiration from how ClickHouse's own integration
+    harness composes real clusters: deterministic hostnames
+    (network aliases), per-node shard macros, and a shared cluster definition
+    named `cluster`.
+
+    `conn`/`env` point at node 1 i.e the initiator every query-service query and
+    migration goes through. Per-node containers are exposed via `nodes` so
+    tests can assert shard-local state.
+    """
+    coordinator = next(iter(keeper.container_configs.values()))
+
+    def create() -> types.TestContainerClickhouse:
+        clickhouse_version = request.config.getoption("--clickhouse-version")
+
+        # Unique aliases per creation: docker allows duplicate network aliases
+        # (DNS round-robin), so a stale cluster must never share names with a
+        # fresh one.
+        suffix = uuid4().hex[:6]
+        aliases = [f"signoz-ch-{suffix}-{i:02d}" for i in range(1, shards + 1)]
+        remote_servers = render_remote_servers([(alias, 9000) for alias in aliases], secret=cache_key)
+        # Own DDL queue path: the keeper instance may be shared with other
+        # environments under --reuse; its DDL queue stays separate.
+        distributed_ddl_path = f"/clickhouse/{cache_key}-{suffix}/task_queue/ddl"
+
+        nodes: list[types.TestContainerDocker] = []
+        started: list[ClickHouseContainer] = []
+        try:
+            for i, alias in enumerate(aliases, start=1):
+                node_config = render_node_config(
+                    keeper_address=coordinator.address,
+                    keeper_port=coordinator.port,
+                    shard=f"{i:02d}",
+                    remote_servers=remote_servers,
+                    distributed_ddl_path=distributed_ddl_path,
+                )
+
+                tmp_dir = tmpfs(f"clickhouse-{suffix}-{i:02d}")
+                cluster_config_file_path = os.path.join(tmp_dir, "cluster.xml")
+                with open(cluster_config_file_path, "w", encoding="utf-8") as f:
+                    f.write(node_config)
+                custom_function_file_path = os.path.join(tmp_dir, "custom-function.xml")
+                with open(custom_function_file_path, "w", encoding="utf-8") as f:
+                    f.write(CUSTOM_FUNCTION_CONFIG)
+                users_config_file_path = os.path.join(tmp_dir, "users.xml")
+                with open(users_config_file_path, "w", encoding="utf-8") as f:
+                    f.write(CLUSTER_USERS_CONFIG)
+
+                container = ClickHouseContainer(
+                    image=f"clickhouse/clickhouse-server:{clickhouse_version}",
+                    port=9000,
+                    username=CLICKHOUSE_USERNAME,
+                    password=CLICKHOUSE_PASSWORD,
+                )
+                container.with_volume_mapping(cluster_config_file_path, "/etc/clickhouse-server/config.d/cluster.xml")
+                container.with_volume_mapping(custom_function_file_path, "/etc/clickhouse-server/custom-function.xml")
+                container.with_volume_mapping(users_config_file_path, "/etc/clickhouse-server/users.d/integration-cluster.xml")
+                container.with_network(network)
+                container.with_network_aliases(alias)
+                container.start()
+                started.append(container)
+
+                install_histogram_quantile(container)
+
+                nodes.append(
+                    types.TestContainerDocker(
+                        id=container.get_wrapped_container().id,
+                        host_configs={
+                            "9000": types.TestContainerUrlConfig(
+                                "tcp",
+                                container.get_container_host_ip(),
+                                container.get_exposed_port(9000),
+                            ),
+                            "8123": types.TestContainerUrlConfig(
+                                "tcp",
+                                container.get_container_host_ip(),
+                                container.get_exposed_port(8123),
+                            ),
+                        },
+                        container_configs={
+                            "9000": types.TestContainerUrlConfig("tcp", alias, 9000),
+                            "8123": types.TestContainerUrlConfig("tcp", alias, 8123),
+                        },
+                    )
+                )
+        except Exception:
+            for container in started:
+                container.stop()
+            raise
+
+        connection = clickhouse_connect.get_client(
+            user=CLICKHOUSE_USERNAME,
+            password=CLICKHOUSE_PASSWORD,
+            host=nodes[0].host_configs["8123"].address,
+            port=nodes[0].host_configs["8123"].port,
+        )
+
+        return types.TestContainerClickhouse(
+            container=nodes[0],
+            conn=connection,
+            env={
+                "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN": f"tcp://{CLICKHOUSE_USERNAME}:{CLICKHOUSE_PASSWORD}@{aliases[0]}:{9000}",
+                "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME": CLICKHOUSE_USERNAME,
+                "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD": CLICKHOUSE_PASSWORD,
+                "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER": "cluster",
+            },
+            nodes=nodes,
+        )
+
+    def delete(resource: types.TestContainerClickhouse) -> None:
+        client = docker.from_env()
+        for node in resource.nodes or [resource.container]:
+            try:
+                client.containers.get(container_id=node.id).stop()
+                client.containers.get(container_id=node.id).remove(v=True)
+            except docker.errors.NotFound:
+                logger.info(
+                    "Skipping removal of Clickhouse cluster node, node(%s) not found. Maybe it was manually removed?",
+                    {"id": node.id},
+                )
+
+    def restore(cache: dict) -> types.TestContainerClickhouse:
+        nodes = [types.TestContainerDocker.from_cache(node) for node in cache["nodes"]]
+        env = cache["env"]
+        host_config = nodes[0].host_configs["8123"]
+
+        conn = clickhouse_connect.get_client(
+            user=env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME"],
+            password=env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD"],
+            host=host_config.address,
+            port=host_config.port,
+        )
+
+        return types.TestContainerClickhouse(
+            container=nodes[0],
+            conn=conn,
+            env=env,
+            nodes=nodes,
+        )
+
+    return reuse.wrap(
+        request,
+        pytestconfig,
+        cache_key,
+        empty=lambda: types.TestContainerClickhouse(
+            container=types.TestContainerDocker(id="", host_configs={}, container_configs={}),
+            conn=None,
+            env={},
+        ),
+        create=create,
+        delete=delete,
+        restore=restore,
+    )
+
+
 @pytest.fixture(name="check_query_log")
 def check_query_log(
     signoz: types.SigNoz,

tests/fixtures/keeper.py

@@ -0,0 +1,120 @@
+import os
+from collections.abc import Generator
+from typing import Any
+
+import docker
+import docker.errors
+import pytest
+from testcontainers.core.container import DockerContainer, Network
+
+from fixtures import reuse, types
+from fixtures.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+KEEPER_CONFIG = """
+<clickhouse>
+    <listen_host>0.0.0.0</listen_host>
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse-keeper/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>warning</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>localhost</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</clickhouse>
+"""
+
+
+def create_clickhouse_keeper(
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+    cache_key: str = "clickhousekeeper",
+) -> types.TestContainerDocker:
+
+    def create() -> types.TestContainerDocker:
+        keeper_version = request.config.getoption("--clickhouse-version")
+
+        tmp_dir = tmpfs(cache_key)
+        keeper_config_file_path = os.path.join(tmp_dir, "keeper_config.xml")
+        with open(keeper_config_file_path, "w", encoding="utf-8") as f:
+            f.write(KEEPER_CONFIG)
+
+        container = DockerContainer(image=f"clickhouse/clickhouse-keeper:{keeper_version}")
+        container.with_volume_mapping(keeper_config_file_path, "/etc/clickhouse-keeper/keeper_config.xml")
+        container.with_exposed_ports(9181)
+        container.with_network(network=network)
+
+        container.start()
+        return types.TestContainerDocker(
+            id=container.get_wrapped_container().id,
+            host_configs={
+                "9181": types.TestContainerUrlConfig(
+                    scheme="tcp",
+                    address=container.get_container_host_ip(),
+                    port=container.get_exposed_port(9181),
+                )
+            },
+            container_configs={
+                "9181": types.TestContainerUrlConfig(
+                    scheme="tcp",
+                    address=container.get_wrapped_container().name,
+                    port=9181,
+                )
+            },
+        )
+
+    def delete(container: types.TestContainerDocker):
+        client = docker.from_env()
+        try:
+            client.containers.get(container_id=container.id).stop()
+            client.containers.get(container_id=container.id).remove(v=True)
+        except docker.errors.NotFound:
+            logger.info(
+                "Skipping removal of ClickHouse Keeper, Keeper(%s) not found. Maybe it was manually removed?",
+                {"id": container.id},
+            )
+
+    def restore(cache: dict) -> types.TestContainerDocker:
+        return types.TestContainerDocker.from_cache(cache)
+
+    return reuse.wrap(
+        request,
+        pytestconfig,
+        cache_key,
+        lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
+        create,
+        delete,
+        restore,
+    )
+
+
+@pytest.fixture(name="keeper", scope="package")
+def keeper(
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.TestContainerDocker:
+    """
+    Package-scoped fixture for ClickHouse Keeper TestContainer.
+    """
+    return create_clickhouse_keeper(
+        tmpfs=tmpfs,
+        network=network,
+        request=request,
+        pytestconfig=pytestconfig,
+    )

tests/fixtures/metricreduction.py

@@ -0,0 +1,83 @@
+import datetime
+from collections.abc import Sequence
+
+import clickhouse_connect.driver.client
+
+from fixtures.metrics import MetricsBufferSample, MetricsBufferTimeSeries
+
+
+def local_series_counts(
+    node_conns: list[clickhouse_connect.driver.client.Client],
+    table: str,
+    metric_name: str,
+) -> list[int]:
+    """Distinct series per node via the LOCAL (non-distributed) table."""
+    return [
+        int(
+            conn.query(
+                f"SELECT count(DISTINCT fingerprint) FROM signoz_metrics.{table} WHERE metric_name = %(metric_name)s",
+                parameters={"metric_name": metric_name},
+            ).result_rows[0][0]
+        )
+        for conn in node_conns
+    ]
+
+
+def assert_spans_shards(
+    node_conns: list[clickhouse_connect.driver.client.Client],
+    table: str,
+    metric_name: str,
+    total: int,
+) -> None:
+    """Guard for distributed tests: a green run on a cluster proves nothing
+    unless the seeded series actually landed on more than one shard."""
+    counts = local_series_counts(node_conns, table, metric_name)
+    assert sum(counts) == total, f"expected {total} series in {table} across shards, got {counts}"
+    assert min(counts) > 0, f"seeded series in {table} all landed on one shard: {counts}"
+
+
+def build_recent_gauge_data(
+    metric_name: str,
+    base_epoch: int,
+    services: Sequence[str],
+    pods_per_service: int,
+    minutes: int,
+    value: float = 1.0,
+) -> tuple[list[MetricsBufferTimeSeries], list[MetricsBufferSample]]:
+    """Collector-shaped buffer rows for a gauge under a reduction rule that
+    keeps `service`: per raw series a raw series row (is_reduced=false, full
+    labels, reduced_fingerprint -> group) plus the group's reduced series row
+    (is_reduced=true, kept labels), and one raw sample per series per minute
+    carrying both fingerprints. Returns (time_series, samples) for
+    insert_buffer_metrics."""
+    reduced_series = {
+        service: MetricsBufferTimeSeries(
+            metric_name=metric_name,
+            labels={"service": service},
+            timestamp=datetime.datetime.fromtimestamp(base_epoch, tz=datetime.UTC),
+            is_reduced=True,
+        )
+        for service in services
+    }
+    raw_series = [
+        MetricsBufferTimeSeries(
+            metric_name=metric_name,
+            labels={"service": service, "pod": f"pod-{service}-{i}"},
+            timestamp=datetime.datetime.fromtimestamp(base_epoch, tz=datetime.UTC),
+            reduced_fingerprint=reduced_series[service].fingerprint,
+        )
+        for service in services
+        for i in range(pods_per_service)
+    ]
+    samples = [
+        MetricsBufferSample(
+            metric_name=metric_name,
+            fingerprint=ts.fingerprint,
+            timestamp=datetime.datetime.fromtimestamp(base_epoch + minute * 60, tz=datetime.UTC),
+            value=value,
+            reduced_fingerprint=ts.reduced_fingerprint,
+        )
+        for ts in raw_series
+        for minute in range(minutes)
+    ]
+    return raw_series + list(reduced_series.values()), samples

tests/fixtures/metrics.py

@@ -11,6 +11,14 @@ import pytest
 from fixtures import types
 from fixtures.time import parse_timestamp
 
+_REDUCED_METRICS_TABLES_TO_TRUNCATE = [
+    "time_series_v4_reduced",
+    "samples_v4_reduced_last_60s",
+    "samples_v4_reduced_sum_60s",
+    "time_series_v4_buffer",
+    "samples_v4_buffer",
+]
+
 
 class MetricsTimeSeries(ABC):
     """Represents a row in the time_series_v4 table."""
@@ -28,7 +36,6 @@ class MetricsTimeSeries(ABC):
     attrs: dict[str, str]
     scope_attrs: dict[str, str]
     resource_attrs: dict[str, str]
-    __normalized: bool
 
     def __init__(
         self,
@@ -60,7 +67,6 @@ class MetricsTimeSeries(ABC):
         self.scope_attrs = scope_attrs
         self.resource_attrs = resource_attrs
         self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
-        self.__normalized = False
 
         # Calculate fingerprint from metric_name + labels
         fingerprint_str = metric_name + self.labels
@@ -81,7 +87,6 @@ class MetricsTimeSeries(ABC):
             self.attrs,
             self.scope_attrs,
             self.resource_attrs,
-            self.__normalized,
         ]
 
 
@@ -414,6 +419,263 @@ class Metrics(ABC):
         return metrics
 
 
+class MetricsReducedTimeSeries(ABC):
+    """Represents a row in the time_series_v4_reduced table i.e what
+    the time_series_v4_reduced_mv materializes for a metric under a
+    reduction rule. One row per kept-label group. `fingerprint` holds the
+    reduced fingerprint and `labels` contains only the kept labels.
+
+    The fingerprint recipe (md5, like MetricsTimeSeries) does not match the
+    collector's real hash; it only needs to be consistent with the
+    reduced_fingerprint used in the reduced samples rows.
+    """
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        metric_name: str,
+        kept_labels: dict[str, str],
+        timestamp: datetime.datetime,
+        temporality: str = "Unspecified",
+        description: str = "",
+        unit: str = "",
+        type_: str = "Gauge",
+        is_monotonic: bool = False,
+        env: str = "default",
+    ) -> None:
+        kept_labels = dict(kept_labels)
+        kept_labels["__name__"] = metric_name
+        self.env = env
+        # mirror time_series_v4_reduced_mv: monotonic cumulative counters are
+        # reduced as deltas
+        if temporality == "Cumulative" and is_monotonic:
+            temporality = "Delta"
+        self.temporality = temporality
+        self.metric_name = metric_name
+        self.description = description
+        self.unit = unit
+        self.type = type_
+        self.is_monotonic = is_monotonic
+        self.labels = json.dumps(kept_labels, separators=(",", ":"))
+        self.attrs = kept_labels
+        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3) // 3600000 * 3600000)
+
+        fingerprint_str = metric_name + self.labels
+        self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
+
+    def to_row(self) -> list:
+        return [
+            self.env,
+            self.temporality,
+            self.metric_name,
+            self.description,
+            self.unit,
+            self.type,
+            self.is_monotonic,
+            self.fingerprint,
+            self.unix_milli,
+            self.labels,
+            self.attrs,
+            {},
+            {},
+        ]
+
+
+class MetricsReducedSampleLast60s(ABC):
+    """Represents a row in the samples_v4_reduced_last_60s table. One 60s
+    bucket per reduced group, as the samples_v4_reduced_last_60s_mv refresh
+    would emit it (gauges and non-monotonic cumulative sums)."""
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        metric_name: str,
+        reduced_fingerprint: np.uint64,
+        timestamp: datetime.datetime,
+        sum_last: float,
+        min_value: float,
+        max_value: float,
+        sum_values: float,
+        count_series: int,
+        count_samples: int,
+        temporality: str = "Unspecified",
+        env: str = "default",
+        computed_at: datetime.datetime | None = None,
+    ) -> None:
+        self.env = env
+        self.temporality = temporality
+        self.metric_name = metric_name
+        self.reduced_fingerprint = reduced_fingerprint
+        # buckets are 60s-aligned: intDiv(unix_milli, 60000) * 60000
+        self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
+        self.sum_last = np.float64(sum_last)
+        self.min = np.float64(min_value)
+        self.max = np.float64(max_value)
+        self.sum_values = np.float64(sum_values)
+        self.count_series = np.uint64(count_series)
+        self.count_samples = np.uint64(count_samples)
+        # the refresh stamps now(); default to shortly after the bucket closes
+        if computed_at is None:
+            computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
+        self.computed_at = computed_at
+
+    def to_row(self) -> list:
+        return [
+            self.env,
+            self.temporality,
+            self.metric_name,
+            self.reduced_fingerprint,
+            self.unix_milli,
+            self.sum_last,
+            self.min,
+            self.max,
+            self.sum_values,
+            self.count_series,
+            self.count_samples,
+            self.computed_at,
+        ]
+
+
+class MetricsReducedSampleSum60s(ABC):
+    """Represents a row in the samples_v4_reduced_sum_60s table. One 60s
+    bucket per reduced group for delta counters and histograms."""
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        metric_name: str,
+        reduced_fingerprint: np.uint64,
+        timestamp: datetime.datetime,
+        sum_value: float,
+        count_series: int,
+        count_samples: int,
+        temporality: str = "Delta",
+        env: str = "default",
+        computed_at: datetime.datetime | None = None,
+    ) -> None:
+        self.env = env
+        self.temporality = temporality
+        self.metric_name = metric_name
+        self.reduced_fingerprint = reduced_fingerprint
+        self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
+        self.sum = np.float64(sum_value)
+        self.count_series = np.uint64(count_series)
+        self.count_samples = np.uint64(count_samples)
+        if computed_at is None:
+            computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
+        self.computed_at = computed_at
+
+    def to_row(self) -> list:
+        return [
+            self.env,
+            self.temporality,
+            self.metric_name,
+            self.reduced_fingerprint,
+            self.unix_milli,
+            self.sum,
+            self.count_series,
+            self.count_samples,
+            self.computed_at,
+        ]
+
+
+class MetricsBufferTimeSeries(ABC):
+    """Represents a row in the time_series_v4_buffer table. This is the collector's
+    universal landing target under cardinality control. For a ruled metric the
+    collector writes two rows per series: the raw one (is_reduced=false, full
+    labels, reduced_fingerprint pointing at its group) and the group's reduced
+    one (is_reduced=true, kept labels, fingerprint = reduced fingerprint)."""
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        metric_name: str,
+        labels: dict[str, str],
+        timestamp: datetime.datetime,
+        reduced_fingerprint: np.uint64 | int = 0,
+        is_reduced: bool = False,
+        temporality: str = "Unspecified",
+        description: str = "",
+        unit: str = "",
+        type_: str = "Gauge",
+        is_monotonic: bool = False,
+        env: str = "default",
+    ) -> None:
+        labels = dict(labels)
+        labels["__name__"] = metric_name
+        self.env = env
+        self.temporality = temporality
+        self.metric_name = metric_name
+        self.description = description
+        self.unit = unit
+        self.type = type_
+        self.is_monotonic = is_monotonic
+        self.reduced_fingerprint = np.uint64(reduced_fingerprint)
+        self.is_reduced = is_reduced
+        self.labels = json.dumps(labels, separators=(",", ":"))
+        self.attrs = labels
+        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3) // 3600000 * 3600000)
+
+        fingerprint_str = metric_name + self.labels
+        self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
+
+    def to_row(self) -> list:
+        return [
+            self.env,
+            self.temporality,
+            self.metric_name,
+            self.description,
+            self.unit,
+            self.type,
+            self.is_monotonic,
+            self.fingerprint,
+            self.reduced_fingerprint,
+            self.is_reduced,
+            self.unix_milli,
+            self.labels,
+            self.attrs,
+            {},
+            {},
+        ]
+
+
+class MetricsBufferSample(ABC):
+    """Represents a row in the samples_v4_buffer table. Ruled samples carry
+    the raw fingerprint plus the group's reduced_fingerprint; unruled samples
+    have reduced_fingerprint = 0."""
+
+    def __init__(  # pylint: disable=too-many-arguments
+        self,
+        metric_name: str,
+        fingerprint: np.uint64,
+        timestamp: datetime.datetime,
+        value: float,
+        reduced_fingerprint: np.uint64 | int = 0,
+        is_monotonic: bool = False,
+        temporality: str = "Unspecified",
+        env: str = "default",
+        flags: int = 0,
+    ) -> None:
+        self.env = env
+        self.temporality = temporality
+        self.metric_name = metric_name
+        self.fingerprint = fingerprint
+        self.reduced_fingerprint = np.uint64(reduced_fingerprint)
+        self.is_monotonic = is_monotonic
+        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
+        self.value = np.float64(value)
+        self.flags = np.uint32(flags)
+
+    def to_row(self) -> list:
+        return [
+            self.env,
+            self.temporality,
+            self.metric_name,
+            self.fingerprint,
+            self.reduced_fingerprint,
+            self.is_monotonic,
+            self.unix_milli,
+            self.value,
+            self.flags,
+        ]
+
+
 def insert_metrics_to_clickhouse(conn, metrics: list[Metrics]) -> None:
     """
     Insert metrics into ClickHouse tables.
@@ -449,7 +711,6 @@ def insert_metrics_to_clickhouse(conn, metrics: list[Metrics]) -> None:
                 "attrs",
                 "scope_attrs",
                 "resource_attrs",
-                "__normalized",
             ],
             data=[ts.to_row() for ts in time_series_map.values()],
         )
@@ -576,6 +837,161 @@ def insert_metrics(
     )
 
 
+def insert_reduced_metrics_to_clickhouse(
+    conn,
+    time_series: list[MetricsReducedTimeSeries],
+    last_samples: list[MetricsReducedSampleLast60s] | None = None,
+    sum_samples: list[MetricsReducedSampleSum60s] | None = None,
+) -> None:
+    """Insert reduced series into distributed_time_series_v4_reduced and 60s
+    buckets into the reduced samples tables. These tables exist only when
+    the schema migrator version includes the metrics cardinality-control
+    migration."""
+    if time_series:
+        conn.insert(
+            database="signoz_metrics",
+            table="distributed_time_series_v4_reduced",
+            column_names=[
+                "env",
+                "temporality",
+                "metric_name",
+                "description",
+                "unit",
+                "type",
+                "is_monotonic",
+                "fingerprint",
+                "unix_milli",
+                "labels",
+                "attrs",
+                "scope_attrs",
+                "resource_attrs",
+            ],
+            data=[ts.to_row() for ts in time_series],
+        )
+
+    if last_samples:
+        conn.insert(
+            database="signoz_metrics",
+            table="distributed_samples_v4_reduced_last_60s",
+            column_names=[
+                "env",
+                "temporality",
+                "metric_name",
+                "reduced_fingerprint",
+                "unix_milli",
+                "sum_last",
+                "min",
+                "max",
+                "sum_values",
+                "count_series",
+                "count_samples",
+                "computed_at",
+            ],
+            data=[sample.to_row() for sample in last_samples],
+        )
+
+    if sum_samples:
+        conn.insert(
+            database="signoz_metrics",
+            table="distributed_samples_v4_reduced_sum_60s",
+            column_names=[
+                "env",
+                "temporality",
+                "metric_name",
+                "reduced_fingerprint",
+                "unix_milli",
+                "sum",
+                "count_series",
+                "count_samples",
+                "computed_at",
+            ],
+            data=[sample.to_row() for sample in sum_samples],
+        )
+
+
+def insert_buffer_metrics_to_clickhouse(
+    conn,
+    time_series: list[MetricsBufferTimeSeries],
+    samples: list[MetricsBufferSample],
+) -> None:
+    if time_series:
+        conn.insert(
+            database="signoz_metrics",
+            table="distributed_time_series_v4_buffer",
+            column_names=[
+                "env",
+                "temporality",
+                "metric_name",
+                "description",
+                "unit",
+                "type",
+                "is_monotonic",
+                "fingerprint",
+                "reduced_fingerprint",
+                "is_reduced",
+                "unix_milli",
+                "labels",
+                "attrs",
+                "scope_attrs",
+                "resource_attrs",
+            ],
+            data=[ts.to_row() for ts in time_series],
+        )
+
+    if samples:
+        conn.insert(
+            database="signoz_metrics",
+            table="distributed_samples_v4_buffer",
+            column_names=[
+                "env",
+                "temporality",
+                "metric_name",
+                "fingerprint",
+                "reduced_fingerprint",
+                "is_monotonic",
+                "unix_milli",
+                "value",
+                "flags",
+            ],
+            data=[sample.to_row() for sample in samples],
+        )
+
+
+@pytest.fixture(name="insert_reduced_metrics", scope="function")
+def insert_reduced_metrics(
+    clickhouse: types.TestContainerClickhouse,
+) -> Generator[Callable[..., None], Any]:
+    def _insert_reduced_metrics(
+        time_series: list[MetricsReducedTimeSeries],
+        last_samples: list[MetricsReducedSampleLast60s] | None = None,
+        sum_samples: list[MetricsReducedSampleSum60s] | None = None,
+    ) -> None:
+        insert_reduced_metrics_to_clickhouse(clickhouse.conn, time_series, last_samples, sum_samples)
+
+    yield _insert_reduced_metrics
+
+    cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
+    for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
+        clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
+
+
+@pytest.fixture(name="insert_buffer_metrics", scope="function")
+def insert_buffer_metrics(
+    clickhouse: types.TestContainerClickhouse,
+) -> Generator[Callable[..., None], Any]:
+    def _insert_buffer_metrics(
+        time_series: list[MetricsBufferTimeSeries],
+        samples: list[MetricsBufferSample],
+    ) -> None:
+        insert_buffer_metrics_to_clickhouse(clickhouse.conn, time_series, samples)
+
+    yield _insert_buffer_metrics
+
+    cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
+    for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
+        clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
+
+
 @pytest.fixture(name="remove_metrics_ttl_and_storage_settings", scope="function")
 def remove_metrics_ttl_and_storage_settings(signoz: types.SigNoz):
     """

tests/fixtures/migrator.py

@@ -8,7 +8,7 @@ from fixtures.logger import setup_logger
 logger = setup_logger(__name__)
 
 
-def create_migrator(
+def create_migrator(  # pylint: disable=too-many-arguments,too-many-positional-arguments
     network: Network,
     clickhouse: types.TestContainerClickhouse,
     request: pytest.FixtureRequest,
@@ -17,18 +17,19 @@ def create_migrator(
     env_overrides: dict | None = None,
 ) -> types.Operation:
     """
-    Factory function for running schema migrations.
-    Accepts optional env_overrides to customize the migrator environment.
+    Factory function for running schema migrations. Accepts optional
+    env_overrides to customize the migrator environment; the release comes
+    from the --schema-migrator-version option.
     """
 
     def create() -> None:
-        version = request.config.getoption("--schema-migrator-version")
+        migrator_version = request.config.getoption("--schema-migrator-version")
         client = docker.from_env()
 
         environment = dict(env_overrides) if env_overrides else {}
 
         container = client.containers.run(
-            image=f"signoz/signoz-schema-migrator:{version}",
+            image=f"signoz/signoz-schema-migrator:{migrator_version}",
             command=f"sync --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN']}",
             detach=True,
             auto_remove=False,
@@ -47,7 +48,7 @@ def create_migrator(
         container.remove()
 
         container = client.containers.run(
-            image=f"signoz/signoz-schema-migrator:{version}",
+            image=f"signoz/signoz-schema-migrator:{migrator_version}",
             command=f"async --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN']}",
             detach=True,
             auto_remove=False,

tests/fixtures/querier.py

@@ -79,10 +79,13 @@ class BuilderQuery:
     name: str = "A"
     source: str | None = None
     limit: int | None = None
+    offset: int | None = None
     filter_expression: str | None = None
+    having_expression: str | None = None
     select_fields: list[TelemetryFieldKey] | None = None
     order: list[OrderBy] | None = None
     aggregations: list[Aggregation | MetricAggregation] | None = None
+    group_by: list[TelemetryFieldKey] | None = None
     step_interval: int | None = None
 
     def to_dict(self) -> dict:
@@ -94,14 +97,20 @@ class BuilderQuery:
             spec["source"] = self.source
         if self.limit is not None:
             spec["limit"] = self.limit
+        if self.offset is not None:
+            spec["offset"] = self.offset
         if self.filter_expression:
             spec["filter"] = {"expression": self.filter_expression}
+        if self.having_expression:
+            spec["having"] = {"expression": self.having_expression}
         if self.select_fields:
             spec["selectFields"] = [f.to_dict() for f in self.select_fields]
         if self.order:
             spec["order"] = [o.to_dict() if hasattr(o, "to_dict") else o for o in self.order]
         if self.aggregations:
             spec["aggregations"] = [agg.to_dict() if hasattr(agg, "to_dict") else agg for agg in self.aggregations]
+        if self.group_by:
+            spec["groupBy"] = [k.to_dict() for k in self.group_by]
         if self.step_interval is not None:
             spec["stepInterval"] = self.step_interval
 
@@ -189,6 +198,38 @@ def make_query_request(
     )
 
 
+def aligned_epoch(ago: timedelta, step_seconds: int = DEFAULT_STEP_INTERVAL) -> int:
+    """Epoch seconds for `now - ago`, floored to a step boundary so seeded
+    points land exactly on the query's toStartOfInterval buckets."""
+    epoch = (int((datetime.now(tz=UTC) - ago).timestamp()) // step_seconds) * step_seconds
+    if epoch % 3600 == 0:
+        epoch += step_seconds
+    return epoch
+
+
+def query_metric_values(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    signoz: types.SigNoz,
+    token: str,
+    metric_name: str,
+    start_epoch: int,
+    end_epoch: int,
+    time_agg: str,
+    space_agg: str,
+    step_interval: int = DEFAULT_STEP_INTERVAL,
+) -> list[dict]:
+    """Run a single metrics builder query over [start_epoch, end_epoch) in
+    epoch seconds and return its series values sorted by timestamp."""
+    response = make_query_request(
+        signoz,
+        token,
+        start_ms=start_epoch * 1000,
+        end_ms=end_epoch * 1000,
+        queries=[build_builder_query("A", metric_name, time_agg, space_agg, step_interval=step_interval)],
+    )
+    assert response.status_code == HTTPStatus.OK, response.text
+    return sorted(get_series_values(response.json(), "A"), key=lambda v: v["timestamp"])
+
+
 def build_builder_query(
     name: str,
     metric_name: str,

tests/fixtures/types.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Literal
 from urllib.parse import urljoin
 
@@ -84,11 +84,16 @@ class TestContainerClickhouse:
     container: TestContainerDocker
     conn: clickhouse_connect.driver.client.Client
     env: dict[str, str]
+    # Per-node containers when running a multi-node cluster. Empty for the
+    # default single-node setup; nodes[0] is the node `conn`/`env` point at
+    # (the initiator every query goes through).
+    nodes: list[TestContainerDocker] = field(default_factory=list)
 
     def __cache__(self) -> dict:
         return {
             "container": self.container.__cache__(),
             "env": self.env,
+            "nodes": [node.__cache__() for node in self.nodes],
         }
 
     def __log__(self) -> str:

tests/fixtures/zookeeper.py

@@ -1,67 +0,0 @@
-import docker
-import docker.errors
-import pytest
-from testcontainers.core.container import DockerContainer, Network
-
-from fixtures import reuse, types
-from fixtures.logger import setup_logger
-
-logger = setup_logger(__name__)
-
-
-@pytest.fixture(name="zookeeper", scope="package")
-def zookeeper(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
-    """
-    Package-scoped fixture for Zookeeper TestContainer.
-    """
-
-    def create() -> types.TestContainerDocker:
-        version = request.config.getoption("--zookeeper-version")
-
-        container = DockerContainer(image=f"signoz/zookeeper:{version}")
-        container.with_env("ALLOW_ANONYMOUS_LOGIN", "yes")
-        container.with_exposed_ports(2181)
-        container.with_network(network=network)
-
-        container.start()
-        return types.TestContainerDocker(
-            id=container.get_wrapped_container().id,
-            host_configs={
-                "2181": types.TestContainerUrlConfig(
-                    scheme="tcp",
-                    address=container.get_container_host_ip(),
-                    port=container.get_exposed_port(2181),
-                )
-            },
-            container_configs={
-                "2181": types.TestContainerUrlConfig(
-                    scheme="tcp",
-                    address=container.get_wrapped_container().name,
-                    port=2181,
-                )
-            },
-        )
-
-    def delete(container: types.TestContainerDocker):
-        client = docker.from_env()
-        try:
-            client.containers.get(container_id=container.id).stop()
-            client.containers.get(container_id=container.id).remove(v=True)
-        except docker.errors.NotFound:
-            logger.info(
-                "Skipping removal of Zookeeper, Zookeeper(%s) not found. Maybe it was manually removed?",
-                {"id": container.id},
-            )
-
-    def restore(cache: dict) -> types.TestContainerDocker:
-        return types.TestContainerDocker.from_cache(cache)
-
-    return reuse.wrap(
-        request,
-        pytestconfig,
-        "zookeeper",
-        lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
-        create,
-        delete,
-        restore,
-    )

tests/integration/testdata/alerts/test_scenarios/threshold_above_at_least_once/rule.json

@@ -25,7 +25,7 @@
           "type": "clickhouse_sql",
           "spec": {
             "name": "A",
-            "query": "WITH __temporal_aggregation_cte AS (\n SELECT \n fingerprint, \n toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, \n avg(value) AS per_series_value \n FROM signoz_metrics.distributed_samples_v4 AS points \n INNER JOIN (\n SELECT fingerprint \n FROM signoz_metrics.time_series_v4 \n WHERE metric_name IN ('request_total_threshold_above_at_least_once') \n AND LOWER(temporality) LIKE LOWER('cumulative') \n AND __normalized = false \n GROUP BY fingerprint\n ) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint \n WHERE metric_name IN ('request_total_threshold_above_at_least_once') \n AND unix_milli >= $start_timestamp_ms \n AND unix_milli < $end_timestamp_ms \n GROUP BY fingerprint, ts \n ORDER BY fingerprint, ts\n), \n__spatial_aggregation_cte AS (\n SELECT \n ts, \n sum(per_series_value) AS value \n FROM __temporal_aggregation_cte \n WHERE isNaN(per_series_value) = 0 \n GROUP BY ts\n) \nSELECT * FROM __spatial_aggregation_cte \nORDER BY ts"
+            "query": "WITH __temporal_aggregation_cte AS (\n SELECT \n fingerprint, \n toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, \n avg(value) AS per_series_value \n FROM signoz_metrics.distributed_samples_v4 AS points \n INNER JOIN (\n SELECT fingerprint \n FROM signoz_metrics.time_series_v4 \n WHERE metric_name IN ('request_total_threshold_above_at_least_once') \n AND LOWER(temporality) LIKE LOWER('cumulative') \n GROUP BY fingerprint\n ) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint \n WHERE metric_name IN ('request_total_threshold_above_at_least_once') \n AND unix_milli >= $start_timestamp_ms \n AND unix_milli < $end_timestamp_ms \n GROUP BY fingerprint, ts \n ORDER BY fingerprint, ts\n), \n__spatial_aggregation_cte AS (\n SELECT \n ts, \n sum(per_series_value) AS value \n FROM __temporal_aggregation_cte \n WHERE isNaN(per_series_value) = 0 \n GROUP BY ts\n) \nSELECT * FROM __spatial_aggregation_cte \nORDER BY ts"
           }
         }
       ]

tests/integration/tests/clickhousecluster/01_basic_cluster.py

@@ -0,0 +1,52 @@
+import clickhouse_connect.driver.client
+
+from fixtures import types
+
+TOTAL_ROWS = 64
+
+
+def test_topology(
+    clickhouse: types.TestContainerClickhouse,
+    clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
+) -> None:
+    aliases = {node.container_configs["9000"].address for node in clickhouse.nodes}
+
+    # Every node sees the same 2-shard cluster definition and identifies
+    # exactly itself as the local replica
+
+    for i, conn in enumerate(clickhouse_node_conns, start=1):
+        rows = conn.query("SELECT shard_num, host_name, is_local FROM system.clusters WHERE cluster = 'cluster' ORDER BY shard_num").result_rows
+        assert [row[0] for row in rows] == [1, 2], f"node {i}: expected 2 shards, got {rows}"
+        assert {row[1] for row in rows} == aliases, f"node {i}: cluster hosts {rows} != node aliases {aliases}"
+        local = [row[0] for row in rows if row[2]]
+        assert local == [i], f"node {i}: expected to be local for shard {i} only, got {local}"
+
+
+def test_replicated_distributed_round_trip(
+    clickhouse: types.TestContainerClickhouse,
+    clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
+) -> None:
+    # ON CLUSTER DDL reaches both nodes, Replicated engines register with the
+    # keeper via per-node macros, and a sharded Distributed insert scatters rows
+    # across shards while the distributed read returns the union.
+    conn = clickhouse.conn
+    try:
+        conn.query("CREATE DATABASE IF NOT EXISTS it_cluster ON CLUSTER 'cluster'")
+        conn.query("CREATE TABLE it_cluster.events ON CLUSTER 'cluster' (id UInt64, payload String) ENGINE = ReplicatedMergeTree ORDER BY id")
+        conn.query("CREATE TABLE it_cluster.distributed_events ON CLUSTER 'cluster' AS it_cluster.events ENGINE = Distributed('cluster', 'it_cluster', 'events', cityHash64(id))")
+
+        conn.insert(
+            database="it_cluster",
+            table="distributed_events",
+            column_names=["id", "payload"],
+            data=[[i, f"payload-{i:03d}"] for i in range(TOTAL_ROWS)],
+        )
+
+        distributed_count = int(conn.query("SELECT count() FROM it_cluster.distributed_events").result_rows[0][0])
+        assert distributed_count == TOTAL_ROWS
+
+        local_counts = [int(node_conn.query("SELECT count() FROM it_cluster.events").result_rows[0][0]) for node_conn in clickhouse_node_conns]
+        assert sum(local_counts) == TOTAL_ROWS, f"local counts {local_counts} do not add up to {TOTAL_ROWS}"
+        assert min(local_counts) > 0, f"all rows landed on one shard: {local_counts}"
+    finally:
+        conn.query("DROP DATABASE IF EXISTS it_cluster ON CLUSTER 'cluster' SYNC")

tests/integration/tests/clickhousecluster/conftest.py

@@ -0,0 +1,44 @@
+from collections.abc import Generator
+from typing import Any
+
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.clickhouse import create_clickhouse_cluster
+from fixtures.keeper import create_clickhouse_keeper
+
+
+@pytest.fixture(name="keeper", scope="package")
+def keeper_cluster(
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.TestContainerDocker:
+    return create_clickhouse_keeper(
+        tmpfs=tmpfs,
+        network=network,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="keeper_cluster",
+    )
+
+
+@pytest.fixture(name="clickhouse", scope="package")
+def clickhouse_cluster(
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    keeper: types.TestContainerDocker,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.TestContainerClickhouse:
+    return create_clickhouse_cluster(
+        tmpfs=tmpfs,
+        network=network,
+        keeper=keeper,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="clickhouse_cluster",
+        shards=2,
+    )

tests/integration/tests/metricreduction/01_reduced_gauge.py

@@ -0,0 +1,203 @@
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import clickhouse_connect.driver.client
+import pytest
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.metricreduction import assert_spans_shards
+from fixtures.metrics import (
+    Metrics,
+    MetricsReducedSampleLast60s,
+    MetricsReducedTimeSeries,
+)
+from fixtures.querier import aligned_epoch, query_metric_values
+
+
+def test_query_spanning_rule_activation_combines_raw_and_reduced_data(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_metrics: Callable[[list[Metrics]], None],
+    insert_reduced_metrics: Callable[..., None],
+    clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
+) -> None:
+    """Before a reduction rule activates, data lives in the raw tables; after,
+    only the reduced tables have data. A single query spanning the activation
+    time must return one continuous series with no gap and no double counting:
+    32 raw series at 2.0 collapse into 16 groups whose per-minute total is
+    4.0, so the summed value stays 320 per step on both sides. Enough series
+    are seeded that both shards hold data (checked below), so correct totals
+    also prove the queries read every shard."""
+    metric_name = "test_reduction_activation_boundary"
+    base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
+    services = [f"svc-{i:02d}" for i in range(16)]
+
+    # first 30 minutes: raw data (2 pods per service, one sample per minute)
+    insert_metrics(
+        [
+            Metrics(
+                metric_name=metric_name,
+                labels={"service": service, "pod": f"{service}-pod-{pod}"},
+                timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
+                value=2.0,
+                type_="Gauge",
+                is_monotonic=False,
+            )
+            for service in services
+            for pod in range(2)
+            for minute in range(30)
+        ]
+    )
+
+    # next 30 minutes: reduced data only (one row per service per minute)
+    time_series = [
+        MetricsReducedTimeSeries(
+            metric_name=metric_name,
+            kept_labels={"service": service},
+            timestamp=datetime.fromtimestamp(base_epoch + 30 * 60, tz=UTC),
+        )
+        for service in services
+    ]
+    insert_reduced_metrics(
+        time_series,
+        [
+            MetricsReducedSampleLast60s(
+                metric_name=metric_name,
+                reduced_fingerprint=ts.fingerprint,
+                timestamp=datetime.fromtimestamp(base_epoch + (30 + minute) * 60, tz=UTC),
+                sum_last=4.0,
+                min_value=2.0,
+                max_value=2.0,
+                sum_values=4.0,
+                count_series=2,
+                count_samples=2,
+            )
+            for ts in time_series
+            for minute in range(30)
+        ],
+    )
+
+    assert_spans_shards(clickhouse_node_conns, "time_series_v4", metric_name, total=len(services) * 2)
+    assert_spans_shards(clickhouse_node_conns, "time_series_v4_reduced", metric_name, total=len(services))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 3600, "sum", "sum", step_interval=300)
+
+    assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(12)]
+    assert [v["value"] for v in values] == [320.0] * 12
+
+
+@pytest.mark.parametrize(
+    "space_agg, expected",
+    [
+        ("sum", 12.0),  # sum_last: 4 + 8
+        ("avg", 3.0),  # sum(sum_last) / sum(count_series): 12 / 4
+        ("min", 1.0),  # min(min)
+        ("max", 6.0),  # max(max)
+    ],
+)
+def test_aggregations_across_series(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_reduced_metrics: Callable[..., None],
+    space_agg: str,
+    expected: float,
+) -> None:
+    """Aggregating across series reads the pre-aggregated reduced columns:
+    sum/avg from sum_last with the count_series weight, min/max from the
+    min/max columns."""
+    metric_name = f"test_reduction_across_series_{space_agg}"
+    base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
+
+    groups = [
+        # (service, sum_last, min, max, count_series)
+        ("a", 4.0, 1.0, 3.0, 2),
+        ("b", 8.0, 2.0, 6.0, 2),
+    ]
+    time_series = {
+        service: MetricsReducedTimeSeries(
+            metric_name=metric_name,
+            kept_labels={"service": service},
+            timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
+        )
+        for service, _, _, _, _ in groups
+    }
+    insert_reduced_metrics(
+        list(time_series.values()),
+        [
+            MetricsReducedSampleLast60s(
+                metric_name=metric_name,
+                reduced_fingerprint=time_series[service].fingerprint,
+                timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
+                sum_last=sum_last,
+                min_value=min_value,
+                max_value=max_value,
+                sum_values=sum_last,
+                count_series=count_series,
+                count_samples=count_series,
+            )
+            for service, sum_last, min_value, max_value, count_series in groups
+            for minute in range(20)
+        ],
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 20 * 60, "avg", space_agg, step_interval=300)
+
+    assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
+    assert [v["value"] for v in values] == [expected] * 4
+
+
+def test_recomputed_minutes_use_only_the_newest_values(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_reduced_metrics: Callable[..., None],
+) -> None:
+    """The collector rewrites recent minutes on every refresh, so the same
+    minute exists multiple times with increasing computed_at. Queries must
+    count each minute once, using its newest version: write the same minutes
+    twice with different values and only the second write may show up."""
+    metric_name = "test_reduction_recompute"
+    base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
+
+    time_series = [
+        MetricsReducedTimeSeries(
+            metric_name=metric_name,
+            kept_labels={"service": service},
+            timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
+        )
+        for service in ("a", "b")
+    ]
+
+    def minute_rows(sum_last: float, computed_at_offset_seconds: int) -> list[MetricsReducedSampleLast60s]:
+        return [
+            MetricsReducedSampleLast60s(
+                metric_name=metric_name,
+                reduced_fingerprint=ts.fingerprint,
+                timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
+                sum_last=sum_last,
+                min_value=sum_last,
+                max_value=sum_last,
+                sum_values=sum_last,
+                count_series=1,
+                count_samples=1,
+                computed_at=datetime.fromtimestamp(base_epoch + minute * 60 + computed_at_offset_seconds, tz=UTC),
+            )
+            for ts in time_series
+            for minute in range(10)
+        ]
+
+    # first write says 1.0; a later rewrite of the same minutes says 5.0
+    insert_reduced_metrics(time_series, minute_rows(sum_last=1.0, computed_at_offset_seconds=120))
+    insert_reduced_metrics(time_series, minute_rows(sum_last=5.0, computed_at_offset_seconds=180))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 10 * 60, "sum", "sum", step_interval=300)
+
+    # 2 groups x 5 minutes x 5.0 per step; the 1.0 rows must not contribute
+    assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(2)]
+    assert [v["value"] for v in values] == [50.0] * 2

tests/integration/tests/metricreduction/02_reduced_counter.py

@@ -0,0 +1,70 @@
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.metrics import (
+    MetricsReducedSampleSum60s,
+    MetricsReducedTimeSeries,
+)
+from fixtures.querier import aligned_epoch, query_metric_values
+
+
+@pytest.mark.parametrize(
+    "time_agg, expected",
+    [
+        # 2 groups x 5 minutes x 30.0 per 300s step
+        ("rate", 1.0),  # 300 / 300s
+        ("increase", 300.0),
+    ],
+)
+def test_counter_rate_and_increase(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_reduced_metrics: Callable[..., None],
+    time_agg: str,
+    expected: float,
+) -> None:
+    metric_name = f"test_reduction_counter_{time_agg}"
+    base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
+
+    # monotonic cumulative counter: MetricsReducedTimeSeries mirrors the
+    # collector's temporality rewrite to Delta
+    time_series = [
+        MetricsReducedTimeSeries(
+            metric_name=metric_name,
+            kept_labels={"service": service},
+            timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
+            temporality="Cumulative",
+            type_="Sum",
+            is_monotonic=True,
+        )
+        for service in ("a", "b")
+    ]
+    assert all(ts.temporality == "Delta" for ts in time_series)
+
+    insert_reduced_metrics(
+        time_series,
+        sum_samples=[
+            MetricsReducedSampleSum60s(
+                metric_name=metric_name,
+                reduced_fingerprint=ts.fingerprint,
+                timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
+                sum_value=30.0,
+                count_series=2,
+                count_samples=2,
+                temporality="Delta",
+            )
+            for ts in time_series
+            for minute in range(20)
+        ],
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 20 * 60, time_agg, "sum", step_interval=300)
+
+    assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
+    assert [v["value"] for v in values] == [expected] * 4

tests/integration/tests/metricreduction/03_reduced_recent_window.py

@@ -0,0 +1,70 @@
+from collections.abc import Callable
+from datetime import timedelta
+from http import HTTPStatus
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.metricreduction import build_recent_gauge_data
+from fixtures.querier import (
+    aligned_epoch,
+    build_builder_query,
+    get_all_series,
+    index_series_by_label,
+    make_query_request,
+    query_metric_values,
+)
+
+SERVICES = ("a", "b")
+PODS_PER_SERVICE = 2
+MINUTES = 20
+
+
+def test_recent_queries_return_full_resolution_totals(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_buffer_metrics: Callable[..., None],
+) -> None:
+    metric_name = "test_reduction_recent_totals"
+    # samples span [now-25m, now-5m); the query window sits inside the last 24h
+    base_epoch = aligned_epoch(timedelta(minutes=25), step_seconds=300)
+    insert_buffer_metrics(*build_recent_gauge_data(metric_name, base_epoch, SERVICES, PODS_PER_SERVICE, MINUTES))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + MINUTES * 60, "sum", "sum", step_interval=300)
+
+    # 4 raw series x 5 samples x 1.0 per step: full raw resolution, and the
+    # reduced series rows must not be counted (their fingerprints match no
+    # samples, and the time-series lookup filters them out)
+    assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
+    assert [v["value"] for v in values] == [float(len(SERVICES) * PODS_PER_SERVICE * 5)] * 4
+
+
+def test_recent_queries_group_by_full_labels(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_buffer_metrics: Callable[..., None],
+) -> None:
+    """Group-by resolves against the raw buffer series rows (full labels), so
+    grouping by the kept label still sees every raw series underneath."""
+    metric_name = "test_reduction_recent_groupby"
+    base_epoch = aligned_epoch(timedelta(minutes=25), step_seconds=300)
+    insert_buffer_metrics(*build_recent_gauge_data(metric_name, base_epoch, SERVICES, PODS_PER_SERVICE, MINUTES))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    response = make_query_request(
+        signoz,
+        token,
+        start_ms=base_epoch * 1000,
+        end_ms=(base_epoch + MINUTES * 60) * 1000,
+        queries=[build_builder_query("A", metric_name, "sum", "sum", step_interval=300, group_by=["service"])],
+    )
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    series_by_service = index_series_by_label(get_all_series(response.json(), "A"), "service")
+    assert set(series_by_service.keys()) == set(SERVICES)
+    for service in SERVICES:
+        values = sorted(series_by_service[service]["values"], key=lambda v: v["timestamp"])
+        # 2 pods x 5 samples x 1.0 per step
+        assert [v["value"] for v in values] == [float(PODS_PER_SERVICE * 5)] * 4

tests/integration/tests/metricreduction/conftest.py

@@ -0,0 +1,93 @@
+from collections.abc import Generator
+from typing import Any
+
+import pytest
+from testcontainers.core.container import Network
+
+from fixtures import types
+from fixtures.auth import register_admin
+from fixtures.clickhouse import create_clickhouse_cluster
+from fixtures.keeper import create_clickhouse_keeper
+from fixtures.migrator import create_migrator
+from fixtures.signoz import create_signoz
+
+
+@pytest.fixture(name="keeper", scope="package")
+def keeper_metricreduction(
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.TestContainerDocker:
+    return create_clickhouse_keeper(
+        tmpfs=tmpfs,
+        network=network,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="keeper_metricreduction",
+    )
+
+
+@pytest.fixture(name="clickhouse", scope="package")
+def clickhouse_metricreduction(
+    tmpfs: Generator[types.LegacyPath, Any],
+    network: Network,
+    keeper: types.TestContainerDocker,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.TestContainerClickhouse:
+    return create_clickhouse_cluster(
+        tmpfs=tmpfs,
+        network=network,
+        keeper=keeper,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="clickhouse_metricreduction",
+        shards=2,
+    )
+
+
+@pytest.fixture(name="migrator", scope="package")
+def migrator_metricreduction(
+    network: Network,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.Operation:
+    return create_migrator(
+        network=network,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="migrator_metricreduction",
+    )
+
+
+@pytest.fixture(name="signoz", scope="package")
+def signoz_metricreduction(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    network: Network,
+    zeus: types.TestContainerDocker,
+    gateway: types.TestContainerDocker,
+    sqlstore: types.TestContainerSQL,
+    clickhouse: types.TestContainerClickhouse,
+    request: pytest.FixtureRequest,
+    pytestconfig: pytest.Config,
+) -> types.SigNoz:
+    return create_signoz(
+        network=network,
+        zeus=zeus,
+        gateway=gateway,
+        sqlstore=sqlstore,
+        clickhouse=clickhouse,
+        request=request,
+        pytestconfig=pytestconfig,
+        cache_key="signoz_metricreduction",
+        env_overrides={
+            "SIGNOZ_FLAGGER_CONFIG_BOOLEAN_ENABLE__METRICS__REDUCTION": True,
+        },
+    )
+
+
+@pytest.fixture(name="create_user_admin", scope="package")
+def create_user_admin_metricreduction(signoz: types.SigNoz, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.Operation:
+    return register_admin(signoz, request, pytestconfig, cache_key="create_user_admin_metricreduction")

tests/integration/tests/querier_json_body/conftest.py

@@ -5,16 +5,6 @@ from fixtures import types
 from fixtures.migrator import create_migrator
 from fixtures.signoz import create_signoz
 
-UNSUPPORTED_CLICKHOUSE_VERSIONS = {"25.5.6"}
-
-
-def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
-    version = config.getoption("--clickhouse-version")
-    if version in UNSUPPORTED_CLICKHOUSE_VERSIONS:
-        skip = pytest.mark.skip(reason=f"JSON body QB tests require ClickHouse > {version}")
-        for item in items:
-            item.add_marker(skip)
-
 
 @pytest.fixture(name="migrator", scope="package")
 def migrator_json(

tests/integration/tests/querierai/01_ai_traces.py

@@ -0,0 +1,843 @@
+"""
+Integration tests for source="ai" over the traces signal.
+
+Data shape (generic OTel gen_ai semantic conventions):
+  - a root span (no gen_ai attributes)
+  - an LLM span carrying gen_ai.request.model (str) and numeric usage attributes
+    (gen_ai.usage.input_tokens / output_tokens / cost) plus gen_ai.user.id
+Each test tags its spans with a unique service.name and filters on it, so tests do
+not interfere with each other's data.
+"""
+
+import json
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from http import HTTPStatus
+
+import pytest
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.querier import (
+    BuilderQuery,
+    OrderBy,
+    RequestType,
+    TelemetryFieldKey,
+    make_query_request,
+)
+from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
+
+
+def _ai_trace(
+    *,
+    now: datetime,
+    service: str,
+    user: str,
+    in_tokens: int | None,
+    out_tokens: int,
+    cost: float,
+    model: str = "gpt-4o-mini",
+    llm_duration_s: float = 1.0,
+    error: bool = False,
+    environment: str = "production",
+) -> list[Traces]:
+    """A minimal AI trace: root span + one LLM span with gen_ai attributes.
+    in_tokens=None omits the input-tokens attribute entirely (not zero)."""
+    trace_id = TraceIdGenerator.trace_id()
+    root_id = TraceIdGenerator.span_id()
+    llm_id = TraceIdGenerator.span_id()
+    resources = {"service.name": service, "deployment.environment": environment}
+
+    root = Traces(
+        timestamp=now - timedelta(seconds=5),
+        duration=timedelta(seconds=llm_duration_s + 0.1),
+        trace_id=trace_id,
+        span_id=root_id,
+        parent_span_id="",
+        name="POST /api/chat",
+        kind=TracesKind.SPAN_KIND_SERVER,
+        status_code=TracesStatusCode.STATUS_CODE_OK,
+        resources=resources,
+        attributes={"http.request.method": "POST"},
+    )
+    attributes = {
+        "gen_ai.request.model": model,
+        "gen_ai.system": "openai",
+        "gen_ai.user.id": user,
+        # numeric values land in attributes_number
+        "gen_ai.usage.output_tokens": out_tokens,
+        "_signoz.gen_ai.total_cost": cost,
+    }
+    if in_tokens is not None:
+        attributes["gen_ai.usage.input_tokens"] = in_tokens
+    llm = Traces(
+        timestamp=now - timedelta(seconds=4),
+        duration=timedelta(seconds=llm_duration_s),
+        trace_id=trace_id,
+        span_id=llm_id,
+        parent_span_id=root_id,
+        name="chat gpt-4o-mini",
+        kind=TracesKind.SPAN_KIND_CLIENT,
+        status_code=(TracesStatusCode.STATUS_CODE_ERROR if error else TracesStatusCode.STATUS_CODE_OK),
+        resources=resources,
+        attributes=attributes,
+    )
+    return [root, llm]
+
+
+def _non_ai_trace(*, now: datetime, service: str) -> list[Traces]:
+    """A plain HTTP trace with no gen_ai attributes; must be excluded by the AI gate."""
+    trace_id = TraceIdGenerator.trace_id()
+    span_id = TraceIdGenerator.span_id()
+    return [
+        Traces(
+            timestamp=now - timedelta(seconds=4),
+            duration=timedelta(seconds=1),
+            trace_id=trace_id,
+            span_id=span_id,
+            parent_span_id="",
+            name="GET /health",
+            kind=TracesKind.SPAN_KIND_SERVER,
+            status_code=TracesStatusCode.STATUS_CODE_OK,
+            resources={"service.name": service},
+            attributes={"http.request.method": "GET"},
+        )
+    ]
+
+
+def _window_ms(now: datetime) -> tuple[int, int]:
+    start_ms = int((now - timedelta(minutes=10)).timestamp() * 1000)
+    end_ms = int((now + timedelta(minutes=1)).timestamp() * 1000)
+    return start_ms, end_ms
+
+
+def test_ai_list_excludes_non_ai(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    Trace-list panel (requestType="trace"): returns AI traces and excludes the
+    non-AI trace. Asserts on the raw response payload to stay agnostic to the exact
+    row schema.
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-list"
+
+    ai = _ai_trace(now=now, service=service, user="alice", in_tokens=100, out_tokens=20, cost=0.5)
+    non_ai = _non_ai_trace(now=now, service=service)
+    ai_trace_id = ai[0].trace_id
+    non_ai_trace_id = non_ai[0].trace_id
+    insert_traces(ai + non_ai)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression=f"service.name = '{service}'",
+        limit=10,
+    )
+
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    body = json.dumps(response.json())
+    assert ai_trace_id in body, f"expected AI trace {ai_trace_id} in list response"
+    assert non_ai_trace_id not in body, f"non-AI trace {non_ai_trace_id} should be excluded by the gate"
+
+
+def _ai_trace_mixed_spans(*, now: datetime, service: str, user: str) -> list[Traces]:
+    """
+    Root + one LLM span + one tool span + one agent span. The gate matches all three
+    child spans, but only the LLM span carries gen_ai.request.model.
+    """
+    trace_id = TraceIdGenerator.trace_id()
+    root_id = TraceIdGenerator.span_id()
+    resources = {"service.name": service, "deployment.environment": "production"}
+
+    def _span(name, kind, attributes, offset_s):
+        return Traces(
+            timestamp=now - timedelta(seconds=offset_s),
+            duration=timedelta(seconds=0.5),
+            trace_id=trace_id,
+            span_id=TraceIdGenerator.span_id(),
+            parent_span_id=root_id,
+            name=name,
+            kind=kind,
+            status_code=TracesStatusCode.STATUS_CODE_OK,
+            resources=resources,
+            attributes=attributes,
+        )
+
+    root = Traces(
+        timestamp=now - timedelta(seconds=5),
+        duration=timedelta(seconds=4),
+        trace_id=trace_id,
+        span_id=root_id,
+        parent_span_id="",
+        name="POST /api/chat",
+        kind=TracesKind.SPAN_KIND_SERVER,
+        status_code=TracesStatusCode.STATUS_CODE_OK,
+        resources=resources,
+        attributes={"http.request.method": "POST"},
+    )
+    llm = _span(
+        "chat gpt-4o-mini",
+        TracesKind.SPAN_KIND_CLIENT,
+        {
+            "gen_ai.request.model": "gpt-4o-mini",
+            "gen_ai.system": "openai",
+            "gen_ai.user.id": user,
+            "gen_ai.usage.input_tokens": 100,
+            "gen_ai.usage.output_tokens": 20,
+        },
+        4,
+    )
+    tool = _span(
+        "execute_tool",
+        TracesKind.SPAN_KIND_INTERNAL,
+        {
+            "gen_ai.tool.name": "get_weather",
+            "gen_ai.tool.type": "function",
+        },
+        3,
+    )
+    agent = _span(
+        "agent.step",
+        TracesKind.SPAN_KIND_INTERNAL,
+        {
+            "gen_ai.agent.name": "chat-agent",
+        },
+        2,
+    )
+    return [root, llm, tool, agent]
+
+
+def test_ai_list_having_aggregate_filter(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    Aggregate filter written in the SAME filter box: the span-level predicate narrows
+    to the service, the trace-level `output_tokens > 100` keeps the large-token
+    trace and drops the small one (split internally into WHERE + HAVING). Both
+    spellings of a trace-level aggregate — bare and `trace.` — behave identically
+    (unit tests pin them to byte-identical SQL; this covers the wiring once
+    end-to-end). An output-only aggregate is rejected under either spelling.
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-having"
+
+    small = _ai_trace(now=now, service=service, user="alice", in_tokens=10, out_tokens=20, cost=0.1)
+    large = _ai_trace(now=now, service=service, user="bob", in_tokens=10, out_tokens=500, cost=0.2)
+    small_id = small[0].trace_id
+    large_id = large[0].trace_id
+    insert_traces(small + large)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    for spelling in ("output_tokens", "trace.output_tokens"):
+        query = BuilderQuery(
+            signal="traces",
+            source="ai",
+            name="A",
+            filter_expression=f"service.name = '{service}' AND {spelling} > 100",
+            limit=10,
+        )
+        response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+        assert response.status_code == HTTPStatus.OK, f"{spelling}: {response.text}"
+
+        body = json.dumps(response.json())
+        assert large_id in body, f"{spelling}: trace with 500 out-tokens should pass > 100"
+        assert small_id not in body, f"{spelling}: trace with 20 out-tokens should be filtered out by HAVING"
+
+    # output-only aggregate gets the targeted rejection.
+    bad = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression="trace.span_count > 3",
+        limit=10,
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [bad.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "cannot be used" in response.text
+
+
+def test_ai_list_order_limit_offset(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """Trace list honors order by (aggregate column) + limit + offset (pagination)."""
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-order"
+
+    traces: list[Traces] = []
+    for out in (100, 200, 300, 400, 500):
+        traces += _ai_trace(now=now, service=service, user="u", in_tokens=10, out_tokens=out, cost=0.1)
+    insert_traces(traces)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    def page(offset: int) -> list[int]:
+        query = BuilderQuery(
+            signal="traces",
+            source="ai",
+            name="A",
+            filter_expression=f"service.name = '{service}'",
+            order=[OrderBy(key=TelemetryFieldKey(name="output_tokens"), direction="desc")],
+            limit=2,
+            offset=offset,
+        )
+        resp = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+        assert resp.status_code == HTTPStatus.OK, resp.text
+        rows = resp.json()["data"]["data"]["results"][0]["rows"]
+        return [int(r["data"]["output_tokens"]) for r in rows]
+
+    assert page(0) == [500, 400], "first page: highest output_tokens, desc"
+    assert page(2) == [300, 200], "second page (offset 2): next two, desc"
+
+
+def test_ai_span_list_limit(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """Span list honors limit (delegated raw path): 6 gen_ai spans available, capped to 4."""
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-spanlimit"
+    insert_traces(_ai_trace_mixed_spans(now=now, service=service, user="a") + _ai_trace_mixed_spans(now=now, service=service, user="b"))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression=f"service.name = '{service}'",
+        limit=4,
+    )
+    resp = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type=RequestType.RAW)
+    assert resp.status_code == HTTPStatus.OK, resp.text
+    rows = resp.json()["data"]["data"]["results"][0]["rows"]
+    assert len(rows) == 4, f"limit should cap at 4 (6 gen_ai spans available), got {len(rows)}"
+
+
+def test_ai_span_list_excludes_non_gen_ai_spans(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    Span list (requestType=raw): returns only the gen_ai spans (LLM/tool/agent); the
+    root span of the same trace (no gen_ai attributes) is excluded by the span-level gate.
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-spanlist"
+    insert_traces(_ai_trace_mixed_spans(now=now, service=service, user="alice"))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression=f"service.name = '{service}'",
+        select_fields=[TelemetryFieldKey(name="name", field_context="span")],
+        limit=50,
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type=RequestType.RAW)
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    rows = response.json()["data"]["data"]["results"][0]["rows"]
+    names = sorted(r["data"]["name"] for r in rows)
+    assert names == ["agent.step", "chat gpt-4o-mini", "execute_tool"], names
+    assert "POST /api/chat" not in names  # root span excluded
+
+
+def test_ai_list_having_or_aggregates(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    Two trace-level aggregates OR-ed within the filter box (regression guard for OR-group
+    whitespace handling): output_tokens > 100 OR input_tokens > 1000 keeps only the
+    large-output trace (input_tokens is 10 for both, so that branch never matches).
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-having-or"
+
+    small = _ai_trace(now=now, service=service, user="a", in_tokens=10, out_tokens=20, cost=0.1)
+    large = _ai_trace(now=now, service=service, user="b", in_tokens=10, out_tokens=500, cost=0.2)
+    small_id, large_id = small[0].trace_id, large[0].trace_id
+    insert_traces(small + large)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression=f"service.name = '{service}' AND (output_tokens > 100 OR input_tokens > 1000)",
+        limit=10,
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    body = json.dumps(response.json())
+    assert large_id in body
+    assert small_id not in body
+
+
+def test_ai_list_resource_filter_isolates_by_fingerprint(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    A resource attribute in the filter is pulled into the __resource_filter fingerprint
+    CTE (see maybeAttachResourceFilter). Two traces on the same service but different
+    deployment.environment: `resource.deployment.environment = 'production'` must keep
+    the production trace and drop the staging one — the fingerprint prune isolates by
+    the resource, not by any span attribute.
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-resfilter"
+
+    prod = _ai_trace(now=now, service=service, user="a", in_tokens=10, out_tokens=20, cost=0.1, environment="production")
+    stag = _ai_trace(now=now, service=service, user="b", in_tokens=10, out_tokens=20, cost=0.1, environment="staging")
+    prod_id, stag_id = prod[0].trace_id, stag[0].trace_id
+    insert_traces(prod + stag)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression=(f"resource.service.name = '{service}' AND resource.deployment.environment = 'production'"),
+        limit=10,
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    body = json.dumps(response.json())
+    assert prod_id in body, "production trace should match the resource filter"
+    assert stag_id not in body, "staging trace should be excluded by the resource fingerprint prune"
+
+
+def test_ai_list_rejects_aggregate_or_span_filter(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    Aggregate (HAVING) columns may not be OR-ed with span-level keys in the trace
+    list; a span-OR-span filter is fine.
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-orfilter"
+    # seed a trace so service.name resolves as a known key in this window (resource
+    # keys are discovered from ingested data).
+    insert_traces(_ai_trace(now=now, service=service, user="a", in_tokens=10, out_tokens=20, cost=0.1))
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    # aggregate OR span -> rejected
+    bad = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        limit=10,
+        filter_expression=f"output_tokens > 1000 OR service.name = '{service}'",
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [bad.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "cannot be combined" in response.text
+
+    # span OR span -> accepted (result content doesn't matter; just not an error)
+    ok = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        limit=10,
+        filter_expression=f"service.name = '{service}' OR has_error = true",
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [ok.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.OK, response.text
+
+
+def test_ai_list_nested_group_span_or_and_aggregate(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    A complex filter that mixes all three routing paths in one expression:
+        service.name = X AND (has_error = true OR gen_ai.request.model = 'gpt-4o') AND total_tokens > 100
+    The nested (span OR span) group must not flatten (precedence), the span predicates
+    go to WHERE as a trace-existence check, and the new `total_tokens` aggregate goes to
+    HAVING. Three traces isolate each discriminator:
+      - t_ok:      gpt-4o, out=500 -> OR matches (model) AND total_tokens>100  -> IN
+      - t_or_miss: gpt-4o-mini, out=500 -> OR fails (no error, wrong model)    -> OUT
+      - t_agg_miss: gpt-4o, out=20  -> OR matches but total_tokens<=100         -> OUT
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-nested"
+
+    t_ok = _ai_trace(now=now, service=service, user="a", model="gpt-4o", in_tokens=10, out_tokens=500, cost=0.1)
+    t_or_miss = _ai_trace(now=now, service=service, user="b", model="gpt-4o-mini", in_tokens=10, out_tokens=500, cost=0.1)
+    t_agg_miss = _ai_trace(now=now, service=service, user="c", model="gpt-4o", in_tokens=10, out_tokens=20, cost=0.1)
+    insert_traces(t_ok + t_or_miss + t_agg_miss)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression=(f"service.name = '{service}' AND (has_error = true OR gen_ai.request.model = 'gpt-4o') AND total_tokens > 100"),
+        limit=10,
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    body = json.dumps(response.json())
+    assert t_ok[0].trace_id in body
+    assert t_or_miss[0].trace_id not in body, "nested (span OR span) group must exclude the wrong-model, no-error trace"
+    assert t_agg_miss[0].trace_id not in body, "HAVING total_tokens > 100 must exclude the low-token trace"
+
+
+def test_ai_list_rejects_unknown_aggregate_key(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    """A trace-level filter on an unknown aggregate name is rejected, not silently run."""
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        limit=10,
+        filter_expression="trace.bogus_tokens > 1",
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+
+
+def test_ai_list_rejects_order_by_span_attribute(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    """Only gen_ai-scoped aggregates are orderable; ordering by a span/resource key errors."""
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        limit=5,
+        order=[OrderBy(key=TelemetryFieldKey(name="service.name"), direction="asc")],
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+    assert "order key" in response.text
+
+
+def test_ai_list_total_tokens_output_only(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    A trace whose LLM span carries only output tokens (no input-tokens attribute at
+    all) must still total: total_tokens is coalesce(sum(in),0)+coalesce(sum(out),0),
+    since sum over an absent attribute is NULL and NULL + n = NULL in ClickHouse.
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-total-coalesce"
+    insert_traces(_ai_trace(now=now, service=service, user="a", in_tokens=None, out_tokens=300, cost=0.1))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression=f"service.name = '{service}'",
+        limit=10,
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    rows = response.json()["data"]["data"]["results"][0]["rows"]
+    assert len(rows) == 1, f"expected one trace, got: {rows}"
+    data = rows[0]["data"]
+    assert data["input_tokens"] is None, data  # attribute absent -> NULL, not 0
+    assert data["output_tokens"] == 300, data
+    assert data["total_tokens"] == 300, f"total must coalesce the missing input side: {data}"
+
+
+def test_ai_list_variable_in_aggregate_filter(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """A query variable in a trace-level condition is substituted into the HAVING."""
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-having-var"
+
+    small = _ai_trace(now=now, service=service, user="a", in_tokens=10, out_tokens=20, cost=0.1)
+    large = _ai_trace(now=now, service=service, user="b", in_tokens=10, out_tokens=500, cost=0.2)
+    small_id, large_id = small[0].trace_id, large[0].trace_id
+    insert_traces(small + large)
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        filter_expression=f"service.name = '{service}' AND trace.output_tokens > $threshold",
+        limit=10,
+    )
+    response = make_query_request(
+        signoz,
+        token,
+        start_ms,
+        end_ms,
+        [query.to_dict()],
+        request_type="trace",
+        variables={"threshold": {"type": "custom", "value": 100}},
+    )
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    body = json.dumps(response.json())
+    assert large_id in body
+    assert small_id not in body
+
+
+def _ai_trace_two_llm(*, now: datetime, service: str) -> list[Traces]:
+    """Root + two LLM spans at different times, each with distinct input/output messages."""
+    trace_id = TraceIdGenerator.trace_id()
+    root_id = TraceIdGenerator.span_id()
+    resources = {"service.name": service}
+
+    def _llm(offset_s: float, prompt: str, answer: str) -> Traces:
+        return Traces(
+            timestamp=now - timedelta(seconds=offset_s),
+            duration=timedelta(seconds=1),
+            trace_id=trace_id,
+            span_id=TraceIdGenerator.span_id(),
+            parent_span_id=root_id,
+            name="chat",
+            kind=TracesKind.SPAN_KIND_CLIENT,
+            status_code=TracesStatusCode.STATUS_CODE_OK,
+            resources=resources,
+            attributes={
+                "gen_ai.request.model": "gpt-4o-mini",
+                "gen_ai.input.messages": prompt,
+                "gen_ai.output.messages": answer,
+            },
+        )
+
+    root = Traces(
+        timestamp=now - timedelta(seconds=5),
+        duration=timedelta(seconds=4),
+        trace_id=trace_id,
+        span_id=root_id,
+        parent_span_id="",
+        name="POST /api/chat",
+        kind=TracesKind.SPAN_KIND_SERVER,
+        status_code=TracesStatusCode.STATUS_CODE_OK,
+        resources=resources,
+        attributes={"http.request.method": "POST"},
+    )
+    # earlier call is the "first" (its input is the prompt), later call is the "last"
+    # (its output is the final answer).
+    first = _llm(4, "first prompt", "first answer")
+    last = _llm(2, "second prompt", "second answer")
+    return [root, first, last]
+
+
+def test_ai_list_messages_first_input_last_output(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    `input` is the FIRST LLM span's prompt (argMin over timestamp) and `output` is the
+    LAST LLM span's answer (argMax) — the question -> final-answer preview.
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-messages"
+    insert_traces(_ai_trace_two_llm(now=now, service=service))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        limit=10,
+        filter_expression=f"service.name = '{service}'",
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    rows = response.json()["data"]["data"]["results"][0]["rows"]
+    assert len(rows) == 1, f"expected one trace, got: {rows}"
+    data = rows[0]["data"]
+    assert data["input"] == "first prompt", f"input should be the earliest call's prompt: {data}"
+    assert data["output"] == "second answer", f"output should be the latest call's answer: {data}"
+
+
+def _ai_trace_for_metrics(*, now: datetime, service: str) -> list[Traces]:
+    """
+    Root + one errored LLM span (tokens/cost) + three tool spans (two 'get_weather',
+    one 'get_time') + one agent span, so the derived per-trace metrics have distinct
+    expected values. The agent span is in the gen_ai gate but carries no request.model,
+    so it must NOT count toward llm_call_count (only span_count / last_activity_time).
+    """
+    trace_id = TraceIdGenerator.trace_id()
+    root_id = TraceIdGenerator.span_id()
+    resources = {"service.name": service}
+
+    def _tool(name: str, offset_s: float) -> Traces:
+        return Traces(
+            timestamp=now - timedelta(seconds=offset_s),
+            duration=timedelta(seconds=0.2),
+            trace_id=trace_id,
+            span_id=TraceIdGenerator.span_id(),
+            parent_span_id=root_id,
+            name="execute_tool",
+            kind=TracesKind.SPAN_KIND_INTERNAL,
+            status_code=TracesStatusCode.STATUS_CODE_OK,
+            resources=resources,
+            attributes={"gen_ai.tool.name": name, "gen_ai.tool.type": "function"},
+        )
+
+    root = Traces(
+        timestamp=now - timedelta(seconds=5),
+        duration=timedelta(seconds=4),
+        trace_id=trace_id,
+        span_id=root_id,
+        parent_span_id="",
+        name="POST /api/chat",
+        kind=TracesKind.SPAN_KIND_SERVER,
+        status_code=TracesStatusCode.STATUS_CODE_OK,
+        resources=resources,
+        attributes={"http.request.method": "POST"},
+    )
+    llm = Traces(
+        timestamp=now - timedelta(seconds=4),
+        duration=timedelta(seconds=2),
+        trace_id=trace_id,
+        span_id=TraceIdGenerator.span_id(),
+        parent_span_id=root_id,
+        name="chat gpt-4o-mini",
+        kind=TracesKind.SPAN_KIND_CLIENT,
+        status_code=TracesStatusCode.STATUS_CODE_ERROR,  # -> has_error, drives error_count
+        resources=resources,
+        attributes={
+            "gen_ai.request.model": "gpt-4o-mini",
+            "gen_ai.usage.input_tokens": 100,
+            "gen_ai.usage.output_tokens": 20,
+            "_signoz.gen_ai.total_cost": 0.5,
+        },
+    )
+    agent = Traces(
+        timestamp=now - timedelta(seconds=1),
+        duration=timedelta(seconds=0.5),
+        trace_id=trace_id,
+        span_id=TraceIdGenerator.span_id(),
+        parent_span_id=root_id,
+        name="agent.step",
+        kind=TracesKind.SPAN_KIND_INTERNAL,
+        status_code=TracesStatusCode.STATUS_CODE_OK,
+        resources=resources,
+        attributes={"gen_ai.agent.name": "chat-agent"},
+    )
+    return [root, llm, _tool("get_weather", 3), _tool("get_weather", 2.5), _tool("get_time", 2), agent]
+
+
+def test_ai_list_enrichment_values(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    End-to-end values of the derived per-trace columns (only integration can check that
+    ClickHouse computes uniqIf / sum+sum / countIf(predicate) correctly, not just that
+    the SQL is shaped right). One trace: root + 1 errored LLM + 3 tool spans
+    (get_weather x2, get_time x1) + 1 agent span. The tool and agent spans are in the
+    gen_ai gate but carry no request.model, so llm_call_count stays 1 while span_count
+    counts them all.
+    """
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    service = "ai-it-metrics"
+    insert_traces(_ai_trace_for_metrics(now=now, service=service))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    start_ms, end_ms = _window_ms(now)
+
+    query = BuilderQuery(
+        signal="traces",
+        source="ai",
+        name="A",
+        limit=10,
+        filter_expression=f"service.name = '{service}'",
+    )
+    response = make_query_request(signoz, token, start_ms, end_ms, [query.to_dict()], request_type="trace")
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    rows = response.json()["data"]["data"]["results"][0]["rows"]
+    assert len(rows) == 1, f"expected one trace, got: {rows}"
+    data = rows[0]["data"]
+
+    assert data["span_count"] == 6, data  # root + llm + 3 tools + agent
+    assert data["llm_call_count"] == 1, data  # only the request.model span, not tool/agent
+    assert data["tool_call_count"] == 3, data  # all three tool spans
+    assert data["distinct_tool_count"] == 2, data  # get_weather, get_time
+    assert data["input_tokens"] == 100, data
+    assert data["output_tokens"] == 20, data
+    assert data["total_tokens"] == 120, data  # input + output
+    assert data["estimated_cost_usd"] == pytest.approx(0.5), data
+    assert data["error_count"] == 1, data  # the errored LLM span
+    assert data["max_llm_latency_ns"] > 0, data  # scoped max over LLM spans