Merge branch 'main' into issue_4522

Co-authored-by: Gaurav Tewari <tewarig@users.noreply.github.com>

frontend/src/components/Logs/TableView/useLogsTableColumns.tsx

@@ -69,6 +69,8 @@ export function useLogsTableColumns({
 					id: 'timestamp',
 					header: 'Timestamp',
 					accessorFn: (log): unknown => log.timestamp,
+					canBeHidden: false,
+					enableRemove: false,
 					width: { default: 170, min: 170 },
 					cell: ({ value }): ReactElement => {
 						const ts = value as string | number;
@@ -92,6 +94,7 @@ export function useLogsTableColumns({
 					header: 'Body',
 					accessorFn: (log): string => getBodyDisplayString(log.body),
 					canBeHidden: false,
+					enableRemove: false,
 					width: { default: '100%', min: 300 },
 					cell: ({ value, isActive }): ReactElement => (
 						<TanStackTable.Text

frontend/src/components/LogsFormatOptionsMenu/__tests__/LogsFormatOptionsMenu.test.tsx

@@ -62,6 +62,7 @@ describe('LogsFormatOptionsMenu (unit)', () => {
 						onSearch: jest.fn(),
 						onSelect: jest.fn(),
 						onRemove: jest.fn(),
+						onReorder: jest.fn(),
 					},
 				}}
 			/>,

frontend/src/components/TanStackTableView/TanStackTable.tsx

@@ -322,9 +322,7 @@ function TanStackTableInner<TData>(
 	});
 
 	const hasSingleColumn = useMemo(
-		() =>
-			effectiveColumns.filter((c) => !c.pin && c.enableRemove !== false).length <=
-			1,
+		() => effectiveColumns.filter((c) => !c.pin).length <= 1,
 		[effectiveColumns],
 	);
 

frontend/src/container/LiveLogs/LiveLogsList/index.tsx

@@ -16,7 +16,6 @@ import { useLogsTableColumns } from 'components/Logs/TableView/useLogsTableColum
 import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
 import type { TanStackTableHandle } from 'components/TanStackTableView';
 import TanStackTable from 'components/TanStackTableView';
-import { useHiddenColumnIds } from 'components/TanStackTableView/useColumnStore';
 import { CARD_BODY_STYLE } from 'constants/card';
 import { LOCALSTORAGE } from 'constants/localStorage';
 import { OptionFormatTypes } from 'constants/optionsFormatTypes';
@@ -24,13 +23,11 @@ import { QueryParams } from 'constants/query';
 import { InfinityWrapperStyled } from 'container/LogsExplorerList/styles';
 import { convertKeysToColumnFields } from 'container/LogsExplorerList/utils';
 import { useOptionsMenu } from 'container/OptionsMenu';
-import { defaultLogsSelectedColumns } from 'container/OptionsMenu/constants';
 import { useCopyLogLink } from 'hooks/logs/useCopyLogLink';
 import useLogDetailHandlers from 'hooks/logs/useLogDetailHandlers';
 import useScrollToLog from 'hooks/logs/useScrollToLog';
 import { useIsDarkMode } from 'hooks/useDarkMode';
 import { useEventSource } from 'providers/EventSource';
-import { usePreferenceContext } from 'providers/preferences/context/PreferenceContextProvider';
 // interfaces
 import { ILog } from 'types/api/logs/log';
 import { DataSource, StringOperators } from 'types/common/queryBuilder';
@@ -54,9 +51,6 @@ function LiveLogsList({
 	const { isConnectionLoading } = useEventSource();
 
 	const { activeLogId } = useCopyLogLink();
-	const { logs: logsPreferences } = usePreferenceContext();
-	const hiddenColumnIds = useHiddenColumnIds(LOCALSTORAGE.LOGS_LIST_COLUMNS);
-	const hasReconciledHiddenColumnsRef = useRef(false);
 
 	const {
 		activeLog,
@@ -72,7 +66,7 @@ function LiveLogsList({
 		[logs],
 	);
 
-	const { options } = useOptionsMenu({
+	const { options, config } = useOptionsMenu({
 		storageKey: LOCALSTORAGE.LOGS_LIST_OPTIONS,
 		dataSource: DataSource.LOGS,
 		aggregateOperator: StringOperators.NOOP,
@@ -83,16 +77,7 @@ function LiveLogsList({
 		[formattedLogs, activeLogId],
 	);
 
-	const selectedFields = convertKeysToColumnFields([
-		...defaultLogsSelectedColumns,
-		...options.selectColumns,
-	]);
-
-	const syncedSelectedColumns = useMemo(
-		() =>
-			options.selectColumns.filter(({ name }) => !hiddenColumnIds.includes(name)),
-		[options.selectColumns, hiddenColumnIds],
-	);
+	const selectedFields = convertKeysToColumnFields(options.selectColumns);
 
 	const logsColumns = useLogsTableColumns({
 		fields: selectedFields,
@@ -100,30 +85,6 @@ function LiveLogsList({
 		appendTo: 'end',
 	});
 
-	useEffect(() => {
-		if (hasReconciledHiddenColumnsRef.current) {
-			return;
-		}
-
-		hasReconciledHiddenColumnsRef.current = true;
-
-		if (syncedSelectedColumns.length === options.selectColumns.length) {
-			return;
-		}
-
-		logsPreferences.updateColumns(syncedSelectedColumns);
-	}, [logsPreferences, options.selectColumns.length, syncedSelectedColumns]);
-
-	const handleColumnRemove = useCallback(
-		(columnId: string) => {
-			const updatedColumns = options.selectColumns.filter(
-				({ name }) => name !== columnId,
-			);
-			logsPreferences.updateColumns(updatedColumns);
-		},
-		[options.selectColumns, logsPreferences],
-	);
-
 	const makeOnLogCopy = useCallback(
 		(log: ILog) =>
 			(event: MouseEvent<HTMLElement>): void => {
@@ -237,7 +198,7 @@ function LiveLogsList({
 							ref={ref as React.Ref<TanStackTableHandle>}
 							columns={logsColumns}
 							columnStorageKey={LOCALSTORAGE.LOGS_LIST_COLUMNS}
-							onColumnRemove={handleColumnRemove}
+							onColumnRemove={config?.addColumn?.onRemove}
 							plainTextCellLineClamp={options.maxLines}
 							cellTypographySize={options.fontSize}
 							data={formattedLogs}

frontend/src/container/LogsExplorerList/index.tsx

@@ -18,21 +18,19 @@ import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
 import Spinner from 'components/Spinner';
 import type { TanStackTableHandle } from 'components/TanStackTableView';
 import TanStackTable from 'components/TanStackTableView';
-import { useHiddenColumnIds } from 'components/TanStackTableView/useColumnStore';
+import type { TableColumnDef } from 'components/TanStackTableView/types';
 import { CARD_BODY_STYLE } from 'constants/card';
 import { LOCALSTORAGE } from 'constants/localStorage';
 import { QueryParams } from 'constants/query';
 import EmptyLogsSearch from 'container/EmptyLogsSearch/EmptyLogsSearch';
 import { LogsLoading } from 'container/LogsLoading/LogsLoading';
 import { useOptionsMenu } from 'container/OptionsMenu';
-import { defaultLogsSelectedColumns } from 'container/OptionsMenu/constants';
 import { FontSize } from 'container/OptionsMenu/types';
 import { useCopyLogLink } from 'hooks/logs/useCopyLogLink';
 import useLogDetailHandlers from 'hooks/logs/useLogDetailHandlers';
 import useScrollToLog from 'hooks/logs/useScrollToLog';
 import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
 import { useIsDarkMode } from 'hooks/useDarkMode';
-import { usePreferenceContext } from 'providers/preferences/context/PreferenceContextProvider';
 import APIError from 'types/api/error';
 // interfaces
 import { ILog } from 'types/api/logs/log';
@@ -69,10 +67,6 @@ function LogsExplorerList({
 	const [, setCopy] = useCopyToClipboard();
 	const isDarkMode = useIsDarkMode();
 	const { activeLogId } = useCopyLogLink();
-	const { logs: logsPreferences } = usePreferenceContext();
-	const hiddenColumnIds = useHiddenColumnIds(LOCALSTORAGE.LOGS_LIST_COLUMNS);
-	const hasReconciledHiddenColumnsRef = useRef(false);
-
 	const {
 		activeLog,
 		onAddToQuery,
@@ -81,7 +75,7 @@ function LogsExplorerList({
 		handleCloseLogDetail,
 	} = useLogDetailHandlers();
 
-	const { options } = useOptionsMenu({
+	const { options, config } = useOptionsMenu({
 		storageKey: LOCALSTORAGE.LOGS_LIST_OPTIONS,
 		dataSource: DataSource.LOGS,
 		aggregateOperator:
@@ -97,28 +91,15 @@ function LogsExplorerList({
 	);
 
 	const selectedFields = useMemo(
-		() =>
-			convertKeysToColumnFields([
-				...defaultLogsSelectedColumns,
-				...options.selectColumns,
-			]),
-		[options],
+		() => convertKeysToColumnFields(options.selectColumns),
+		[options.selectColumns],
 	);
 
-	const syncedSelectedColumns = useMemo(
-		() =>
-			options.selectColumns.filter(({ name }) => !hiddenColumnIds.includes(name)),
-		[options.selectColumns, hiddenColumnIds],
-	);
-
-	const handleColumnRemove = useCallback(
-		(columnId: string) => {
-			const updatedColumns = options.selectColumns.filter(
-				({ name }) => name !== columnId,
-			);
-			logsPreferences.updateColumns(updatedColumns);
+	const handleColumnOrderChange = useCallback(
+		(cols: TableColumnDef<ILog>[]): void => {
+			config?.addColumn?.onReorder(cols.map((c) => c.id));
 		},
-		[options.selectColumns, logsPreferences],
+		[config],
 	);
 
 	const logsColumns = useLogsTableColumns({
@@ -161,20 +142,6 @@ function LogsExplorerList({
 		}
 	}, [isLoading, isFetching, isError, logs.length]);
 
-	useEffect(() => {
-		if (hasReconciledHiddenColumnsRef.current) {
-			return;
-		}
-
-		hasReconciledHiddenColumnsRef.current = true;
-
-		if (syncedSelectedColumns.length === options.selectColumns.length) {
-			return;
-		}
-
-		logsPreferences.updateColumns(syncedSelectedColumns);
-	}, [logsPreferences, options.selectColumns.length, syncedSelectedColumns]);
-
 	const getItemContent = useCallback(
 		(_: number, log: ILog): JSX.Element => {
 			if (options.format === 'raw') {
@@ -237,7 +204,8 @@ function LogsExplorerList({
 					ref={ref as React.Ref<TanStackTableHandle>}
 					columns={logsColumns}
 					columnStorageKey={LOCALSTORAGE.LOGS_LIST_COLUMNS}
-					onColumnRemove={handleColumnRemove}
+					onColumnRemove={config?.addColumn?.onRemove}
+					onColumnOrderChange={handleColumnOrderChange}
 					plainTextCellLineClamp={options.maxLines}
 					cellTypographySize={options.fontSize}
 					data={logs}

frontend/src/container/OptionsMenu/__test__/ensureLogsRequiredColumns.test.ts

@@ -0,0 +1,79 @@
+import { TelemetryFieldKey } from 'api/v5/v5';
+
+import {
+	defaultLogsSelectedColumns,
+	ensureLogsRequiredColumns,
+} from '../constants';
+
+const TIMESTAMP = defaultLogsSelectedColumns.find(
+	(c) => c.name === 'timestamp',
+);
+const BODY = defaultLogsSelectedColumns.find((c) => c.name === 'body');
+
+if (!TIMESTAMP || !BODY) {
+	throw new Error('defaults missing timestamp/body — test fixture invalid');
+}
+
+const ATTR_A: TelemetryFieldKey = {
+	name: 'service.name',
+	signal: 'logs',
+	fieldContext: 'resource',
+	fieldDataType: 'string',
+};
+const ATTR_B: TelemetryFieldKey = {
+	name: 'severity_text',
+	signal: 'logs',
+	fieldContext: 'log',
+	fieldDataType: 'string',
+};
+
+describe('ensureLogsRequiredColumns', () => {
+	it('prepends both timestamp + body to an empty list', () => {
+		expect(ensureLogsRequiredColumns([])).toStrictEqual([TIMESTAMP, BODY]);
+	});
+
+	it('prepends only `body` when `timestamp` is already present', () => {
+		expect(ensureLogsRequiredColumns([TIMESTAMP, ATTR_A])).toStrictEqual([
+			BODY,
+			TIMESTAMP,
+			ATTR_A,
+		]);
+	});
+
+	it('prepends only `timestamp` when `body` is already present', () => {
+		expect(ensureLogsRequiredColumns([BODY, ATTR_A])).toStrictEqual([
+			TIMESTAMP,
+			BODY,
+			ATTR_A,
+		]);
+	});
+
+	it('returns the same array when both are present (no duplicates, original order preserved)', () => {
+		const input = [TIMESTAMP, BODY, ATTR_A, ATTR_B];
+		expect(ensureLogsRequiredColumns(input)).toBe(input);
+	});
+
+	it('preserves a non-default order when both are present', () => {
+		const input = [ATTR_A, BODY, ATTR_B, TIMESTAMP];
+		expect(ensureLogsRequiredColumns(input)).toStrictEqual(input);
+	});
+
+	it('prepends both when neither is present in a list of user attributes', () => {
+		expect(ensureLogsRequiredColumns([ATTR_A, ATTR_B])).toStrictEqual([
+			TIMESTAMP,
+			BODY,
+			ATTR_A,
+			ATTR_B,
+		]);
+	});
+
+	it('does not duplicate if a required column appears twice in the input', () => {
+		// Tolerant of malformed input — invariant only adds *missing* required
+		// columns; it does not deduplicate existing entries (that's a separate
+		// concern, not its job).
+		const input = [BODY, BODY, ATTR_A];
+		const result = ensureLogsRequiredColumns(input);
+		expect(result.filter((c) => c.name === 'timestamp')).toHaveLength(1);
+		expect(result[0]).toStrictEqual(TIMESTAMP);
+	});
+});

frontend/src/container/OptionsMenu/constants.ts

@@ -35,6 +35,32 @@ export const defaultLogsSelectedColumns: TelemetryFieldKey[] = [
 	},
 ];
 
+const LOGS_REQUIRED_COLUMNS = ['timestamp', 'body'] as const;
+
+/**
+ * Always-on invariant: every logs selectColumns array must contain `body` and
+ * `timestamp`. Applied at both loader and writer boundaries so the picker, the
+ * table, and persisted state can never diverge into a "missing required
+ * column" state.
+ */
+export function ensureLogsRequiredColumns(
+	columns: TelemetryFieldKey[],
+): TelemetryFieldKey[] {
+	const missing = LOGS_REQUIRED_COLUMNS.filter(
+		(name) => !columns.some((c) => c.name === name),
+	);
+	if (missing.length === 0) {
+		return columns;
+	}
+	const defaultsByName = new Map(
+		defaultLogsSelectedColumns.map((c) => [c.name, c]),
+	);
+	const prepended = missing
+		.map((name) => defaultsByName.get(name))
+		.filter((c): c is TelemetryFieldKey => c !== undefined);
+	return [...prepended, ...columns];
+}
+
 export const defaultTraceSelectedColumns: TelemetryFieldKey[] = [
 	{
 		name: 'service.name',

frontend/src/container/OptionsMenu/types.ts

@@ -40,5 +40,6 @@ export type OptionsMenuConfig = {
 		isFetching: boolean;
 		value: TelemetryFieldKey[];
 		onRemove: (key: string) => void;
+		onReorder: (orderedIds: string[]) => void;
 	};
 };

frontend/src/container/OptionsMenu/useOptionsMenu.ts

@@ -187,30 +187,6 @@ const useOptionsMenu = ({
 			searchedAttributesDataV5?.data.data.keys || {},
 		).flat();
 		if (searchedAttributesDataList.length) {
-			if (dataSource === DataSource.LOGS) {
-				const logsSelectedColumns: TelemetryFieldKey[] =
-					defaultLogsSelectedColumns.map((e) => ({
-						...e,
-						name: e.name,
-						signal: e.signal as SignalType,
-						fieldContext: e.fieldContext as FieldContext,
-						fieldDataType: e.fieldDataType as FieldDataType,
-					}));
-				return [
-					...logsSelectedColumns,
-					...searchedAttributesDataList
-						.filter((attribute) => attribute.name !== 'body')
-						// eslint-disable-next-line sonarjs/no-identical-functions
-						.map((e) => ({
-							...e,
-							name: e.name,
-							signal: e.signal as SignalType,
-							fieldContext: e.fieldContext as FieldContext,
-							fieldDataType: e.fieldDataType as FieldDataType,
-						})),
-				];
-			}
-			// eslint-disable-next-line sonarjs/no-identical-functions
 			return searchedAttributesDataList.map((e) => ({
 				...e,
 				name: e.name,
@@ -297,24 +273,9 @@ const useOptionsMenu = ({
 				return [...acc, column];
 			}, [] as TelemetryFieldKey[]);
 
-			const optionsData: OptionsQuery = {
-				...defaultOptionsQuery,
-				selectColumns: newSelectedColumns,
-				format: preferences?.formatting?.format || defaultOptionsQuery.format,
-				maxLines: preferences?.formatting?.maxLines || defaultOptionsQuery.maxLines,
-				fontSize: preferences?.formatting?.fontSize || defaultOptionsQuery.fontSize,
-			};
-
 			updateColumns(newSelectedColumns);
-			handleRedirectWithOptionsData(optionsData);
 		},
-		[
-			searchedAttributeKeys,
-			selectedColumnKeys,
-			preferences,
-			handleRedirectWithOptionsData,
-			updateColumns,
-		],
+		[searchedAttributeKeys, selectedColumnKeys, preferences, updateColumns],
 	);
 
 	const handleRemoveSelectedColumn = useCallback(
@@ -327,27 +288,12 @@ const useOptionsMenu = ({
 				notifications.error({
 					message: 'There must be at least one selected column',
 				});
-			} else {
-				const optionsData: OptionsQuery = {
-					...defaultOptionsQuery,
-					selectColumns: newSelectedColumns || [],
-					format: preferences?.formatting?.format || defaultOptionsQuery.format,
-					maxLines:
-						preferences?.formatting?.maxLines || defaultOptionsQuery.maxLines,
-					fontSize:
-						preferences?.formatting?.fontSize || defaultOptionsQuery.fontSize,
-				};
-				updateColumns(newSelectedColumns || []);
-				handleRedirectWithOptionsData(optionsData);
+				return;
 			}
+
+			updateColumns(newSelectedColumns || []);
 		},
-		[
-			dataSource,
-			notifications,
-			preferences,
-			handleRedirectWithOptionsData,
-			updateColumns,
-		],
+		[dataSource, notifications, preferences, updateColumns],
 	);
 
 	const handleFormatChange = useCallback(
@@ -414,6 +360,18 @@ const useOptionsMenu = ({
 		setSearchText(value);
 	}, []);
 
+	const reorderSelectColumns = useCallback(
+		(orderedIds: string[]): void => {
+			const current = preferences?.columns ?? [];
+			const byName = new Map(current.map((f) => [f.name, f]));
+			const reordered = orderedIds
+				.map((id) => byName.get(id))
+				.filter((f): f is TelemetryFieldKey => f !== undefined);
+			updateColumns(reordered);
+		},
+		[preferences, updateColumns],
+	);
+
 	const handleFocus = (): void => {
 		setIsFocused(true);
 	};
@@ -436,6 +394,7 @@ const useOptionsMenu = ({
 				onSelect: handleSelectColumns,
 				onRemove: handleRemoveSelectedColumn,
 				onSearch: handleSearchAttribute,
+				onReorder: reorderSelectColumns,
 			},
 			format: {
 				value: preferences?.formatting?.format || defaultOptionsQuery.format,
@@ -457,6 +416,7 @@ const useOptionsMenu = ({
 			handleSelectColumns,
 			handleRemoveSelectedColumn,
 			handleSearchAttribute,
+			reorderSelectColumns,
 			handleFormatChange,
 			handleMaxLinesChange,
 			handleFontSizeChange,

frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/OptionRenderer.styles.scss

@@ -9,6 +9,19 @@
 		width: 0.2rem;
 		height: 0.2rem;
 	}
+
+	.option-value {
+		overflow: hidden;
+		text-overflow: ellipsis;
+		white-space: nowrap;
+		min-width: 0;
+	}
+
+	.option-meta-data-container {
+		display: flex;
+		gap: 8px;
+		flex-shrink: 0;
+	}
 }
 
 .option-renderer-tooltip {

frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/style.ts

@@ -24,7 +24,7 @@ export const StyledCheckOutlined = styled(Check)`
 
 export const TagContainer = styled(Badge)`
 	&&& {
-		display: inline-block;
+		display: flex;
 		border-radius: 3px;
 		padding: 0.1rem 0.2rem;
 		font-weight: 300;

frontend/src/container/TracesExplorer/ListView/index.tsx

@@ -30,10 +30,7 @@ import { useGetQueryRange } from 'hooks/queryBuilder/useGetQueryRange';
 import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
 import { Pagination } from 'hooks/queryPagination';
 import { getDefaultPaginationConfig } from 'hooks/queryPagination/utils';
-import useDragColumns from 'hooks/useDragColumns';
-import { getDraggedColumns } from 'hooks/useDragColumns/utils';
 import useUrlQueryData from 'hooks/useUrlQueryData';
-import { RowData } from 'lib/query/createTableColumnsFromQuery';
 import { ArrowUp10, Minus } from '@signozhq/icons';
 import { useTimezone } from 'providers/Timezone';
 import { AppState } from 'store/reducers';
@@ -85,10 +82,6 @@ function ListView({
 		},
 	});
 
-	const { draggedColumns, onDragColumns } = useDragColumns<RowData>(
-		LOCALSTORAGE.TRACES_LIST_COLUMNS,
-	);
-
 	const { queryData: paginationQueryData } = useUrlQueryData<Pagination>(
 		QueryParams.pagination,
 	);
@@ -100,6 +93,19 @@ function ListView({
 		[stagedQuery, orderBy],
 	);
 
+	// TEMP — remove after traces moves to TanStack table.
+	// - Drag updates selectColumns; raw queryKey would churn on reorder.
+	// - Trace API fetches only listed columns → add/remove must refetch.
+	// - Sorted-name signature: stable on reorder, changes on add/remove.
+	const selectColumnsSignature = useMemo(
+		() =>
+			(options?.selectColumns ?? [])
+				.map((c) => c.name)
+				.sort()
+				.join(','),
+		[options?.selectColumns],
+	);
+
 	const queryKey = useMemo(
 		() => [
 			REACT_QUERY_KEY.GET_QUERY_RANGE,
@@ -109,7 +115,7 @@ function ListView({
 			stagedQuery,
 			panelType,
 			paginationConfig,
-			options?.selectColumns,
+			selectColumnsSignature,
 			orderBy,
 		],
 		[
@@ -117,7 +123,7 @@ function ListView({
 			panelType,
 			globalSelectedTime,
 			paginationConfig,
-			options?.selectColumns,
+			selectColumnsSignature,
 			maxTime,
 			minTime,
 			orderBy,
@@ -182,13 +188,14 @@ function ListView({
 
 	const { formatTimezoneAdjustedTimestamp } = useTimezone();
 
-	const columns = useMemo(() => {
-		const updatedColumns = getListColumns(
-			options?.selectColumns || [],
-			formatTimezoneAdjustedTimestamp,
-		);
-		return getDraggedColumns(updatedColumns, draggedColumns);
-	}, [options?.selectColumns, formatTimezoneAdjustedTimestamp, draggedColumns]);
+	const columns = useMemo(
+		() =>
+			getListColumns(
+				options?.selectColumns || [],
+				formatTimezoneAdjustedTimestamp,
+			),
+		[options?.selectColumns, formatTimezoneAdjustedTimestamp],
+	);
 
 	const transformedQueryTableData = useMemo(
 		() => transformDataWithDate(queryTableData) || [],
@@ -196,9 +203,16 @@ function ListView({
 	);
 
 	const handleDragColumn = useCallback(
-		(fromIndex: number, toIndex: number) =>
-			onDragColumns(columns, fromIndex, toIndex),
-		[columns, onDragColumns],
+		(fromIndex: number, toIndex: number): void => {
+			const reordered = [...columns];
+			const [moved] = reordered.splice(fromIndex, 1);
+			reordered.splice(toIndex, 0, moved);
+			const orderedIds = reordered
+				.map((c) => String(('dataIndex' in c && c.dataIndex) || c.key || ''))
+				.filter(Boolean);
+			config?.addColumn?.onReorder(orderedIds);
+		},
+		[columns, config],
 	);
 
 	const handleOrderChange = useCallback((value: string) => {

frontend/src/pages/LogsExplorer/index.tsx

@@ -3,7 +3,6 @@ import { useQueryClient } from 'react-query';
 import * as Sentry from '@sentry/react';
 import getLocalStorageKey from 'api/browser/localstorage/get';
 import setLocalStorageApi from 'api/browser/localstorage/set';
-import { TelemetryFieldKey } from 'api/v5/v5';
 import cx from 'classnames';
 import ExplorerCard from 'components/ExplorerCard/ExplorerCard';
 import QueryCancelledPlaceholder from 'components/QueryCancelledPlaceholder';
@@ -15,12 +14,6 @@ import { PANEL_TYPES } from 'constants/queryBuilder';
 import { usePageActions } from 'container/AIAssistant/pageActions/usePageActions';
 import LogExplorerQuerySection from 'container/LogExplorerQuerySection';
 import LogsExplorerViewsContainer from 'container/LogsExplorerViews';
-import {
-	defaultLogsSelectedColumns,
-	defaultOptionsQuery,
-	URL_OPTIONS,
-} from 'container/OptionsMenu/constants';
-import { OptionsQuery } from 'container/OptionsMenu/types';
 import LeftToolbarActions from 'container/QueryBuilder/components/ToolbarActions/LeftToolbarActions';
 import RightToolbarActions from 'container/QueryBuilder/components/ToolbarActions/RightToolbarActions';
 import Toolbar from 'container/Toolbar/Toolbar';
@@ -31,11 +24,9 @@ import {
 	useHandleExplorerTabChange,
 } from 'hooks/useHandleExplorerTabChange';
 import { useIsAIAssistantEnabled } from 'hooks/useIsAIAssistantEnabled';
-import useUrlQueryData from 'hooks/useUrlQueryData';
-import { defaultTo, isEmpty, isEqual, isNull } from 'lodash-es';
+import { defaultTo, isEmpty, isNull } from 'lodash-es';
 import ErrorBoundaryFallback from 'pages/ErrorBoundaryFallback/ErrorBoundaryFallback';
 import { EventSourceProvider } from 'providers/EventSource';
-import { usePreferenceContext } from 'providers/preferences/context/PreferenceContextProvider';
 import { Warning } from 'types/api';
 import { DataSource } from 'types/common/queryBuilder';
 import {
@@ -62,8 +53,6 @@ function LogsExplorer(): JSX.Element {
 	const [selectedView, setSelectedView] = useState<ExplorerViews>(
 		() => panelTypeToExplorerView[panelTypesFromUrl],
 	);
-	const { logs } = usePreferenceContext();
-	const { preferences } = logs;
 
 	const [showFilters, setShowFilters] = useState<boolean>(() => {
 		const localStorageValue = getLocalStorageKey(
@@ -182,116 +171,6 @@ function LogsExplorer(): JSX.Element {
 		setShowFilters((prev) => !prev);
 	};
 
-	const { redirectWithQuery: redirectWithOptionsData } =
-		useUrlQueryData<OptionsQuery>(URL_OPTIONS, defaultOptionsQuery);
-
-	// Get and parse stored columns from localStorage
-	const logListOptionsFromLocalStorage = useMemo(() => {
-		const data = getLocalStorageKey(LOCALSTORAGE.LOGS_LIST_OPTIONS);
-
-		if (!data) {
-			return null;
-		}
-
-		try {
-			return JSON.parse(data);
-		} catch {
-			return null;
-		}
-	}, []);
-
-	// Check if the columns have the required columns (timestamp, body)
-	const hasRequiredColumns = useCallback(
-		(columns?: TelemetryFieldKey[] | null): boolean => {
-			if (!columns?.length) {
-				return false;
-			}
-
-			const hasTimestamp = columns.some((col) => col.name === 'timestamp');
-			const hasBody = columns.some((col) => col.name === 'body');
-
-			return hasTimestamp && hasBody;
-		},
-		[],
-	);
-
-	// Merge the columns with the required columns (timestamp, body) if missing
-	const mergeWithRequiredColumns = useCallback(
-		(columns: TelemetryFieldKey[]): TelemetryFieldKey[] => [
-			// Add required columns (timestamp, body) if missing
-			...(!hasRequiredColumns(columns) ? defaultLogsSelectedColumns : []),
-			...columns,
-		],
-		[hasRequiredColumns],
-	);
-
-	// Migrate the options query to the new format
-	const migrateOptionsQuery = useCallback(
-		(query: OptionsQuery): OptionsQuery => {
-			// Skip if already migrated
-			if (query.version) {
-				return query;
-			}
-
-			if (logListOptionsFromLocalStorage?.version) {
-				return logListOptionsFromLocalStorage;
-			}
-
-			// Case 1: we have localStorage columns
-			if (logListOptionsFromLocalStorage?.selectColumns?.length > 0) {
-				return {
-					...query,
-					version: 1,
-					selectColumns: mergeWithRequiredColumns(
-						logListOptionsFromLocalStorage.selectColumns,
-					),
-				};
-			}
-
-			// Case 2: No query columns in localStorage in but query has columns
-			if (query.selectColumns.length > 0) {
-				return {
-					...query,
-					version: 1,
-					selectColumns: mergeWithRequiredColumns(query.selectColumns),
-				};
-			}
-
-			// Case 3: No columns anywhere, use defaults
-			return {
-				...query,
-				version: 1,
-				selectColumns: defaultLogsSelectedColumns,
-			};
-		},
-		[mergeWithRequiredColumns, logListOptionsFromLocalStorage],
-	);
-
-	useEffect(() => {
-		if (!preferences) {
-			return;
-		}
-		const migratedQuery = migrateOptionsQuery({
-			selectColumns: preferences.columns || defaultLogsSelectedColumns,
-			maxLines: preferences.formatting?.maxLines || defaultOptionsQuery.maxLines,
-			format: preferences.formatting?.format || defaultOptionsQuery.format,
-			fontSize: preferences.formatting?.fontSize || defaultOptionsQuery.fontSize,
-			version: preferences.formatting?.version,
-		});
-		// Only redirect if the query was actually modified
-		if (
-			!isEqual(migratedQuery, {
-				selectColumns: preferences?.columns,
-				maxLines: preferences?.formatting?.maxLines,
-				format: preferences?.formatting?.format,
-				fontSize: preferences?.formatting?.fontSize,
-				version: preferences?.formatting?.version,
-			})
-		) {
-			redirectWithOptionsData(migratedQuery);
-		}
-	}, [migrateOptionsQuery, preferences, redirectWithOptionsData]);
-
 	const toolbarViews = useMemo(
 		() => ({
 			list: {

frontend/src/providers/preferences/__tests__/PreferencesProvider.integration.test.tsx

@@ -108,7 +108,9 @@ describe('PreferencesProvider integration', () => {
 				},
 			);
 
-			expect(Number(screen.getByTestId('logs-columns-len').textContent)).toBe(1);
+			// Loader's ensureLogsRequiredColumns prepends timestamp + body, so the
+			// 1 column in localStorage becomes 3 in preferences.
+			expect(Number(screen.getByTestId('logs-columns-len').textContent)).toBe(3);
 		});
 
 		it('direct mode updateColumns persists to localStorage', async () => {
@@ -126,8 +128,11 @@ describe('PreferencesProvider integration', () => {
 			const stored = getLocalStorageJSON<LogsLocalOptions>(
 				LOCALSTORAGE.LOGS_LIST_OPTIONS,
 			);
+			// Writer's ensureLogsRequiredColumns prepends `body` when only
+			// `timestamp` was passed in (defaults.slice(0,1) is just timestamp).
 			expect(stored?.selectColumns).toStrictEqual([
-				defaultLogsSelectedColumns[0] as TelemetryFieldKey,
+				defaultLogsSelectedColumns[1] as TelemetryFieldKey, // body
+				defaultLogsSelectedColumns[0] as TelemetryFieldKey, // timestamp
 			]);
 		});
 
@@ -183,7 +188,9 @@ describe('PreferencesProvider integration', () => {
 				value: originalLocation,
 			});
 
-			expect(Number(screen.getByTestId('logs-columns-len').textContent)).toBe(1);
+			// Loader's ensureLogsRequiredColumns prepends timestamp + body, so the
+			// URL's 1 column becomes 3 in preferences.
+			expect(Number(screen.getByTestId('logs-columns-len').textContent)).toBe(3);
 		});
 
 		it('updateFormatting persists to localStorage in direct mode', async () => {

frontend/src/providers/preferences/__tests__/logsUpdaterConfig.test.ts

@@ -1,7 +1,10 @@
 import { TelemetryFieldKey } from 'api/v5/v5';
 import { LOCALSTORAGE } from 'constants/localStorage';
 import { LogViewMode } from 'container/LogsTable';
-import { defaultOptionsQuery } from 'container/OptionsMenu/constants';
+import {
+	defaultLogsSelectedColumns,
+	defaultOptionsQuery,
+} from 'container/OptionsMenu/constants';
 import { FontSize } from 'container/OptionsMenu/types';
 import {
 	FormattingOptions,
@@ -85,18 +88,21 @@ describe('logsUpdaterConfig', () => {
 
 		logsUpdater.updateColumns(newColumns, PreferenceMode.DIRECT);
 
+		// Writer guards body+timestamp via ensureLogsRequiredColumns invariant
+		const guardedColumns = [...defaultLogsSelectedColumns, ...newColumns];
+
 		// Should update URL
 		expect(redirectWithOptionsData).toHaveBeenCalledWith({
 			...defaultOptionsQuery,
 			...mockPreferences.formatting,
-			selectColumns: newColumns,
+			selectColumns: guardedColumns,
 		});
 
-		// Should update localStorage
+		// Should update localStorage with the guarded shape
 		const storedData = JSON.parse(
 			mockLocalStorage[LOCALSTORAGE.LOGS_LIST_OPTIONS],
 		);
-		expect(storedData.selectColumns).toStrictEqual(newColumns);
+		expect(storedData.selectColumns).toStrictEqual(guardedColumns);
 		expect(storedData.maxLines).toBe(1); // Should preserve other fields
 
 		// Should not update saved view preferences

frontend/src/providers/preferences/__tests__/usePreferenceLoader.test.tsx

@@ -1,4 +1,5 @@
 import { renderHook, waitFor } from '@testing-library/react';
+import { defaultLogsSelectedColumns } from 'container/OptionsMenu/constants';
 import { DataSource } from 'types/common/queryBuilder';
 
 import logsLoaderConfig from '../configs/logsLoaderConfig';
@@ -60,9 +61,9 @@ describe('usePreferenceLoader', () => {
 			expect(result.current.loading).toBe(false);
 		});
 
-		// Should have loaded from local storage (highest priority)
+		// Loader wraps with ensureLogsRequiredColumns — body+timestamp always prepended
 		expect(result.current.preferences).toStrictEqual({
-			columns: [{ name: 'local-column' }],
+			columns: [...defaultLogsSelectedColumns, { name: 'local-column' }],
 			formatting: { maxLines: 5, format: 'table', fontSize: 'medium', version: 1 },
 		});
 		expect(result.current.error).toBeNull();

frontend/src/providers/preferences/configs/logsUpdaterConfig.ts

@@ -3,7 +3,10 @@ import getLocalStorageKey from 'api/browser/localstorage/get';
 import setLocalStorageKey from 'api/browser/localstorage/set';
 import { TelemetryFieldKey } from 'api/v5/v5';
 import { LOCALSTORAGE } from 'constants/localStorage';
-import { defaultOptionsQuery } from 'container/OptionsMenu/constants';
+import {
+	defaultOptionsQuery,
+	ensureLogsRequiredColumns,
+} from 'container/OptionsMenu/constants';
 import { FontSize, OptionsQuery } from 'container/OptionsMenu/types';
 
 import { FormattingOptions, PreferenceMode, Preferences } from '../types';
@@ -18,11 +21,12 @@ const getLogsUpdaterConfig = (
 	updateFormatting: (newFormatting: FormattingOptions, mode: string) => void;
 } => ({
 	updateColumns: (newColumns: TelemetryFieldKey[], mode: string): void => {
+		const guardedColumns = ensureLogsRequiredColumns(newColumns);
 		if (mode === PreferenceMode.SAVED_VIEW) {
 			setSavedViewPreferences((prev) => {
 				if (!prev) {
 					return {
-						columns: newColumns,
+						columns: guardedColumns,
 						formatting: {
 							maxLines: 1,
 							format: 'table',
@@ -34,7 +38,7 @@ const getLogsUpdaterConfig = (
 
 				return {
 					...prev,
-					columns: newColumns,
+					columns: guardedColumns,
 				};
 			});
 		}
@@ -44,14 +48,14 @@ const getLogsUpdaterConfig = (
 			redirectWithOptionsData({
 				...defaultOptionsQuery,
 				...preferences?.formatting,
-				selectColumns: newColumns,
+				selectColumns: guardedColumns,
 			});
 
 			// Also update local storage
 			const local = JSON.parse(
 				getLocalStorageKey(LOCALSTORAGE.LOGS_LIST_OPTIONS) || '{}',
 			);
-			local.selectColumns = newColumns;
+			local.selectColumns = guardedColumns;
 			setLocalStorageKey(LOCALSTORAGE.LOGS_LIST_OPTIONS, JSON.stringify(local));
 		}
 	},

frontend/src/providers/preferences/loader/usePreferenceLoader.ts

@@ -1,5 +1,6 @@
 import { useEffect, useState } from 'react';
 import { TelemetryFieldKey } from 'api/v5/v5';
+import { ensureLogsRequiredColumns } from 'container/OptionsMenu/constants';
 import { has } from 'lodash-es';
 import { DataSource } from 'types/common/queryBuilder';
 
@@ -51,7 +52,11 @@ function logsPreferencesLoader(): {
 	columns: TelemetryFieldKey[];
 	formatting: FormattingOptions;
 } {
-	return preferencesLoader(logsLoaderConfig);
+	const result = preferencesLoader<{
+		columns: TelemetryFieldKey[];
+		formatting: FormattingOptions;
+	}>(logsLoaderConfig);
+	return { ...result, columns: ensureLogsRequiredColumns(result.columns) };
 }
 
 function tracesPreferencesLoader(): {

frontend/src/providers/preferences/sync/usePreferenceSync.ts

@@ -1,7 +1,10 @@
 /* eslint-disable sonarjs/cognitive-complexity */
 import { useEffect, useState } from 'react';
 import { TelemetryFieldKey } from 'api/v5/v5';
-import { defaultLogsSelectedColumns } from 'container/OptionsMenu/constants';
+import {
+	defaultLogsSelectedColumns,
+	ensureLogsRequiredColumns,
+} from 'container/OptionsMenu/constants';
 import { defaultSelectedColumns as defaultTracesSelectedColumns } from 'container/TracesExplorer/ListView/configs';
 import { useGetAllViews } from 'hooks/saveViews/useGetAllViews';
 import { DataSource } from 'types/common/queryBuilder';
@@ -54,9 +57,10 @@ export function usePreferenceSync({
 		let columns: TelemetryFieldKey[] = [];
 		let formatting: FormattingOptions | undefined;
 		if (dataSource === DataSource.LOGS) {
-			columns =
+			columns = ensureLogsRequiredColumns(
 				updateExtraDataSelectColumns(parsedExtraData?.selectColumns) ||
-				defaultLogsSelectedColumns;
+					defaultLogsSelectedColumns,
+			);
 			formatting = {
 				maxLines: parsedExtraData?.maxLines ?? 1,
 				format: parsedExtraData?.format ?? 'table',

pkg/telemetrylogs/condition_builder.go

@@ -186,7 +186,7 @@ func (c *conditionBuilder) conditionFor(
 		column := columns[0]
 		if len(key.Evolutions) > 0 {
 			// we will use the corresponding column and its evolution entry for the query
-			newColumns, _, err := selectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
+			newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
 			if err != nil {
 				return "", err
 			}

pkg/telemetrylogs/field_mapper.go

@@ -3,11 +3,7 @@ package telemetrylogs
 import (
 	"context"
 	"fmt"
-	"slices"
-	"sort"
-	"strconv"
 	"strings"
-	"time"
 
 	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
 	"github.com/SigNoz/signoz-otel-collector/utils"
@@ -137,113 +133,6 @@ func (m *fieldMapper) getColumn(ctx context.Context, key *telemetrytypes.Telemet
 	return nil, qbtypes.ErrColumnNotFound
 }
 
-// selectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
-// Logic:
-//   - Finds the latest base evolution (<= tsStartTime) across ALL columns
-//   - Rejects all evolutions before this latest base evolution
-//   - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
-//   - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime
-//   - Results are sorted by ReleaseTime descending (newest first)
-func selectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) {
-
-	sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions))
-	copy(sortedEvolutions, evolutions)
-
-	// sort the evolutions by ReleaseTime ascending
-	sort.Slice(sortedEvolutions, func(i, j int) bool {
-		return sortedEvolutions[i].ReleaseTime.Before(sortedEvolutions[j].ReleaseTime)
-	})
-
-	tsStartTime := time.Unix(0, int64(tsStart))
-	tsEndTime := time.Unix(0, int64(tsEnd))
-
-	// Build evolution map: column name -> evolution
-	evolutionMap := make(map[string]*telemetrytypes.EvolutionEntry)
-	for _, evolution := range sortedEvolutions {
-		if _, exists := evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))]; exists {
-			// since if there is duplicate we would just use the oldest one.
-			continue
-		}
-		evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))] = evolution
-	}
-
-	// Find the latest base evolution (<= tsStartTime) across ALL columns
-	// Evolutions are sorted, so we can break early
-	var latestBaseEvolutionAcrossAll *telemetrytypes.EvolutionEntry
-	for _, evolution := range sortedEvolutions {
-		if evolution.ReleaseTime.After(tsStartTime) {
-			break
-		}
-		latestBaseEvolutionAcrossAll = evolution
-	}
-
-	// We shouldn't reach this, it basically means there is something wrong with the evolutions data
-	if latestBaseEvolutionAcrossAll == nil {
-		return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
-	}
-
-	columnLookUpMap := make(map[string]*schema.Column)
-	for _, column := range columns {
-		columnLookUpMap[column.Name] = column
-	}
-
-	// Collect column-evolution pairs
-	type colEvoPair struct {
-		column    *schema.Column
-		evolution *telemetrytypes.EvolutionEntry
-	}
-	pairs := []colEvoPair{}
-
-	for _, evolution := range evolutionMap {
-		// Reject evolutions before the latest base evolution
-		if evolution.ReleaseTime.Before(latestBaseEvolutionAcrossAll.ReleaseTime) {
-			continue
-		}
-		// skip evolutions after tsEndTime
-		if evolution.ReleaseTime.After(tsEndTime) || evolution.ReleaseTime.Equal(tsEndTime) {
-			continue
-		}
-
-		if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
-			return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
-		}
-
-		pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
-	}
-
-	// If no pairs found, fall back to latestBaseEvolutionAcrossAll for matching columns
-	if len(pairs) == 0 {
-		for _, column := range columns {
-			// Use latestBaseEvolutionAcrossAll if this column name matches its column name
-			if column.Name == latestBaseEvolutionAcrossAll.ColumnName {
-				pairs = append(pairs, colEvoPair{column, latestBaseEvolutionAcrossAll})
-			}
-		}
-	}
-
-	// Sort by ReleaseTime descending (newest first)
-	slices.SortFunc(pairs, func(a, b colEvoPair) int {
-		// Sort by ReleaseTime descending (newest first)
-		if a.evolution.ReleaseTime.After(b.evolution.ReleaseTime) {
-			return -1
-		}
-		if a.evolution.ReleaseTime.Before(b.evolution.ReleaseTime) {
-			return 1
-		}
-		return 0
-	})
-
-	// Extract results
-	newColumns := make([]*schema.Column, len(pairs))
-	evolutionsEntries := make([]*telemetrytypes.EvolutionEntry, len(pairs))
-	for i, pair := range pairs {
-		newColumns[i] = pair.column
-		evolutionsEntries[i] = pair.evolution
-	}
-
-	return newColumns, evolutionsEntries, nil
-}
-
 func (m *fieldMapper) FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
 	columns, err := m.getColumn(ctx, key)
 	if err != nil {
@@ -254,7 +143,7 @@ func (m *fieldMapper) FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *
 	var evolutionsEntries []*telemetrytypes.EvolutionEntry
 	if len(key.Evolutions) > 0 {
 		// we will use the corresponding column and its evolution entry for the query
-		newColumns, evolutionsEntries, err = selectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
+		newColumns, evolutionsEntries, err = qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
 		if err != nil {
 			return "", err
 		}

pkg/telemetrylogs/field_mapper_test.go

@@ -536,390 +536,6 @@ func TestFieldForWithEvolutions(t *testing.T) {
 	}
 }
 
-func TestSelectEvolutionsForColumns(t *testing.T) {
-	testCases := []struct {
-		name            string
-		columns         []*schema.Column
-		evolutions      []*telemetrytypes.EvolutionEntry
-		tsStart         uint64
-		tsEnd           uint64
-		expectedColumns []string // column names
-		expectedEvols   []string // evolution column names
-		expectedError   bool
-		errorStr        string
-	}{
-		{
-			name: "New evolutions at tsStartTime - should include latest evolution",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resource"},
-			expectedEvols:   []string{"resource"},
-		},
-		{
-			name: "New evolutions after tsStartTime but less than tsEndTime - should include both",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resource", "resources_string"}, // sorted by ReleaseTime desc
-			expectedEvols:   []string{"resource", "resources_string"},
-		},
-		{
-			name: "Columns without matching evolutions - should exclude them",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],          // no evolution for this
-				logsV2Columns["attributes_string"], // no evolution for this
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string"},
-			expectedEvols:   []string{"resources_string"},
-		},
-		{
-			name: "New evolutions at tsEndTime - should not include new evolution",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string"},
-			expectedEvols:   []string{"resources_string"},
-		},
-		{
-			name: "New evolutions after tsEndTime - should exclude new",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string"},
-			expectedEvols:   []string{"resources_string"},
-		},
-		{
-			name:    "Empty columns array",
-			columns: []*schema.Column{},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{},
-			expectedEvols:   []string{},
-			expectedError:   true,
-			errorStr:        "column resources_string not found",
-		},
-		{
-			name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
-			columns: []*schema.Column{
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resource"},
-			expectedEvols:   []string{"resource"}, // should use first one (older)
-		},
-		{
-			name: "Genuine Duplicate evolutions with new version- should consider both",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      2,
-					ReleaseTime:  time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string", "resource"},
-			expectedEvols:   []string{"resources_string", "resource"}, // should use first one (older)
-		},
-		{
-			name: "Evolution exactly at tsEndTime",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					ReleaseTime:  time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), // exactly at tsEnd
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string"}, // resource excluded because After(tsEnd) is true
-			expectedEvols:   []string{"resources_string"},
-		},
-		{
-			name: "Single evolution after tsStartTime - JSON body",
-			columns: []*schema.Column{
-				logsV2Columns[LogsV2BodyV2Column],
-				logsV2Columns[LogsV2BodyPromotedColumn],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   LogsV2BodyV2Column,
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextBody,
-					FieldName:    "__all__",
-					ReleaseTime:  time.Unix(0, 0),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   LogsV2BodyPromotedColumn,
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextBody,
-					FieldName:    "user.name",
-					ReleaseTime:  time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column}, // sorted by ReleaseTime desc (newest first)
-			expectedEvols:   []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column},
-		},
-		{
-			name: "No evolution after tsStartTime - JSON body",
-			columns: []*schema.Column{
-				logsV2Columns[LogsV2BodyV2Column],
-				logsV2Columns[LogsV2BodyPromotedColumn],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   LogsV2BodyV2Column,
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextBody,
-					FieldName:    "__all__",
-					ReleaseTime:  time.Unix(0, 0),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   LogsV2BodyPromotedColumn,
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextBody,
-					FieldName:    "user.name",
-					ReleaseTime:  time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{LogsV2BodyPromotedColumn},
-			expectedEvols:   []string{LogsV2BodyPromotedColumn},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			resultColumns, resultEvols, err := selectEvolutionsForColumns(tc.columns, tc.evolutions, tc.tsStart, tc.tsEnd)
-
-			if tc.expectedError {
-				assert.Contains(t, err.Error(), tc.errorStr)
-			} else {
-				require.NoError(t, err)
-				assert.Equal(t, len(tc.expectedColumns), len(resultColumns), "column count mismatch")
-				assert.Equal(t, len(tc.expectedEvols), len(resultEvols), "evolution count mismatch")
-
-				resultColumnNames := make([]string, len(resultColumns))
-				for i, col := range resultColumns {
-					resultColumnNames[i] = col.Name
-				}
-				resultEvolNames := make([]string, len(resultEvols))
-				for i, evol := range resultEvols {
-					resultEvolNames[i] = evol.ColumnName
-				}
-
-				for i := range tc.expectedColumns {
-					assert.Equal(t, resultColumnNames[i], tc.expectedColumns[i], "expected column missing: "+tc.expectedColumns[i])
-				}
-				for i := range tc.expectedEvols {
-					assert.Equal(t, resultEvolNames[i], tc.expectedEvols[i], "expected evolution missing: "+tc.expectedEvols[i])
-				}
-				// Verify sorting: should be descending by ReleaseTime
-				for i := 0; i < len(resultEvols)-1; i++ {
-					assert.True(t, !resultEvols[i].ReleaseTime.Before(resultEvols[i+1].ReleaseTime),
-						"evolutions should be sorted descending by ReleaseTime")
-				}
-			}
-		})
-	}
-}
-
 func TestFieldForWithMaterialized(t *testing.T) {
 	ctx := context.Background()
 

pkg/telemetrymetadata/metadata.go

@@ -344,6 +344,11 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
 			})
 		}
 	}
+
+	if err = t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
+		return nil, false, err
+	}
+
 	return keys, complete, nil
 }
 
@@ -689,7 +694,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
 		}
 	}
 
-	if _, err := t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
+	if err := t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
 		return nil, false, err
 	}
 
@@ -2370,8 +2375,8 @@ func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Conte
 	return entries, nil
 }
 
-// Get retrieves all evolutions for the given selectors from DB.
-func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) (map[string][]*telemetrytypes.EvolutionEntry, error) {
+// updateColumnEvolutionMetadataForKeys updates the evolution field for keys.
+func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) error {
 
 	var metadataKeySelectors []*telemetrytypes.EvolutionSelector
 	for _, keySelector := range keysToUpdate {
@@ -2385,7 +2390,7 @@ func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Co
 
 	evolutions, err := k.fetchEvolutionEntryFromClickHouse(ctx, metadataKeySelectors)
 	if err != nil {
-		return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to fetch evolution from clickhouse %s", err.Error())
+		return errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to fetch evolution from clickhouse %s", err.Error())
 	}
 
 	evolutionsByUniqueKey := make(map[string][]*telemetrytypes.EvolutionEntry)
@@ -2416,7 +2421,7 @@ func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Co
 			}
 		}
 	}
-	return evolutionsByUniqueKey, nil
+	return nil
 }
 
 // chunkSizeFirstSeenMetricMetadata limits the number of tuples per SQL query to avoid hitting the max_query_size limit.

pkg/telemetrymetadata/metadata_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
 	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/telemetryaudit"
 	"github.com/SigNoz/signoz/pkg/telemetrylogs"
@@ -17,7 +18,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	cmock "github.com/SigNoz/clickhouse-go-mock"
 	"github.com/stretchr/testify/assert"
-	"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
 	"github.com/stretchr/testify/require"
 )
 
@@ -89,6 +89,19 @@ func TestGetKeys(t *testing.T) {
 			{Name: "tag_data_type", Type: "String"},
 			{Name: "priority", Type: "UInt8"},
 		}, [][]any{{"http.method", "tag", "String", 1}, {"http.method", "tag", "String", 1}}))
+
+	mock.ExpectQuery(`FROM signoz_metadata\.distributed_column_evolution_metadata`).
+		WithArgs(nil, nil, nil, nil, nil, nil, nil, nil).
+		WillReturnRows(cmock.NewRows([]cmock.ColumnType{
+			{Name: "signal", Type: "String"},
+			{Name: "column_name", Type: "String"},
+			{Name: "column_type", Type: "String"},
+			{Name: "field_context", Type: "String"},
+			{Name: "field_name", Type: "String"},
+			{Name: "version", Type: "UInt32"},
+			{Name: "release_time", Type: "Float64"},
+		}, [][]any{}))
+
 	keys, _, err := metadata.GetKeys(context.Background(), &telemetrytypes.FieldKeySelector{
 		Signal:        telemetrytypes.SignalTraces,
 		FieldContext:  telemetrytypes.FieldContextSpan,
@@ -247,6 +260,27 @@ func TestApplyBackwardCompatibleKeys(t *testing.T) {
 					}, rows))
 			}
 
+			// getTracesKeys / getLogsKeys both fetch evolution metadata; return an empty
+			// result so the existing test data flows through unchanged. Each input key
+			// becomes one selector contributing four bound args.
+			if hasTraces || hasLogs {
+				evoArgs := make([]any, 0, len(tt.inputKeys)*4)
+				for range tt.inputKeys {
+					evoArgs = append(evoArgs, nil, nil, nil, nil)
+				}
+				mock.ExpectQuery(`FROM signoz_metadata\.distributed_column_evolution_metadata`).
+					WithArgs(evoArgs...).
+					WillReturnRows(cmock.NewRows([]cmock.ColumnType{
+						{Name: "signal", Type: "String"},
+						{Name: "column_name", Type: "String"},
+						{Name: "column_type", Type: "String"},
+						{Name: "field_context", Type: "String"},
+						{Name: "field_name", Type: "String"},
+						{Name: "version", Type: "UInt32"},
+						{Name: "release_time", Type: "Float64"},
+					}, [][]any{}))
+			}
+
 			selectors := []*telemetrytypes.FieldKeySelector{}
 			for _, key := range tt.inputKeys {
 				selectors = append(selectors, &telemetrytypes.FieldKeySelector{

pkg/telemetrytraces/condition_builder.go

@@ -161,7 +161,33 @@ func (c *conditionBuilder) conditionFor(
 	case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
 
 		var value any
-		switch columns[0].Type.GetType() {
+		column := columns[0]
+		if len(key.Evolutions) > 0 {
+			// we will use the corresponding column and its evolution entry for the query
+			newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
+			if err != nil {
+				return "", err
+			}
+
+			if len(newColumns) == 0 {
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no valid evolution found for field %s in the given time range", key.Name)
+			}
+
+			// Multiple columns means fieldExpression is a multiIf returning NULL when none match,
+			// so a simple null check is sufficient.
+			if len(newColumns) > 1 {
+				if operator == qbtypes.FilterOperatorExists {
+					return sb.IsNotNull(fieldExpression), nil
+				} else {
+					return sb.IsNull(fieldExpression), nil
+				}
+			}
+
+			// otherwise we have to find the correct exist operator based on the column type
+			column = newColumns[0]
+		}
+
+		switch column.Type.GetType() {
 		case schema.ColumnTypeEnumJSON:
 			if operator == qbtypes.FilterOperatorExists {
 				return sb.IsNotNull(fieldExpression), nil
@@ -178,7 +204,7 @@ func (c *conditionBuilder) conditionFor(
 				return sb.E(fieldExpression, value), nil
 			}
 		case schema.ColumnTypeEnumLowCardinality:
-			switch elementType := columns[0].Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
+			switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
 			case schema.ColumnTypeEnumString:
 				value = ""
 				if operator == qbtypes.FilterOperatorExists {
@@ -202,14 +228,14 @@ func (c *conditionBuilder) conditionFor(
 				return sb.E(fieldExpression, value), nil
 			}
 		case schema.ColumnTypeEnumMap:
-			keyType := columns[0].Type.(schema.MapColumnType).KeyType
+			keyType := column.Type.(schema.MapColumnType).KeyType
 			if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
-				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, columns[0].Type)
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
 			}
 
-			switch valueType := columns[0].Type.(schema.MapColumnType).ValueType; valueType.GetType() {
+			switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
 			case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
-				leftOperand := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
+				leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
 				if key.Materialized {
 					leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
 				}
@@ -222,7 +248,7 @@ func (c *conditionBuilder) conditionFor(
 				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
 			}
 		default:
-			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", columns[0].Type)
+			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type)
 		}
 	}
 	return "", nil

pkg/telemetrytraces/condition_builder_test.go

@@ -3,6 +3,7 @@ package telemetrytraces
 import (
 	"context"
 	"testing"
+	"time"
 
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -14,6 +15,7 @@ import (
 func TestConditionFor(t *testing.T) {
 	ctx := context.Background()
 
+	mockEvolution := mockEvolutionData(time.Date(2025, 10, 26, 0, 10, 0, 0, time.UTC))
 	testCases := []struct {
 		name          string
 		key           telemetrytypes.TelemetryFieldKey
@@ -213,6 +215,7 @@ func TestConditionFor(t *testing.T) {
 				Name:          "service.name",
 				FieldContext:  telemetrytypes.FieldContextResource,
 				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    mockEvolution,
 			},
 			operator:      qbtypes.FilterOperatorExists,
 			value:         nil,
@@ -225,6 +228,7 @@ func TestConditionFor(t *testing.T) {
 				Name:          "service.name",
 				FieldContext:  telemetrytypes.FieldContextResource,
 				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    mockEvolution,
 			},
 			operator:      qbtypes.FilterOperatorNotExists,
 			value:         nil,
@@ -302,3 +306,85 @@ func TestConditionFor(t *testing.T) {
 		})
 	}
 }
+
+func TestConditionForResourceWithEvolution(t *testing.T) {
+	ctx := context.Background()
+	releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
+	evolutions := mockEvolutionData(releaseTime)
+
+	testCases := []struct {
+		name        string
+		key         telemetrytypes.TelemetryFieldKey
+		operator    qbtypes.FilterOperator
+		tsStart     uint64
+		tsEnd       uint64
+		expectedSQL string
+	}{
+		{
+			name: "Exists - window after release - JSON only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    evolutions,
+			},
+			operator:    qbtypes.FilterOperatorExists,
+			tsStart:     uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:       uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedSQL: "WHERE resource.`service.name`::String IS NOT NULL",
+		},
+		{
+			name: "NotExists - window after release - JSON only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    evolutions,
+			},
+			operator:    qbtypes.FilterOperatorNotExists,
+			tsStart:     uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:       uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedSQL: "WHERE resource.`service.name`::String IS NULL",
+		},
+		{
+			name: "Exists - window before release - map only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    evolutions,
+			},
+			operator:    qbtypes.FilterOperatorExists,
+			tsStart:     uint64(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:       uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedSQL: "WHERE mapContains(resources_string, 'service.name') = ?",
+		},
+		{
+			name: "Exists - window straddles release - multiIf null check",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    evolutions,
+			},
+			operator:    qbtypes.FilterOperatorExists,
+			tsStart:     uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:       uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL",
+		},
+	}
+
+	fm := NewFieldMapper()
+	conditionBuilder := NewConditionBuilder(fm)
+
+	for _, tc := range testCases {
+		sb := sqlbuilder.NewSelectBuilder()
+		t.Run(tc.name, func(t *testing.T) {
+			cond, err := conditionBuilder.ConditionFor(ctx, tc.tsStart, tc.tsEnd, &tc.key, tc.operator, nil, sb)
+			require.NoError(t, err)
+			sb.Where(cond)
+			sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+			assert.Contains(t, sql, tc.expectedSQL)
+		})
+	}
+}

pkg/telemetrytraces/const.go

@@ -1,6 +1,8 @@
 package telemetrytraces
 
-import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+import (
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
 
 var (
 	IntrinsicFields = map[string]telemetrytypes.TelemetryFieldKey{

pkg/telemetrytraces/field_mapper.go

@@ -174,7 +174,7 @@ func (m *defaultFieldMapper) getColumn(
 ) ([]*schema.Column, error) {
 	switch key.FieldContext {
 	case telemetrytypes.FieldContextResource:
-		return []*schema.Column{indexV3Columns["resource"]}, nil
+		return []*schema.Column{indexV3Columns["resources_string"], indexV3Columns["resource"]}, nil
 	case telemetrytypes.FieldContextScope:
 		return []*schema.Column{}, qbtypes.ErrColumnNotFound
 	case telemetrytypes.FieldContextAttribute:
@@ -254,63 +254,92 @@ func (m *defaultFieldMapper) FieldFor(
 	if err != nil {
 		return "", err
 	}
-	if len(columns) != 1 {
-		return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "expected exactly 1 column, got %d", len(columns))
+
+	var newColumns []*schema.Column
+	var evolutionsEntries []*telemetrytypes.EvolutionEntry
+	if len(key.Evolutions) > 0 {
+		// we will use the corresponding column and its evolution entry for the query
+		newColumns, evolutionsEntries, err = qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		newColumns = columns
 	}
-	column := columns[0]
 
-	switch column.Type.GetType() {
-	case schema.ColumnTypeEnumJSON:
-		// json is only supported for resource context as of now
-		if key.FieldContext != telemetrytypes.FieldContextResource {
-			return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
-		}
-		oldColumn := indexV3Columns["resources_string"]
-		oldKeyName := fmt.Sprintf("%s['%s']", oldColumn.Name, key.Name)
-		// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
-		// once clickHouse dependency is updated, we need to check if we can remove it.
-		if key.Materialized {
-			oldKeyName = telemetrytypes.FieldKeyToMaterializedColumnName(key)
-			oldKeyNameExists := telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
-			return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, %s==true, %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldKeyNameExists, oldKeyName), nil
-		} else {
-			return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
-		}
-	case schema.ColumnTypeEnumString,
-		schema.ColumnTypeEnumUInt64,
-		schema.ColumnTypeEnumUInt32,
-		schema.ColumnTypeEnumInt8,
-		schema.ColumnTypeEnumInt16,
-		schema.ColumnTypeEnumBool,
-		schema.ColumnTypeEnumDateTime64,
-		schema.ColumnTypeEnumFixedString:
-		return column.Name, nil
-	case schema.ColumnTypeEnumLowCardinality:
-		switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
-		case schema.ColumnTypeEnumString:
-			return column.Name, nil
-		default:
-			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for low cardinality column type %s", elementType, column.Type)
-		}
-	case schema.ColumnTypeEnumMap:
-		keyType := column.Type.(schema.MapColumnType).KeyType
-		if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
-			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
+	exprs := []string{}
+	existExpr := []string{}
+	for i, column := range newColumns {
+		// Use evolution column name if available, otherwise use the column name
+		columnName := column.Name
+		if evolutionsEntries != nil && evolutionsEntries[i] != nil {
+			columnName = evolutionsEntries[i].ColumnName
 		}
 
-		switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
-		case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
-			// a key could have been materialized, if so return the materialized column name
-			if key.Materialized {
-				return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
+		switch column.Type.GetType() {
+		case schema.ColumnTypeEnumJSON:
+			// json is only supported for resource context as of now
+			if key.FieldContext != telemetrytypes.FieldContextResource {
+				return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
+			}
+			// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
+			// once clickHouse dependency is updated, we need to check if we can remove it.
+			exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
+			existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
+		case schema.ColumnTypeEnumString,
+			schema.ColumnTypeEnumUInt64,
+			schema.ColumnTypeEnumUInt32,
+			schema.ColumnTypeEnumInt8,
+			schema.ColumnTypeEnumInt16,
+			schema.ColumnTypeEnumBool,
+			schema.ColumnTypeEnumDateTime64,
+			schema.ColumnTypeEnumFixedString:
+			exprs = append(exprs, column.Name)
+		case schema.ColumnTypeEnumLowCardinality:
+			switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
+			case schema.ColumnTypeEnumString:
+				exprs = append(exprs, column.Name)
+			default:
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for low cardinality column type %s", elementType, column.Type)
+			}
+		case schema.ColumnTypeEnumMap:
+			keyType := column.Type.(schema.MapColumnType).KeyType
+			if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
+			}
+
+			switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
+			case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
+				// a key could have been materialized, if so return the materialized column name
+				if key.Materialized {
+					exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
+					existExpr = append(existExpr, fmt.Sprintf("%s==true", telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)))
+				} else {
+					exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
+					existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
+				}
+			default:
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
 			}
-			return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
-		default:
-			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
 		}
 	}
+
+	if len(exprs) == 1 {
+		return exprs[0], nil
+	} else if len(exprs) > 1 {
+		// Ensure existExpr has the same length as exprs
+		if len(existExpr) != len(exprs) {
+			return "", errors.New(errors.TypeInternal, errors.CodeInternal, "length of exist exprs doesn't match to that of exprs")
+		}
+		finalExprs := []string{}
+		for i, expr := range exprs {
+			finalExprs = append(finalExprs, fmt.Sprintf("%s, %s", existExpr[i], expr))
+		}
+		return "multiIf(" + strings.Join(finalExprs, ", ") + ", NULL)", nil
+	}
+
 	// should not reach here
-	return column.Name, nil
+	return columns[0].Name, nil
 }
 
 // ColumnExpressionFor returns the column expression for the given field

pkg/telemetrytraces/field_mapper_test.go

@@ -3,6 +3,7 @@ package telemetrytraces
 import (
 	"context"
 	"testing"
+	"time"
 
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -13,6 +14,7 @@ import (
 func TestGetFieldKeyName(t *testing.T) {
 	ctx := context.Background()
 
+	mockEvolution := mockEvolutionData(time.Date(2024, 6, 2, 0, 0, 0, 0, time.UTC))
 	testCases := []struct {
 		name           string
 		key            telemetrytypes.TelemetryFieldKey
@@ -63,6 +65,7 @@ func TestGetFieldKeyName(t *testing.T) {
 			key: telemetrytypes.TelemetryFieldKey{
 				Name:         "service.name",
 				FieldContext: telemetrytypes.FieldContextResource,
+				Evolutions:   mockEvolution,
 			},
 			expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
 			expectedError:  nil,
@@ -74,6 +77,7 @@ func TestGetFieldKeyName(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextResource,
 				FieldDataType: telemetrytypes.FieldDataTypeString,
 				Materialized:  true,
+				Evolutions:    mockEvolution,
 			},
 			expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`==true, `resource_string_deployment$$environment`, NULL)",
 			expectedError:  nil,
@@ -92,7 +96,7 @@ func TestGetFieldKeyName(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			fm := NewFieldMapper()
-			result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
+			result, err := fm.FieldFor(ctx, uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), uint64(time.Date(2024, 6, 5, 0, 0, 0, 0, time.UTC).UnixNano()), &tc.key)
 
 			if tc.expectedError != nil {
 				assert.Equal(t, tc.expectedError, err)
@@ -103,3 +107,86 @@ func TestGetFieldKeyName(t *testing.T) {
 		})
 	}
 }
+
+func TestFieldForResourceWithEvolution(t *testing.T) {
+	ctx := context.Background()
+	releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
+	evolutions := mockEvolutionData(releaseTime)
+
+	testCases := []struct {
+		name           string
+		key            telemetrytypes.TelemetryFieldKey
+		tsStart        uint64
+		tsEnd          uint64
+		expectedResult string
+	}{
+		{
+			name: "Window straddles release - both columns",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+				Evolutions:   evolutions,
+			},
+			tsStart:        uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
+		},
+		{
+			name: "Window fully after release - JSON column only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+				Evolutions:   evolutions,
+			},
+			tsStart:        uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "resource.`service.name`::String",
+		},
+		{
+			name: "Window fully before release - map column only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+				Evolutions:   evolutions,
+			},
+			tsStart:        uint64(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "resources_string['service.name']",
+		},
+		{
+			name: "Window fully after release - materialized resource",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "deployment.environment",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Materialized:  true,
+				Evolutions:    evolutions,
+			},
+			tsStart:        uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "resource.`deployment.environment`::String",
+		},
+		{
+			name: "Window straddles release - materialized resource",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "deployment.environment",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Materialized:  true,
+				Evolutions:    evolutions,
+			},
+			tsStart:        uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`==true, `resource_string_deployment$$environment`, NULL)",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			fm := NewFieldMapper()
+			result, err := fm.FieldFor(ctx, tc.tsStart, tc.tsEnd, &tc.key)
+			require.NoError(t, err)
+			assert.Equal(t, tc.expectedResult, result)
+		})
+	}
+}

pkg/telemetrytraces/statement_builder.go

@@ -82,13 +82,6 @@ func (b *traceQueryStatementBuilder) Build(
 	start = querybuilder.ToNanoSecs(start)
 	end = querybuilder.ToNanoSecs(end)
 
-	keySelectors := getKeySelectors(query)
-
-	keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
-	if err != nil {
-		return nil, err
-	}
-
 	/*
 		Adding a tech debt note here:
 		This piece of code is a hot fix and should be removed once we close issue: engineering-pod/issues/3622
@@ -124,6 +117,14 @@ func (b *traceQueryStatementBuilder) Build(
 		-------------------------------- End of tech debt ----------------------------
 	*/
 
+	// since we are modifying the selectFields, they might include keys which need evolutions so we should get keys after that.
+	keySelectors := getKeySelectors(query)
+
+	keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
+	if err != nil {
+		return nil, err
+	}
+
 	query = b.adjustKeys(ctx, keys, query, requestType)
 
 	// Create SQL builder

pkg/telemetrytraces/stmt_builder_test.go

@@ -16,6 +16,9 @@ import (
 )
 
 func TestStatementBuilder(t *testing.T) {
+	// releaseTime is chosen so it lands inside the standard [1747947419000, 1747983448000]ms
+	// test window, keeping the multiIf SQL form for resource fields.
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		requestType qbtypes.RequestType
@@ -355,7 +358,7 @@ func TestStatementBuilder(t *testing.T) {
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 
@@ -394,6 +397,7 @@ func TestStatementBuilder(t *testing.T) {
 }
 
 func TestStatementBuilderListQuery(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		requestType qbtypes.RequestType
@@ -650,7 +654,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 
@@ -683,6 +687,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 }
 
 func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		requestType qbtypes.RequestType
@@ -703,6 +708,15 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
 						FieldDataType: telemetrytypes.FieldDataTypeString,
 					},
 				},
+				"service.name": {
+					{
+						Name:          "service.name",
+						Signal:        telemetrytypes.SignalTraces,
+						FieldContext:  telemetrytypes.FieldContextResource,
+						FieldDataType: telemetrytypes.FieldDataTypeString,
+						Evolutions:    mockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
+					},
+				},
 			},
 			query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
 				Signal:       telemetrytypes.SignalTraces,
@@ -728,6 +742,15 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
 						FieldDataType: telemetrytypes.FieldDataTypeString,
 					},
 				},
+				"service.name": {
+					{
+						Name:          "service.name",
+						Signal:        telemetrytypes.SignalTraces,
+						FieldContext:  telemetrytypes.FieldContextResource,
+						FieldDataType: telemetrytypes.FieldDataTypeString,
+						Evolutions:    mockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
+					},
+				},
 			},
 			query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
 				Signal:       telemetrytypes.SignalTraces,
@@ -758,7 +781,7 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
 			mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 			mockMetadataStore.KeysMap = c.keysMap
 			if mockMetadataStore.KeysMap == nil {
-				mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+				mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 			}
 			fl := flaggertest.New(t)
 			aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
@@ -788,7 +811,90 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
 	}
 }
 
+func TestStatementBuilderGroupByResourceEvolution(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
+
+	cases := []struct {
+		name     string
+		startMs  uint64
+		endMs    uint64
+		expected qbtypes.Statement
+	}{
+		{
+			name:    "window straddles release - both JSON and map branches",
+			startMs: 1747947419000, // 2025-05-22 21:56:59 UTC, ~3m before release
+			endMs:   1747983448000, // 2025-05-23 07:57:28 UTC, ~10h after release
+			expected: qbtypes.Statement{
+				Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
+				Args:  []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
+			},
+		},
+		{
+			name:    "window after release - JSON column only",
+			startMs: 1747960000000, // 2025-05-23 00:26:40 UTC, ~2.5h after release
+			endMs:   1747983448000, // 2025-05-23 07:57:28 UTC
+			expected: qbtypes.Statement{
+				Query: "WITH __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
+				Args:  []any{"1747960000000000000", "1747983448000000000", uint64(1747958200), uint64(1747983448), 10, "1747960000000000000", "1747983448000000000", uint64(1747958200), uint64(1747983448)},
+			},
+		},
+		{
+			name:    "window before release - map column only",
+			startMs: 1747900000000, // 2025-05-22 08:26:40 UTC, ~13.5h before release
+			endMs:   1747947000000, // 2025-05-22 21:50:00 UTC, ~10m before release
+			expected: qbtypes.Statement{
+				Query: "WITH __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
+				Args:  []any{true, "1747900000000000000", "1747947000000000000", uint64(1747898200), uint64(1747947000), 10, true, "1747900000000000000", "1747947000000000000", uint64(1747898200), uint64(1747947000)},
+			},
+		},
+	}
+
+	fm := NewFieldMapper()
+	cb := NewConditionBuilder(fm)
+	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
+	fl := flaggertest.New(t)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
+
+	statementBuilder := NewTraceQueryStatementBuilder(
+		instrumentationtest.New().ToProviderSettings(),
+		mockMetadataStore,
+		fm,
+		cb,
+		aggExprRewriter,
+		nil,
+		fl,
+	)
+
+	query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Signal:       telemetrytypes.SignalTraces,
+		StepInterval: qbtypes.Step{Duration: 30 * time.Second},
+		Aggregations: []qbtypes.TraceAggregation{
+			{Expression: "count()"},
+		},
+		Filter: &qbtypes.Filter{},
+		Limit:  10,
+		GroupBy: []qbtypes.GroupByKey{
+			{
+				TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+					Name: "service.name",
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			q, err := statementBuilder.Build(context.Background(), c.startMs, c.endMs, qbtypes.RequestTypeTimeSeries, query, nil)
+			require.NoError(t, err)
+			require.Equal(t, c.expected.Query, q.Query)
+			require.Equal(t, c.expected.Args, q.Args)
+		})
+	}
+}
+
 func TestStatementBuilderTraceQuery(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		requestType qbtypes.RequestType
@@ -911,7 +1017,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 
@@ -944,6 +1050,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
 }
 
 func TestAdjustKey(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		inputKey    telemetrytypes.TelemetryFieldKey
@@ -957,7 +1064,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
+			keysMap:     buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: IntrinsicFields["trace_id"],
 		},
 		{
@@ -967,7 +1074,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextBody, // incorrect context
 				FieldDataType: telemetrytypes.FieldDataTypeInt64,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "duration_nano",
 				FieldContext:  telemetrytypes.FieldContextSpan,    // should be corrected
@@ -981,7 +1088,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextSpan, // correct context
 				FieldDataType: telemetrytypes.FieldDataTypeInt64,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "duration_nano",
 				FieldContext:  telemetrytypes.FieldContextSpan,   // should be corrected
@@ -995,8 +1102,8 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
+			keysMap:     buildCompleteFieldKeyMap(releaseTime),
+			expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
 		},
 		{
 			name: "single matching key with context specified - override",
@@ -1005,8 +1112,8 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextAttribute,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["cart.items_count"][0],
+			keysMap:     buildCompleteFieldKeyMap(releaseTime),
+			expectedKey: *buildCompleteFieldKeyMap(releaseTime)["cart.items_count"][0],
 		},
 		{
 			name: "multiple matching keys - all materialized",
@@ -1043,7 +1150,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "mixed.materialization.key",
 				FieldDataType: telemetrytypes.FieldDataTypeString,
@@ -1057,7 +1164,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextAttribute,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "mixed.materialization.key",
 				FieldContext:  telemetrytypes.FieldContextAttribute,
@@ -1072,7 +1179,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:         "unknown.field",
 				Materialized: false,
@@ -1085,7 +1192,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextAttribute,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "service.name",
 				FieldContext:  telemetrytypes.FieldContextAttribute,
@@ -1100,7 +1207,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "cart.items_count",
 				FieldContext:  telemetrytypes.FieldContextAttribute,
@@ -1115,7 +1222,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "user.id",
 				FieldContext:  telemetrytypes.FieldContextAttribute,
@@ -1158,6 +1265,7 @@ func TestAdjustKey(t *testing.T) {
 }
 
 func TestAdjustKeys(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name                      string
 		query                     qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]
@@ -1183,7 +1291,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedSelectFields: []telemetrytypes.TelemetryFieldKey{
 				{
 					Name:          "service.name",
@@ -1220,7 +1328,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedGroupBy: []qbtypes.GroupByKey{
 				{
 					TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
@@ -1267,7 +1375,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedOrder: []qbtypes.OrderBy{
 				{
 					Key: qbtypes.OrderByKey{
@@ -1326,7 +1434,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedSelectFields: []telemetrytypes.TelemetryFieldKey{
 				{
 					Name:          "trace_id",
@@ -1381,7 +1489,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			// After alias adjustment, name becomes "span.duration" with FieldContextUnspecified
 			// "span.duration" is not in keysMap, so context stays unspecified
 			expectedOrder: []qbtypes.OrderBy{

pkg/telemetrytraces/test_data.go

@@ -1,10 +1,12 @@
 package telemetrytraces
 
 import (
+	"time"
+
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 )
 
-func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
+func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
 	keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
 		"service.name": {
 			{
@@ -115,7 +117,33 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
 	for _, keys := range keysMap {
 		for _, key := range keys {
 			key.Signal = telemetrytypes.SignalTraces
+			if key.FieldContext == telemetrytypes.FieldContextResource {
+				key.Evolutions = mockEvolutionData(releaseTime)
+			}
 		}
 	}
 	return keysMap
 }
+
+// mockEvolutionData returns the canonical resource-column evolution timeline used in tests:
+// the legacy resources_string map at epoch 0 and the JSON resource column released at releaseTime.
+func mockEvolutionData(releaseTime time.Time) []*telemetrytypes.EvolutionEntry {
+	return []*telemetrytypes.EvolutionEntry{
+		{
+			Signal:       telemetrytypes.SignalTraces,
+			ColumnName:   "resources_string",
+			FieldContext: telemetrytypes.FieldContextResource,
+			ColumnType:   "Map(LowCardinality(String), String)",
+			FieldName:    "__all__",
+			ReleaseTime:  time.Unix(0, 0),
+		},
+		{
+			Signal:       telemetrytypes.SignalTraces,
+			ColumnName:   "resource",
+			ColumnType:   "JSON()",
+			FieldContext: telemetrytypes.FieldContextResource,
+			FieldName:    "__all__",
+			ReleaseTime:  releaseTime,
+		},
+	}
+}

pkg/telemetrytraces/trace_operator_cte_builder.go

@@ -417,21 +417,27 @@ func (b *traceOperatorCTEBuilder) buildNotCTE(leftCTE, rightCTE string) (string,
 }
 
 func (b *traceOperatorCTEBuilder) buildFinalQuery(ctx context.Context, selectFromCTE string, requestType qbtypes.RequestType) (*qbtypes.Statement, error) {
+	keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, b.getKeySelectors())
+	if err != nil {
+		return nil, err
+	}
+	b.adjustKeys(keys)
+
 	switch requestType {
 	case qbtypes.RequestTypeRaw:
-		return b.buildListQuery(ctx, selectFromCTE)
+		return b.buildListQuery(ctx, selectFromCTE, keys)
 	case qbtypes.RequestTypeTimeSeries:
-		return b.buildTimeSeriesQuery(ctx, selectFromCTE)
+		return b.buildTimeSeriesQuery(ctx, selectFromCTE, keys)
 	case qbtypes.RequestTypeTrace:
-		return b.buildTraceQuery(ctx, selectFromCTE)
+		return b.buildTraceQuery(ctx, selectFromCTE, keys)
 	case qbtypes.RequestTypeScalar:
-		return b.buildScalarQuery(ctx, selectFromCTE)
+		return b.buildScalarQuery(ctx, selectFromCTE, keys)
 	default:
 		return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type: %s", requestType)
 	}
 }
 
-func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFromCTE string) (*qbtypes.Statement, error) {
+func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFromCTE string, keys map[string][]*telemetrytypes.TelemetryFieldKey) (*qbtypes.Statement, error) {
 	sb := sqlbuilder.NewSelectBuilder()
 
 	// Select core fields
@@ -453,22 +459,6 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom
 		"parent_span_id": true,
 	}
 
-	// Get keys for selectFields
-	keySelectors := b.getKeySelectors()
-	for _, field := range b.operator.SelectFields {
-		keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{
-			Name:          field.Name,
-			Signal:        telemetrytypes.SignalTraces,
-			FieldContext:  field.FieldContext,
-			FieldDataType: field.FieldDataType,
-		})
-	}
-
-	keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
-	if err != nil {
-		return nil, err
-	}
-
 	// Add selectFields using ColumnExpressionFor since we now have all base table columns
 	for _, field := range b.operator.SelectFields {
 		if selectedFields[field.Name] {
@@ -545,6 +535,15 @@ func (b *traceOperatorCTEBuilder) getKeySelectors() []*telemetrytypes.FieldKeySe
 		})
 	}
 
+	for _, field := range b.operator.SelectFields {
+		keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{
+			Name:          field.Name,
+			Signal:        telemetrytypes.SignalTraces,
+			FieldContext:  field.FieldContext,
+			FieldDataType: field.FieldDataType,
+		})
+	}
+
 	for i := range keySelectors {
 		keySelectors[i].Signal = telemetrytypes.SignalTraces
 	}
@@ -552,7 +551,7 @@ func (b *traceOperatorCTEBuilder) getKeySelectors() []*telemetrytypes.FieldKeySe
 	return keySelectors
 }
 
-func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, selectFromCTE string) (*qbtypes.Statement, error) {
+func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, selectFromCTE string, keys map[string][]*telemetrytypes.TelemetryFieldKey) (*qbtypes.Statement, error) {
 	sb := sqlbuilder.NewSelectBuilder()
 
 	sb.Select(fmt.Sprintf(
@@ -560,12 +559,6 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
 		int64(b.operator.StepInterval.Seconds()),
 	))
 
-	keySelectors := b.getKeySelectors()
-	keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
-	if err != nil {
-		return nil, err
-	}
-
 	var allGroupByArgs []any
 
 	for _, gb := range b.operator.GroupBy {
@@ -644,8 +637,7 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
 	combinedArgs := append(allGroupByArgs, allAggChArgs...)
 
 	// Add HAVING clause if specified
-	err = b.addHavingClause(sb)
-	if err != nil {
+	if err := b.addHavingClause(sb); err != nil {
 		return nil, err
 	}
 
@@ -672,17 +664,11 @@ func (b *traceOperatorCTEBuilder) buildTraceSummaryCTE(selectFromCTE string) {
 	b.addCTE("trace_summary", sql, args, []string{"all_spans", selectFromCTE})
 }
 
-func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFromCTE string) (*qbtypes.Statement, error) {
+func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFromCTE string, keys map[string][]*telemetrytypes.TelemetryFieldKey) (*qbtypes.Statement, error) {
 	b.buildTraceSummaryCTE(selectFromCTE)
 
 	sb := sqlbuilder.NewSelectBuilder()
 
-	keySelectors := b.getKeySelectors()
-	keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
-	if err != nil {
-		return nil, err
-	}
-
 	var allGroupByArgs []any
 
 	for _, gb := range b.operator.GroupBy {
@@ -764,8 +750,7 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
 		sb.GroupBy(groupByKeys...)
 	}
 
-	err = b.addHavingClause(sb)
-	if err != nil {
+	if err := b.addHavingClause(sb); err != nil {
 		return nil, err
 	}
 
@@ -821,15 +806,9 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
 	}, nil
 }
 
-func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFromCTE string) (*qbtypes.Statement, error) {
+func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFromCTE string, keys map[string][]*telemetrytypes.TelemetryFieldKey) (*qbtypes.Statement, error) {
 	sb := sqlbuilder.NewSelectBuilder()
 
-	keySelectors := b.getKeySelectors()
-	keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
-	if err != nil {
-		return nil, err
-	}
-
 	var allGroupByArgs []any
 
 	for _, gb := range b.operator.GroupBy {
@@ -911,8 +890,7 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr
 	combinedArgs := append(allGroupByArgs, allAggChArgs...)
 
 	// Add HAVING clause if specified
-	err = b.addHavingClause(sb)
-	if err != nil {
+	if err := b.addHavingClause(sb); err != nil {
 		return nil, err
 	}
 
@@ -955,3 +933,16 @@ func (b *traceOperatorCTEBuilder) aggOrderBy(k qbtypes.OrderBy) (int, bool) {
 	}
 	return 0, false
 }
+
+func (b *traceOperatorCTEBuilder) adjustKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey) {
+	// todo: this needs to be updated w.r.t trace statement builder.
+	for i := range b.operator.SelectFields {
+		querybuilder.AdjustKey(&b.operator.SelectFields[i], keys, nil)
+	}
+	for i := range b.operator.GroupBy {
+		querybuilder.AdjustKey(&b.operator.GroupBy[i].TelemetryFieldKey, keys, nil)
+	}
+	for i := range b.operator.Order {
+		querybuilder.AdjustKey(&b.operator.Order[i].Key.TelemetryFieldKey, keys, nil)
+	}
+}

pkg/telemetrytraces/trace_operator_cte_builder_test.go

@@ -15,6 +15,7 @@ import (
 )
 
 func TestTraceOperatorStatementBuilder(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name           string
 		requestType    qbtypes.RequestType
@@ -466,7 +467,7 @@ func TestTraceOperatorStatementBuilder(t *testing.T) {
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 
@@ -519,6 +520,7 @@ func TestTraceOperatorStatementBuilder(t *testing.T) {
 }
 
 func TestTraceOperatorStatementBuilderErrors(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name           string
 		operator       qbtypes.QueryBuilderTraceOperator
@@ -582,7 +584,7 @@ func TestTraceOperatorStatementBuilderErrors(t *testing.T) {
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 

pkg/telemetrytraces/trace_time_range_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -16,12 +17,13 @@ import (
 )
 
 func TestTraceTimeRangeOptimization(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	mockMetadataStore.KeysMap["trace_id"] = []*telemetrytypes.TelemetryFieldKey{{
 		Name:          "trace_id",
 		FieldContext:  telemetrytypes.FieldContextSpan,

pkg/types/querybuildertypes/querybuildertypesv5/evolution.go

@@ -0,0 +1,119 @@
+package querybuildertypesv5
+
+import (
+	"slices"
+	"sort"
+	"strconv"
+	"time"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// SelectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
+// Logic:
+//   - Finds the latest base evolution (<= tsStartTime) across ALL columns
+//   - Rejects all evolutions before this latest base evolution
+//   - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
+//   - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime
+//   - Results are sorted by ReleaseTime descending (newest first)
+func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) {
+
+	sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions))
+	copy(sortedEvolutions, evolutions)
+
+	// sort the evolutions by ReleaseTime ascending
+	sort.Slice(sortedEvolutions, func(i, j int) bool {
+		return sortedEvolutions[i].ReleaseTime.Before(sortedEvolutions[j].ReleaseTime)
+	})
+
+	tsStartTime := time.Unix(0, int64(tsStart))
+	tsEndTime := time.Unix(0, int64(tsEnd))
+
+	// Build evolution map: column name -> evolution
+	evolutionMap := make(map[string]*telemetrytypes.EvolutionEntry)
+	for _, evolution := range sortedEvolutions {
+		if _, exists := evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))]; exists {
+			// since if there is duplicate we would just use the oldest one.
+			continue
+		}
+		evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))] = evolution
+	}
+
+	// Find the latest base evolution (<= tsStartTime) across ALL columns
+	// Evolutions are sorted, so we can break early
+	var latestBaseEvolutionAcrossAll *telemetrytypes.EvolutionEntry
+	for _, evolution := range sortedEvolutions {
+		if evolution.ReleaseTime.After(tsStartTime) {
+			break
+		}
+		latestBaseEvolutionAcrossAll = evolution
+	}
+
+	// We shouldn't reach this, it basically means there is something wrong with the evolutions data
+	if latestBaseEvolutionAcrossAll == nil {
+		return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
+	}
+
+	columnLookUpMap := make(map[string]*schema.Column)
+	for _, column := range columns {
+		columnLookUpMap[column.Name] = column
+	}
+
+	// Collect column-evolution pairs
+	type colEvoPair struct {
+		column    *schema.Column
+		evolution *telemetrytypes.EvolutionEntry
+	}
+	pairs := []colEvoPair{}
+
+	for _, evolution := range evolutionMap {
+		// Reject evolutions before the latest base evolution
+		if evolution.ReleaseTime.Before(latestBaseEvolutionAcrossAll.ReleaseTime) {
+			continue
+		}
+		// skip evolutions after tsEndTime
+		if evolution.ReleaseTime.After(tsEndTime) || evolution.ReleaseTime.Equal(tsEndTime) {
+			continue
+		}
+
+		if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
+			return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
+		}
+
+		pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
+	}
+
+	// If no pairs found, fall back to latestBaseEvolutionAcrossAll for matching columns
+	if len(pairs) == 0 {
+		for _, column := range columns {
+			// Use latestBaseEvolutionAcrossAll if this column name matches its column name
+			if column.Name == latestBaseEvolutionAcrossAll.ColumnName {
+				pairs = append(pairs, colEvoPair{column, latestBaseEvolutionAcrossAll})
+			}
+		}
+	}
+
+	// Sort by ReleaseTime descending (newest first)
+	slices.SortFunc(pairs, func(a, b colEvoPair) int {
+		// Sort by ReleaseTime descending (newest first)
+		if a.evolution.ReleaseTime.After(b.evolution.ReleaseTime) {
+			return -1
+		}
+		if a.evolution.ReleaseTime.Before(b.evolution.ReleaseTime) {
+			return 1
+		}
+		return 0
+	})
+
+	// Extract results
+	newColumns := make([]*schema.Column, len(pairs))
+	evolutionsEntries := make([]*telemetrytypes.EvolutionEntry, len(pairs))
+	for i, pair := range pairs {
+		newColumns[i] = pair.column
+		evolutionsEntries[i] = pair.evolution
+	}
+
+	return newColumns, evolutionsEntries, nil
+}

pkg/types/querybuildertypes/querybuildertypesv5/evolution_test.go

@@ -0,0 +1,414 @@
+package querybuildertypesv5
+
+import (
+	"testing"
+	"time"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	LogsV2BodyV2Column       = "body_v2"
+	LogsV2BodyPromotedColumn = "body_promoted"
+)
+
+var (
+	resources_string = &schema.Column{Name: "resources_string", Type: schema.MapColumnType{
+		KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		ValueType: schema.ColumnTypeString,
+	}}
+	resource          = &schema.Column{Name: "resource", Type: schema.JSONColumnType{}}
+	attributes_string = &schema.Column{Name: "attributes_string", Type: schema.MapColumnType{
+		KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		ValueType: schema.ColumnTypeString,
+	}}
+	body_v2       = &schema.Column{Name: LogsV2BodyV2Column, Type: schema.JSONColumnType{}}
+	body_promoted = &schema.Column{Name: LogsV2BodyPromotedColumn, Type: schema.JSONColumnType{}}
+)
+
+func TestSelectEvolutionsForColumns(t *testing.T) {
+	testCases := []struct {
+		name            string
+		columns         []*schema.Column
+		evolutions      []*telemetrytypes.EvolutionEntry
+		tsStart         uint64
+		tsEnd           uint64
+		expectedColumns []string // column names
+		expectedEvols   []string // evolution column names
+		expectedError   bool
+		errorStr        string
+	}{
+		{
+			name: "New evolutions at tsStartTime - should include latest evolution",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resource"},
+			expectedEvols:   []string{"resource"},
+		},
+		{
+			name: "New evolutions after tsStartTime but less than tsEndTime - should include both",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resource", "resources_string"}, // sorted by ReleaseTime desc
+			expectedEvols:   []string{"resource", "resources_string"},
+		},
+		{
+			name: "Columns without matching evolutions - should exclude them",
+			columns: []*schema.Column{
+				resources_string,
+				resource,          // no evolution for this
+				attributes_string, // no evolution for this
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string"},
+			expectedEvols:   []string{"resources_string"},
+		},
+		{
+			name: "New evolutions at tsEndTime - should not include new evolution",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string"},
+			expectedEvols:   []string{"resources_string"},
+		},
+		{
+			name: "New evolutions after tsEndTime - should exclude new",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string"},
+			expectedEvols:   []string{"resources_string"},
+		},
+		{
+			name:    "Empty columns array",
+			columns: []*schema.Column{},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{},
+			expectedEvols:   []string{},
+			expectedError:   true,
+			errorStr:        "column resources_string not found",
+		},
+		{
+			name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
+			columns: []*schema.Column{
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resource"},
+			expectedEvols:   []string{"resource"}, // should use first one (older)
+		},
+		{
+			name: "Genuine Duplicate evolutions with new version- should consider both",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      2,
+					ReleaseTime:  time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string", "resource"},
+			expectedEvols:   []string{"resources_string", "resource"}, // should use first one (older)
+		},
+		{
+			name: "Evolution exactly at tsEndTime",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					ReleaseTime:  time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), // exactly at tsEnd
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string"}, // resource excluded because After(tsEnd) is true
+			expectedEvols:   []string{"resources_string"},
+		},
+		{
+			name: "Single evolution after tsStartTime - JSON body",
+			columns: []*schema.Column{
+				body_v2,
+				body_promoted,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   LogsV2BodyV2Column,
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextBody,
+					FieldName:    "__all__",
+					ReleaseTime:  time.Unix(0, 0),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   LogsV2BodyPromotedColumn,
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextBody,
+					FieldName:    "user.name",
+					ReleaseTime:  time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column}, // sorted by ReleaseTime desc (newest first)
+			expectedEvols:   []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column},
+		},
+		{
+			name: "No evolution after tsStartTime - JSON body",
+			columns: []*schema.Column{
+				body_v2,
+				body_promoted,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   LogsV2BodyV2Column,
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextBody,
+					FieldName:    "__all__",
+					ReleaseTime:  time.Unix(0, 0),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   LogsV2BodyPromotedColumn,
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextBody,
+					FieldName:    "user.name",
+					ReleaseTime:  time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{LogsV2BodyPromotedColumn},
+			expectedEvols:   []string{LogsV2BodyPromotedColumn},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			resultColumns, resultEvols, err := SelectEvolutionsForColumns(tc.columns, tc.evolutions, tc.tsStart, tc.tsEnd)
+
+			if tc.expectedError {
+				assert.Contains(t, err.Error(), tc.errorStr)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, len(tc.expectedColumns), len(resultColumns), "column count mismatch")
+				assert.Equal(t, len(tc.expectedEvols), len(resultEvols), "evolution count mismatch")
+
+				resultColumnNames := make([]string, len(resultColumns))
+				for i, col := range resultColumns {
+					resultColumnNames[i] = col.Name
+				}
+				resultEvolNames := make([]string, len(resultEvols))
+				for i, evol := range resultEvols {
+					resultEvolNames[i] = evol.ColumnName
+				}
+
+				for i := range tc.expectedColumns {
+					assert.Equal(t, resultColumnNames[i], tc.expectedColumns[i], "expected column missing: "+tc.expectedColumns[i])
+				}
+				for i := range tc.expectedEvols {
+					assert.Equal(t, resultEvolNames[i], tc.expectedEvols[i], "expected evolution missing: "+tc.expectedEvols[i])
+				}
+				// Verify sorting: should be descending by ReleaseTime
+				for i := 0; i < len(resultEvols)-1; i++ {
+					assert.True(t, !resultEvols[i].ReleaseTime.Before(resultEvols[i+1].ReleaseTime),
+						"evolutions should be sorted descending by ReleaseTime")
+				}
+			}
+		})
+	}
+}

tests/fixtures/traces.py

@@ -6,7 +6,7 @@ import uuid
 from abc import ABC
 from collections.abc import Callable, Generator
 from enum import Enum
-from typing import Any
+from typing import Any, Literal
 from urllib.parse import urlparse
 
 import numpy as np
@@ -236,6 +236,7 @@ class Traces(ABC):
     attributes_number: dict[str, np.float64]
     attributes_bool: dict[str, bool]
     resources_string: dict[str, str]
+    resource_json: dict[str, str]
     events: list[str]
     links: str
     response_status_code: str
@@ -273,6 +274,7 @@ class Traces(ABC):
         links: list[TracesLink] = [],
         trace_state: str = "",
         flags: np.uint32 = 0,
+        resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
     ) -> None:
         if timestamp is None:
             timestamp = datetime.datetime.now()
@@ -322,8 +324,11 @@ class Traces(ABC):
         self.db_name = ""
         self.db_operation = ""
 
-        # Process resources and derive service_name
+        # Process resources and derive service_name. Spans written before the
+        # JSON-resource evolution time only populate resources_string (legacy_only);
+        # spans at or after the evolution time dual-write to both columns.
         self.resources_string = {k: str(v) for k, v in resources.items()}
+        self.resource_json = {} if resource_write_mode == "legacy_only" else dict(self.resources_string)
         self.service_name = self.resources_string.get("service.name", "default-service")
 
         for k, v in self.resources_string.items():
@@ -575,7 +580,7 @@ class Traces(ABC):
                 self.db_operation,
                 self.has_error,
                 self.is_remote,
-                self.resources_string,
+                self.resource_json,
             ],
             dtype=object,
         )

tests/integration/tests/querier/13_traces_resource_evolution.py

@@ -0,0 +1,240 @@
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from http import HTTPStatus
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.querier import (
+    build_group_by_field,
+    build_logs_aggregation,
+    index_series_by_label,
+    make_query_request,
+)
+from fixtures.traces import TraceIdGenerator, Traces
+
+
+# we already create the evolution for resource during schema migration
+# since we have to create test data around it, we need to get the evolution time
+def _get_traces_resource_evolution_time_json(signoz: types.SigNoz) -> datetime:
+    result = signoz.telemetrystore.conn.query(
+        """
+        SELECT release_time
+        FROM signoz_metadata.distributed_column_evolution_metadata
+        WHERE signal = 'traces'
+          AND field_context = 'resource'
+          AND field_name = '__all__'
+          AND column_name = 'resource'
+        LIMIT 1
+        """
+    ).result_rows
+
+    assert result, "Expected traces resource evolution metadata to exist"
+
+    release_time_ns = int(result[0][0])
+    return datetime.fromtimestamp(release_time_ns / 1e9, tz=UTC)
+
+
+# Spans with timestamps before the evolution time will have resources written only to resources_string.
+# Spans with timestamps at or after the evolution time will have resources written to both resources_string and resource (JSON).
+def _build_evolved_span(
+    timestamp: datetime,
+    evolution_time: datetime,
+    service_name: str,
+    name: str,
+) -> Traces:
+    resource_write_mode = "legacy_only" if timestamp < evolution_time else "dual_write"
+    return Traces(
+        timestamp=timestamp,
+        trace_id=TraceIdGenerator.trace_id(),
+        span_id=TraceIdGenerator.span_id(),
+        name=name,
+        resources={
+            "service.name": service_name,
+            "deployment.environment": "integration",
+        },
+        resource_write_mode=resource_write_mode,
+    )
+
+
+def _query_grouped_trace_series(
+    signoz: types.SigNoz,
+    token: str,
+    start: datetime,
+    end: datetime,
+    group_by: str = "service.name",
+    aggregation: str = "count()",
+) -> dict[str, list[dict]]:
+    response = make_query_request(
+        signoz,
+        token,
+        start_ms=int(start.timestamp() * 1000),
+        end_ms=int(end.timestamp() * 1000),
+        request_type="time_series",
+        queries=[
+            {
+                "type": "builder_query",
+                "spec": {
+                    "name": "A",
+                    "signal": "traces",
+                    "stepInterval": 60,
+                    "disabled": False,
+                    "groupBy": [build_group_by_field(group_by)],
+                    "having": {"expression": ""},
+                    "aggregations": [build_logs_aggregation(aggregation)],
+                },
+            }
+        ],
+    )
+
+    assert response.status_code == HTTPStatus.OK
+    assert response.json()["status"] == "success"
+
+    results = response.json()["data"]["data"]["results"]
+    assert len(results) == 1
+
+    aggregations = results[0]["aggregations"]
+    assert len(aggregations) == 1
+
+    return index_series_by_label(aggregations[0]["series"], group_by)
+
+
+def _assert_grouped_series(
+    series_by_group: dict[str, dict],
+    expected_values_by_group: dict[str, dict[int, int]],
+) -> None:
+    assert set(series_by_group.keys()) == set(expected_values_by_group.keys())
+
+    for group_name, expected_by_ts in expected_values_by_group.items():
+        actual_values = sorted(
+            series_by_group[group_name]["values"],
+            key=lambda value: value["timestamp"],
+        )
+        expected_values = [{"timestamp": timestamp, "value": value} for timestamp, value in sorted(expected_by_ts.items())]
+        assert actual_values == expected_values
+
+
+def _test_traces_resource_evolution(
+    signoz: types.SigNoz,
+    token: str,
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    # 1. Get the evolution time.
+    # 2. Ingest spans before the evolution time.
+    # 3. Ingest spans after the evolution time.
+    # 4. Query the spans before the evolution time.
+    # 5. Query the spans after the evolution time.
+    # Both aggregation and group by should be checked.
+    """
+    evolution_time = _get_traces_resource_evolution_time_json(signoz)
+    evolution_time = evolution_time.replace(second=0, microsecond=0)
+
+    before_2 = evolution_time - timedelta(minutes=10)
+    before_1 = evolution_time - timedelta(minutes=5)
+    after_1 = evolution_time + timedelta(minutes=5)
+    after_2 = evolution_time + timedelta(minutes=10)
+
+    insert_traces(
+        [
+            _build_evolved_span(
+                timestamp=before_2,
+                evolution_time=evolution_time,
+                service_name="svc-before-2",
+                name="span before evolution 2",
+            ),
+            _build_evolved_span(
+                timestamp=before_1,
+                evolution_time=evolution_time,
+                service_name="svc-before-1",
+                name="span before evolution 1",
+            ),
+            _build_evolved_span(
+                timestamp=after_1,
+                evolution_time=evolution_time,
+                service_name="svc-after-1",
+                name="span after evolution 1",
+            ),
+            _build_evolved_span(
+                timestamp=after_2,
+                evolution_time=evolution_time,
+                service_name="svc-after-2",
+                name="span after evolution 2",
+            ),
+        ]
+    )
+
+    before_series = _query_grouped_trace_series(signoz, token, before_2 - timedelta(minutes=1), before_1 + timedelta(minutes=1))
+    _assert_grouped_series(
+        before_series,
+        expected_values_by_group={
+            "svc-before-2": {
+                int(before_2.timestamp() * 1000): 1,
+            },
+            "svc-before-1": {
+                int(before_1.timestamp() * 1000): 1,
+            },
+        },
+    )
+
+    after_series = _query_grouped_trace_series(signoz, token, after_1 - timedelta(minutes=1), after_2 + timedelta(minutes=1))
+    _assert_grouped_series(
+        after_series,
+        expected_values_by_group={
+            "svc-after-1": {
+                int(after_1.timestamp() * 1000): 1,
+            },
+            "svc-after-2": {
+                int(after_2.timestamp() * 1000): 1,
+            },
+        },
+    )
+
+    spanning_series = _query_grouped_trace_series(signoz, token, before_2, after_2 + timedelta(minutes=1))
+    _assert_grouped_series(
+        spanning_series,
+        expected_values_by_group={
+            "svc-before-2": {
+                int(before_2.timestamp() * 1000): 1,
+            },
+            "svc-before-1": {
+                int(before_1.timestamp() * 1000): 1,
+            },
+            "svc-after-1": {
+                int(after_1.timestamp() * 1000): 1,
+            },
+            "svc-after-2": {
+                int(after_2.timestamp() * 1000): 1,
+            },
+        },
+    )
+
+    # query to check aggregation on the resource field like count_distinct(service.name)
+    aggregation_series = _query_grouped_trace_series(
+        signoz,
+        token,
+        before_2,
+        after_2 + timedelta(minutes=1),
+        group_by="deployment.environment",
+        aggregation="count_distinct(service.name)",
+    )
+    _assert_grouped_series(
+        aggregation_series,
+        expected_values_by_group={
+            "integration": {
+                int(before_2.timestamp() * 1000): 1,
+                int(before_1.timestamp() * 1000): 1,
+                int(after_1.timestamp() * 1000): 1,
+                int(after_2.timestamp() * 1000): 1,
+            },
+        },
+    )
+
+
+def test_traces_resource_evolution(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    _test_traces_resource_evolution(signoz, token, insert_traces)