feat: adding assets and dashboards for azure app services

* feat: field selector migrated to telemetry field key

* feat: move floating panel to field selector

* feat: sync columns state in logs

* feat: sync columns state in traces

* feat: logs field migration integrate

* feat: traces field migration integrate

* feat: minor refactor

* feat: tests updated

* feat: move to key from name on fields for logs and traces

* feat: update tests

* feat: update tests

frontend/src/components/FieldsSelector/AddedFields.tsx

@@ -27,13 +27,15 @@ function SortableField({
 	field,
 	onRemove,
 	allowDrag,
+	isRequired,
 }: {
 	field: TelemetryFieldKey;
 	onRemove: (field: TelemetryFieldKey) => void;
 	allowDrag: boolean;
+	isRequired: boolean;
 }): JSX.Element {
 	const { attributes, listeners, setNodeRef, transform, transition } =
-		useSortable({ id: field.name });
+		useSortable({ id: field.key as string });
 
 	const style = {
 		transform: CSS.Transform.toString(transform),
@@ -53,15 +55,17 @@ function SortableField({
 				{allowDrag && <GripVertical size={14} />}
 				<span className={styles.fieldKey}>{field.name}</span>
 			</div>
-			<Button
-				className={cx(styles.removeBtn, 'periscope-btn')}
-				variant="outlined"
-				color="destructive"
-				size="sm"
-				onClick={(): void => onRemove(field)}
-			>
-				Remove
-			</Button>
+			{!isRequired && (
+				<Button
+					className={cx(styles.removeBtn, 'periscope-btn')}
+					variant="outlined"
+					color="destructive"
+					size="sm"
+					onClick={(): void => onRemove(field)}
+				>
+					Remove
+				</Button>
+			)}
 		</div>
 	);
 }
@@ -71,6 +75,7 @@ interface AddedFieldsProps {
 	fields: TelemetryFieldKey[];
 	onFieldsChange: (fields: TelemetryFieldKey[]) => void;
 	maxFields?: number;
+	requiredFields?: readonly string[];
 }
 
 function AddedFields({
@@ -78,14 +83,18 @@ function AddedFields({
 	fields,
 	onFieldsChange,
 	maxFields,
+	requiredFields = [],
 }: AddedFieldsProps): JSX.Element {
 	const sensors = useSensors(useSensor(PointerSensor));
 
+	// Contract: caller (FieldsSelector) normalizes `fields` so every entry has
+	// `.key` populated. AddedFields reads it directly.
+
 	const handleDragEnd = (event: DragEndEvent): void => {
 		const { active, over } = event;
 		if (over && active.id !== over.id) {
-			const oldIndex = fields.findIndex((f) => f.name === active.id);
-			const newIndex = fields.findIndex((f) => f.name === over.id);
+			const oldIndex = fields.findIndex((f) => f.key === active.id);
+			const newIndex = fields.findIndex((f) => f.key === over.id);
 			onFieldsChange(arrayMove(fields, oldIndex, newIndex));
 		}
 	};
@@ -99,7 +108,7 @@ function AddedFields({
 	);
 
 	const handleRemove = (field: TelemetryFieldKey): void => {
-		onFieldsChange(fields.filter((f) => f.name !== field.name));
+		onFieldsChange(fields.filter((f) => f.key !== field.key));
 	};
 
 	const allowDrag = inputValue.length === 0;
@@ -125,16 +134,20 @@ function AddedFields({
 							<div className={styles.noValues}>No values found</div>
 						) : (
 							<SortableContext
-								items={fields.map((f) => f.name)}
+								items={fields.map((f) => f.key as string)}
 								strategy={verticalListSortingStrategy}
 								disabled={!allowDrag}
 							>
 								{filteredFields.map((field) => (
 									<SortableField
-										key={field.name}
+										key={field.key}
 										field={field}
 										onRemove={handleRemove}
 										allowDrag={allowDrag}
+										isRequired={
+											requiredFields.includes(field.name) ||
+											requiredFields.includes(field.key as string)
+										}
 									/>
 								))}
 							</SortableContext>

frontend/src/components/FieldsSelector/FieldsSelector.module.scss

@@ -76,11 +76,8 @@
 	min-height: 0;
 	overflow: hidden;
 
-	// Ant Skeleton.Input rendered inside the loading state — override its
-	// hard-coded width.
 	:global(.ant-skeleton-input) {
-		width: 300px;
-		margin: 8px 12px;
+		width: 50% !important;
 	}
 }
 
@@ -95,7 +92,8 @@
 	display: flex;
 	align-items: center;
 	justify-content: space-between;
-	padding: 6px 12px;
+	height: 36px;
+	padding: 0 12px;
 	border-radius: 4px;
 	user-select: none;
 	font-size: 13px;
@@ -132,7 +130,7 @@
 }
 
 .isDragDisabled {
-	padding: 6px 12px;
+	cursor: default;
 }
 
 .otherFieldItem {

frontend/src/components/FieldsSelector/FieldsSelector.tsx

@@ -5,6 +5,7 @@ import { Input } from '@signozhq/ui/input';
 import useDebouncedFn from 'hooks/useDebouncedFunction';
 import { Check, TableColumnsSplit, X } from '@signozhq/icons';
 import { FloatingPanel } from 'periscope/components/FloatingPanel';
+import { buildCompositeKey } from 'container/OptionsMenu/utils';
 import { TelemetryFieldKey } from 'types/api/v5/queryRange';
 import { DataSource } from 'types/common/queryBuilder';
 
@@ -26,33 +27,36 @@ interface FieldsSelectorProps {
 	onClose: () => void;
 	signal: DataSource;
 	maxFields?: number;
+	requiredFields?: readonly string[];
 	width?: number;
 	height?: number;
 	defaultPosition?: { x: number; y: number };
 }
 
-function FieldsSelector({
-	isOpen,
+type FieldsSelectorContentProps = Omit<FieldsSelectorProps, 'isOpen'>;
+
+// Inner component: holds all hooks + UI. Gets mounted/unmounted via the
+// outer gate so opening always seeds a fresh draft from `fields`.
+// Assumes `fields` arrives normalized (key populated) — see outer gate.
+function FieldsSelectorContent({
 	title,
 	fields,
 	onFieldsChange,
 	onClose,
 	signal,
 	maxFields,
+	requiredFields,
 	width = DEFAULT_PANEL_WIDTH,
 	height,
 	defaultPosition,
-}: FieldsSelectorProps): JSX.Element | null {
-	if (!isOpen) {
-		return null;
-	}
-
+}: FieldsSelectorContentProps): JSX.Element {
 	const resolvedHeight =
 		height ?? window.innerHeight - DEFAULT_PANEL_HEIGHT_OFFSET;
 	const resolvedPosition = defaultPosition ?? {
 		x: window.innerWidth - width - DEFAULT_PANEL_RIGHT_INSET,
 		y: DEFAULT_PANEL_TOP_INSET,
 	};
+
 	const [draftFields, setDraftFields] = useState<TelemetryFieldKey[]>(fields);
 	const [inputValue, setInputValue] = useState('');
 	const [debouncedInputValue, setDebouncedInputValue] = useState('');
@@ -72,15 +76,17 @@ function FieldsSelector({
 
 	const handleAdd = useCallback(
 		(field: TelemetryFieldKey): void => {
-			if (maxFields !== undefined && draftFields.length >= maxFields) {
-				return;
-			}
-			if (draftFields.some((f) => f.name === field.name)) {
-				return;
-			}
-			setDraftFields((prev) => [...prev, field]);
+			setDraftFields((prev) => {
+				if (maxFields !== undefined && prev.length >= maxFields) {
+					return prev;
+				}
+				if (prev.some((f) => f.key === field.key)) {
+					return prev;
+				}
+				return [...prev, field];
+			});
 		},
-		[draftFields, maxFields],
+		[maxFields],
 	);
 
 	const handleSave = useCallback((): void => {
@@ -99,7 +105,7 @@ function FieldsSelector({
 		() =>
 			!(
 				draftFields.length === fields.length &&
-				draftFields.every((f, i) => f.name === fields[i]?.name)
+				draftFields.every((f, i) => f.key === fields[i]?.key)
 			),
 		[draftFields, fields],
 	);
@@ -138,6 +144,7 @@ function FieldsSelector({
 					fields={draftFields}
 					onFieldsChange={setDraftFields}
 					maxFields={maxFields}
+					requiredFields={requiredFields}
 				/>
 
 				<OtherFields
@@ -173,4 +180,27 @@ function FieldsSelector({
 	);
 }
 
+// Outer gate: normalizes `fields` once (populates `key` so downstream code
+// can read it directly) and decides whether the inner component renders.
+// When isOpen flips false→true, the inner remounts → draft state seeds fresh.
+function FieldsSelector({
+	isOpen,
+	fields,
+	...rest
+}: FieldsSelectorProps): JSX.Element | null {
+	const normalizedFields = useMemo<TelemetryFieldKey[]>(
+		() =>
+			fields.map((f) => ({
+				...f,
+				key: f.key ?? buildCompositeKey(f.name, f.fieldContext),
+			})),
+		[fields],
+	);
+
+	if (!isOpen) {
+		return null;
+	}
+	return <FieldsSelectorContent {...rest} fields={normalizedFields} />;
+}
+
 export default FieldsSelector;

frontend/src/components/FieldsSelector/OtherFields.tsx

@@ -4,6 +4,7 @@ import { Skeleton } from 'antd';
 import cx from 'classnames';
 import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
 import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
+import { buildCompositeKey } from 'container/OptionsMenu/utils';
 import { useGetQueryKeySuggestions } from 'hooks/querySuggestions/useGetQueryKeySuggestions';
 import {
 	FieldContext,
@@ -47,15 +48,22 @@ function OtherFields({
 
 	const otherFields: TelemetryFieldKey[] = useMemo(() => {
 		const suggestions = Object.values(data?.data.data.keys || {}).flat();
-		const addedNames = new Set(addedFields.map((f) => f.name));
-		return suggestions
-			.filter((attr) => !addedNames.has(attr.name))
-			.map((attr) => ({
+		// Normalize: synthesize `key` once so downstream reads can trust it.
+		const normalizedSuggestions: TelemetryFieldKey[] = suggestions.map(
+			(attr) => ({
 				...attr,
+				key: buildCompositeKey(attr.name, attr.fieldContext as string),
 				signal: attr.signal as SignalType,
 				fieldContext: attr.fieldContext as FieldContext,
 				fieldDataType: attr.fieldDataType as FieldDataType,
-			}));
+			}),
+		);
+		const addedIds = new Set(
+			addedFields.map((f) => f.key ?? buildCompositeKey(f.name, f.fieldContext)),
+		);
+		return normalizedSuggestions.filter(
+			(attr) => !addedIds.has(attr.key as string),
+		);
 	}, [data, addedFields]);
 
 	if (isFetching) {
@@ -64,8 +72,13 @@ function OtherFields({
 				<div className={styles.sectionHeader}>OTHER FIELDS</div>
 				<div className={styles.otherList}>
 					{Array.from({ length: 5 }).map((_, i) => (
-						// eslint-disable-next-line react/no-array-index-key
-						<Skeleton.Input active size="small" key={i} />
+						<div
+							// eslint-disable-next-line react/no-array-index-key
+							key={i}
+							className={cx(styles.fieldItem, styles.otherFieldItem)}
+						>
+							<Skeleton.Input active size="small" block />
+						</div>
 					))}
 				</div>
 			</div>
@@ -83,7 +96,7 @@ function OtherFields({
 						) : (
 							otherFields.map((attr) => (
 								<div
-									key={attr.name}
+									key={attr.key}
 									className={cx(styles.fieldItem, styles.otherFieldItem)}
 								>
 									<span className={styles.fieldKey}>{attr.name}</span>

frontend/src/components/FieldsSelector/__tests__/AddedFields.test.tsx

@@ -0,0 +1,113 @@
+import { render, screen } from 'tests/test-utils';
+import { TelemetryFieldKey } from 'types/api/v5/queryRange';
+
+import AddedFields from '../AddedFields';
+
+// AddedFields assumes the caller has populated `key` (the parent
+// FieldsSelector does this via its normalization useMemo). Tests pre-populate
+// it directly.
+const makeField = (name: string, fieldContext = 'log'): TelemetryFieldKey => ({
+	name,
+	signal: 'logs',
+	fieldContext: fieldContext as TelemetryFieldKey['fieldContext'],
+	fieldDataType: 'string',
+	key: `${fieldContext}.${name}`,
+});
+
+describe('AddedFields — requiredFields', () => {
+	it('renders a Remove button for every field when no requiredFields are passed', () => {
+		const fields = [makeField('a'), makeField('b'), makeField('c')];
+
+		render(
+			<AddedFields inputValue="" fields={fields} onFieldsChange={jest.fn()} />,
+		);
+
+		expect(screen.getAllByRole('button', { name: /remove/i })).toHaveLength(3);
+	});
+
+	it('hides the Remove button for fields whose name is in requiredFields', () => {
+		const fields = [makeField('a'), makeField('b'), makeField('c')];
+
+		render(
+			<AddedFields
+				inputValue=""
+				fields={fields}
+				onFieldsChange={jest.fn()}
+				requiredFields={['a', 'c']}
+			/>,
+		);
+
+		// Only 'b' is removable.
+		const removeButtons = screen.getAllByRole('button', { name: /remove/i });
+		expect(removeButtons).toHaveLength(1);
+	});
+
+	it('still renders the field name for required fields', () => {
+		const fields = [makeField('a'), makeField('b')];
+
+		render(
+			<AddedFields
+				inputValue=""
+				fields={fields}
+				onFieldsChange={jest.fn()}
+				requiredFields={['a']}
+			/>,
+		);
+
+		expect(screen.getByText('a')).toBeInTheDocument();
+		expect(screen.getByText('b')).toBeInTheDocument();
+	});
+
+	it('locks all variants of a required name regardless of fieldContext', () => {
+		// Two `body` fields with different contexts — both should lock when
+		// `body` is in requiredFields.
+		const fields = [makeField('body', 'log'), makeField('body', 'attribute')];
+
+		render(
+			<AddedFields
+				inputValue=""
+				fields={fields}
+				onFieldsChange={jest.fn()}
+				requiredFields={['body']}
+			/>,
+		);
+
+		// Both 'body' variants locked → zero Remove buttons.
+		expect(screen.queryAllByRole('button', { name: /remove/i })).toHaveLength(0);
+	});
+
+	it('treats requiredFields as exact-name match (substring does not lock)', () => {
+		const fields = [makeField('body'), makeField('body_extra')];
+
+		render(
+			<AddedFields
+				inputValue=""
+				fields={fields}
+				onFieldsChange={jest.fn()}
+				requiredFields={['body']}
+			/>,
+		);
+
+		// 'body' locked, 'body_extra' removable.
+		expect(screen.getAllByRole('button', { name: /remove/i })).toHaveLength(1);
+	});
+
+	it('also accepts composite IDs in requiredFields (locks a specific variant)', () => {
+		// Two `body` fields with different contexts.
+		const fields = [makeField('body', 'log'), makeField('body', 'attribute')];
+
+		render(
+			<AddedFields
+				inputValue=""
+				fields={fields}
+				onFieldsChange={jest.fn()}
+				// Composite ID — locks ONLY the log variant, attribute variant stays
+				// removable.
+				requiredFields={['log.body']}
+			/>,
+		);
+
+		// One Remove button: the attribute variant. log variant is locked.
+		expect(screen.getAllByRole('button', { name: /remove/i })).toHaveLength(1);
+	});
+});

frontend/src/components/Logs/TableView/__tests__/useLogsTableColumns.test.tsx

@@ -0,0 +1,106 @@
+import { renderHook } from '@testing-library/react';
+import { FontSize } from 'container/OptionsMenu/types';
+import { IField } from 'types/api/logs/fields';
+
+import { useLogsTableColumns } from '../useLogsTableColumns';
+
+jest.mock('providers/Timezone', () => ({
+	useTimezone: (): { formatTimezoneAdjustedTimestamp: jest.Mock } => ({
+		formatTimezoneAdjustedTimestamp: jest.fn(() => 'TS'),
+	}),
+}));
+
+const field = (name: string): IField => ({
+	name,
+	type: '',
+	dataType: 'string',
+});
+
+describe('useLogsTableColumns — selectColumns-order respected', () => {
+	it('prepends stateIndicator and renders user fields in array order', () => {
+		const { result } = renderHook(() =>
+			useLogsTableColumns({
+				fields: [field('c'), field('a'), field('b')],
+				fontSize: FontSize.SMALL,
+			}),
+		);
+
+		expect(result.current.map((c) => c.id)).toStrictEqual([
+			'state-indicator',
+			'c',
+			'a',
+			'b',
+		]);
+	});
+
+	it('slots body and timestamp at their position in the fields array', () => {
+		const { result } = renderHook(() =>
+			useLogsTableColumns({
+				fields: [
+					field('service.name'),
+					field('body'),
+					field('request.id'),
+					field('timestamp'),
+				],
+				fontSize: FontSize.SMALL,
+			}),
+		);
+
+		// body/timestamp are NOT pinned to fixed positions — they appear where the
+		// caller placed them in `fields`. body/timestamp use composite IDs
+		// ('log.body', 'log.timestamp') since their fieldContext is fixed; user
+		// fields here have empty `type` so their composite collapses to bare name.
+		expect(result.current.map((c) => c.id)).toStrictEqual([
+			'state-indicator',
+			'service.name',
+			'log.body',
+			'request.id',
+			'log.timestamp',
+		]);
+	});
+
+	it('skips the synthetic "id" field name', () => {
+		const { result } = renderHook(() =>
+			useLogsTableColumns({
+				fields: [field('id'), field('a'), field('b')],
+				fontSize: FontSize.SMALL,
+			}),
+		);
+
+		expect(result.current.map((c) => c.id)).toStrictEqual([
+			'state-indicator',
+			'a',
+			'b',
+		]);
+	});
+
+	it('uses the special body/timestamp coldefs (canBeHidden=false), not the generic user field def', () => {
+		const { result } = renderHook(() =>
+			useLogsTableColumns({
+				fields: [field('body'), field('timestamp'), field('user_field')],
+				fontSize: FontSize.SMALL,
+			}),
+		);
+
+		const byId = new Map(result.current.map((c) => [c.id, c]));
+		// body + timestamp are locked from the table-X removal pathway.
+		expect(byId.get('log.body')?.canBeHidden).toBe(false);
+		expect(byId.get('log.body')?.enableRemove).toBe(false);
+		expect(byId.get('log.timestamp')?.canBeHidden).toBe(false);
+		expect(byId.get('log.timestamp')?.enableRemove).toBe(false);
+		// User-added fields stay removable. User field has type='' so composite
+		// collapses to bare name.
+		expect(byId.get('user_field')?.enableRemove).toBe(true);
+	});
+
+	it('renders only the stateIndicator when fields is empty', () => {
+		const { result } = renderHook(() =>
+			useLogsTableColumns({
+				fields: [],
+				fontSize: FontSize.SMALL,
+			}),
+		);
+
+		expect(result.current.map((c) => c.id)).toStrictEqual(['state-indicator']);
+	});
+});

frontend/src/components/Logs/TableView/useLogsTableColumns.tsx

@@ -7,6 +7,7 @@ import {
 	getSanitizedLogBody,
 } from 'container/LogDetailedView/utils';
 import { FontSize } from 'container/OptionsMenu/types';
+import { buildCompositeKey } from 'container/OptionsMenu/utils';
 import { FlatLogData } from 'lib/logs/flatLogData';
 import { useTimezone } from 'providers/Timezone';
 import { IField } from 'types/api/logs/fields';
@@ -18,13 +19,11 @@ import LogStateIndicator from '../LogStateIndicator/LogStateIndicator';
 type UseLogsTableColumnsProps = {
 	fields: IField[];
 	fontSize: FontSize;
-	appendTo?: 'center' | 'end';
 };
 
 export function useLogsTableColumns({
 	fields,
 	fontSize,
-	appendTo = 'center',
 }: UseLogsTableColumnsProps): TableColumnDef<ILog>[] {
 	const { formatTimezoneAdjustedTimestamp } = useTimezone();
 
@@ -47,74 +46,71 @@ export function useLogsTableColumns({
 			),
 		};
 
-		const fieldColumns: TableColumnDef<ILog>[] = fields
-			.filter((f): boolean => !['id', 'body', 'timestamp'].includes(f.name))
-			.map(
-				(f): TableColumnDef<ILog> => ({
-					id: f.name,
-					header: f.name,
-					accessorFn: (log): unknown => FlatLogData(log)[f.name],
-					enableRemove: true,
-					width: { min: 192 },
-					cell: ({ value }): ReactElement => (
-						<TanStackTable.Text>{String(value ?? '')}</TanStackTable.Text>
-					),
-				}),
-			);
+		const timestampCol: TableColumnDef<ILog> = {
+			id: buildCompositeKey('timestamp', 'log'),
+			header: 'Timestamp',
+			accessorFn: (log): unknown => log.timestamp,
+			canBeHidden: false,
+			enableRemove: false,
+			width: { default: 170, min: 170 },
+			cell: ({ value }): ReactElement => {
+				const ts = value as string | number;
+				const formatted =
+					typeof ts === 'string'
+						? formatTimezoneAdjustedTimestamp(ts, DATE_TIME_FORMATS.ISO_DATETIME_MS)
+						: formatTimezoneAdjustedTimestamp(
+								ts / 1e6,
+								DATE_TIME_FORMATS.ISO_DATETIME_MS,
+							);
+				return <TanStackTable.Text>{formatted}</TanStackTable.Text>;
+			},
+		};
 
-		const timestampCol: TableColumnDef<ILog> | null = fields.some(
-			(f) => f.name === 'timestamp',
-		)
-			? {
-					id: 'timestamp',
-					header: 'Timestamp',
-					accessorFn: (log): unknown => log.timestamp,
-					canBeHidden: false,
-					enableRemove: false,
-					width: { default: 170, min: 170 },
-					cell: ({ value }): ReactElement => {
-						const ts = value as string | number;
-						const formatted =
-							typeof ts === 'string'
-								? formatTimezoneAdjustedTimestamp(ts, DATE_TIME_FORMATS.ISO_DATETIME_MS)
-								: formatTimezoneAdjustedTimestamp(
-										ts / 1e6,
-										DATE_TIME_FORMATS.ISO_DATETIME_MS,
-									);
-						return <TanStackTable.Text>{formatted}</TanStackTable.Text>;
-					},
+		const bodyCol: TableColumnDef<ILog> = {
+			id: buildCompositeKey('body', 'log'),
+			header: 'Body',
+			accessorFn: (log): string => getBodyDisplayString(log.body),
+			canBeHidden: false,
+			enableRemove: false,
+			width: { default: '100%', min: 300 },
+			cell: ({ value, isActive }): ReactElement => (
+				<TanStackTable.Text
+					dangerouslySetInnerHTML={{
+						__html: getSanitizedLogBody(value as string, {
+							shouldEscapeHtml: true,
+						}),
+					}}
+					data-active={isActive}
+				/>
+			),
+		};
+
+		const makeUserFieldCol = (f: IField): TableColumnDef<ILog> => ({
+			id: buildCompositeKey(f.name, f.type),
+			header: f.name,
+			accessorFn: (log): unknown => FlatLogData(log)[f.name],
+			enableRemove: true,
+			width: { min: 192 },
+			cell: ({ value }): ReactElement => (
+				<TanStackTable.Text>{String(value ?? '')}</TanStackTable.Text>
+			),
+		});
+
+		const fieldCols = fields
+			.map((f): TableColumnDef<ILog> | null => {
+				if (f.name === 'id') {
+					return null;
 				}
-			: null;
-
-		const bodyCol: TableColumnDef<ILog> | null = fields.some(
-			(f) => f.name === 'body',
-		)
-			? {
-					id: 'body',
-					header: 'Body',
-					accessorFn: (log): string => getBodyDisplayString(log.body),
-					canBeHidden: false,
-					enableRemove: false,
-					width: { default: '100%', min: 300 },
-					cell: ({ value, isActive }): ReactElement => (
-						<TanStackTable.Text
-							dangerouslySetInnerHTML={{
-								__html: getSanitizedLogBody(value as string, {
-									shouldEscapeHtml: true,
-								}),
-							}}
-							data-active={isActive}
-						/>
-					),
+				if (f.name === 'timestamp') {
+					return timestampCol;
 				}
-			: null;
+				if (f.name === 'body') {
+					return bodyCol;
+				}
+				return makeUserFieldCol(f);
+			})
+			.filter((c): c is TableColumnDef<ILog> => c !== null);
 
-		return [
-			stateIndicatorCol,
-			...(timestampCol ? [timestampCol] : []),
-			...(appendTo === 'center' ? fieldColumns : []),
-			...(bodyCol ? [bodyCol] : []),
-			...(appendTo === 'end' ? fieldColumns : []),
-		];
-	}, [fields, appendTo, fontSize, formatTimezoneAdjustedTimestamp]);
+		return [stateIndicatorCol, ...fieldCols];
+	}, [fields, fontSize, formatTimezoneAdjustedTimestamp]);
 }

frontend/src/components/LogsFormatOptionsMenu/LogsFormatOptionsMenu.styles.scss

@@ -256,6 +256,34 @@
 				}
 			}
 
+			.edit-columns-container {
+				padding: 12px;
+
+				.edit-columns-btn {
+					display: flex;
+					width: 100%;
+					height: 20px;
+					padding: 4px 0px;
+					justify-content: space-between;
+					align-items: center;
+					border: none !important;
+
+					.edit-columns-text {
+						color: var(--l2-foreground);
+						font-family: Inter;
+						font-size: 13px;
+						font-style: normal;
+						font-weight: 400;
+						line-height: normal;
+						letter-spacing: 0.14px;
+					}
+				}
+
+				.edit-columns-btn:hover {
+					background-color: unset !important;
+				}
+			}
+
 			.add-new-column-container {
 				display: flex;
 				flex-direction: column;

frontend/src/components/LogsFormatOptionsMenu/LogsFormatOptionsMenu.tsx

@@ -1,12 +1,9 @@
-import { useCallback, useEffect, useRef, useState } from 'react';
-import { Input } from '@signozhq/ui/input';
+import { useCallback, useEffect, useState } from 'react';
 import { Button, InputNumber, Popover, Tooltip } from 'antd';
 import { Typography } from '@signozhq/ui/typography';
-import type { DefaultOptionType } from 'antd/es/select';
 import cx from 'classnames';
 import { LogViewMode } from 'container/LogsTable';
 import { FontSize, OptionsMenuConfig } from 'container/OptionsMenu/types';
-import useDebouncedFn from 'hooks/useDebouncedFunction';
 import {
 	Check,
 	ChevronLeft,
@@ -14,7 +11,6 @@ import {
 	Minus,
 	Plus,
 	SlidersVertical,
-	X,
 } from '@signozhq/icons';
 
 import './LogsFormatOptionsMenu.styles.scss';
@@ -23,14 +19,21 @@ interface LogsFormatOptionsMenuProps {
 	items: any;
 	selectedOptionFormat: any;
 	config: OptionsMenuConfig;
+	onOpenColumns?: () => void;
+}
+
+interface OptionsMenuContentProps extends LogsFormatOptionsMenuProps {
+	closePopover: () => void;
 }
 
 function OptionsMenu({
 	items,
 	selectedOptionFormat,
 	config,
-}: LogsFormatOptionsMenuProps): JSX.Element {
-	const { maxLines, format, addColumn, fontSize } = config;
+	onOpenColumns,
+	closePopover,
+}: OptionsMenuContentProps): JSX.Element {
+	const { maxLines, format, fontSize } = config;
 	const [selectedItem, setSelectedItem] = useState(selectedOptionFormat);
 	const maxLinesNumber = (maxLines?.value as number) || 1;
 	const [maxLinesPerRow, setMaxLinesPerRow] = useState<number>(maxLinesNumber);
@@ -40,13 +43,6 @@ function OptionsMenu({
 	const [isFontSizeOptionsOpen, setIsFontSizeOptionsOpen] =
 		useState<boolean>(false);
 
-	const [showAddNewColumnContainer, setShowAddNewColumnContainer] =
-		useState(false);
-
-	const [selectedValue, setSelectedValue] = useState<string | null>(null);
-	const listRef = useRef<HTMLDivElement>(null);
-	const initialMouseEnterRef = useRef<boolean>(false);
-
 	const onChange = useCallback(
 		(key: LogViewMode) => {
 			if (!format) {
@@ -61,7 +57,6 @@ function OptionsMenu({
 	const handleMenuItemClick = (key: LogViewMode): void => {
 		setSelectedItem(key);
 		onChange(key);
-		setShowAddNewColumnContainer(false);
 	};
 
 	const incrementMaxLinesPerRow = (): void => {
@@ -76,20 +71,6 @@ function OptionsMenu({
 		}
 	};
 
-	const handleSearchValueChange = useDebouncedFn((event): void => {
-		// @ts-expect-error
-		const value = event?.target?.value || '';
-
-		if (addColumn && addColumn?.onSearch) {
-			addColumn?.onSearch(value);
-		}
-	}, 300);
-
-	const handleToggleAddNewColumn = (): void => {
-		addColumn?.onSearch?.('');
-		setShowAddNewColumnContainer(!showAddNewColumnContainer);
-	};
-
 	const handleLinesPerRowChange = (maxLinesPerRow: number | null): void => {
 		if (
 			maxLinesPerRow &&
@@ -100,6 +81,11 @@ function OptionsMenu({
 		}
 	};
 
+	const handleEditColumns = (): void => {
+		onOpenColumns?.();
+		closePopover();
+	};
+
 	useEffect(() => {
 		if (maxLinesPerRow && config && config.maxLines?.onChange) {
 			config.maxLines.onChange(maxLinesPerRow);
@@ -112,110 +98,10 @@ function OptionsMenu({
 		}
 	}, [fontSizeValue]);
 
-	function handleColumnSelection(
-		currentIndex: number,
-		optionsData: DefaultOptionType[],
-	): void {
-		const currentItem = optionsData[currentIndex];
-		const itemLength = optionsData.length;
-		if (addColumn && addColumn?.onSelect) {
-			addColumn?.onSelect(selectedValue, {
-				label: currentItem.label,
-				disabled: false,
-			});
-
-			// if the last element is selected then select the previous one
-			if (currentIndex === itemLength - 1) {
-				// there should be more than 1 element in the list
-				if (currentIndex - 1 >= 0) {
-					const prevValue = optionsData[currentIndex - 1]?.value || null;
-					setSelectedValue(prevValue as string | null);
-				} else {
-					// if there is only one element then just select and do nothing
-					setSelectedValue(null);
-				}
-			} else {
-				// selecting any random element from the list except the last one
-				const nextIndex = currentIndex + 1;
-
-				const nextValue = optionsData[nextIndex]?.value || null;
-
-				setSelectedValue(nextValue as string | null);
-			}
-		}
-	}
-
-	const handleKeyDown = (e: KeyboardEvent): void => {
-		if (!selectedValue) {
-			return;
-		}
-
-		const optionsData = addColumn?.options || [];
-
-		const currentIndex = optionsData.findIndex(
-			(item) => item?.value === selectedValue,
-		);
-
-		const itemLength = optionsData.length;
-
-		switch (e.key) {
-			case 'ArrowUp': {
-				const newValue = optionsData[Math.max(0, currentIndex - 1)]?.value;
-
-				setSelectedValue(newValue as string | null);
-				e.preventDefault();
-				break;
-			}
-			case 'ArrowDown': {
-				const newValue =
-					optionsData[Math.min(itemLength - 1, currentIndex + 1)]?.value;
-
-				setSelectedValue(newValue as string | null);
-				e.preventDefault();
-				break;
-			}
-			case 'Enter':
-				e.preventDefault();
-				handleColumnSelection(currentIndex, optionsData);
-				break;
-			default:
-				break;
-		}
-	};
-
-	useEffect(() => {
-		// Scroll the selected item into view
-		const listNode = listRef.current;
-		if (listNode && selectedValue) {
-			const optionsData = addColumn?.options || [];
-			const currentIndex = optionsData.findIndex(
-				(item) => item?.value === selectedValue,
-			);
-			const itemNode = listNode.children[currentIndex] as HTMLElement;
-			if (itemNode) {
-				itemNode.scrollIntoView({
-					behavior: 'smooth',
-					block: 'nearest',
-				});
-			}
-		}
-	}, [selectedValue]);
-
-	useEffect(() => {
-		window.addEventListener('keydown', handleKeyDown);
-		return (): void => {
-			window.removeEventListener('keydown', handleKeyDown);
-		};
-	}, [selectedValue]);
-
 	return (
 		<div
-			className={cx(
-				'nested-menu-container',
-				showAddNewColumnContainer ? 'active' : '',
-			)}
+			className="nested-menu-container"
 			onClick={(event): void => {
-				// this is to restrict click events to propogate to parent
 				event.stopPropagation();
 			}}
 		>
@@ -269,71 +155,7 @@ function OptionsMenu({
 						</Button>
 					</div>
 				</div>
-			) : null}
-
-			{showAddNewColumnContainer && (
-				<div className="add-new-column-container">
-					<div className="add-new-column-header">
-						<div className="title">
-							<div className="periscope-btn ghost" onClick={handleToggleAddNewColumn}>
-								<ChevronLeft
-									size={14}
-									className="back-icon"
-									onClick={handleToggleAddNewColumn}
-								/>
-							</div>
-							Add New Column
-						</div>
-
-						<Input
-							tabIndex={0}
-							type="text"
-							autoFocus
-							onFocus={addColumn?.onFocus}
-							onChange={handleSearchValueChange}
-							placeholder="Search..."
-						/>
-					</div>
-
-					<div className="add-new-column-content">
-						{addColumn?.isFetching && (
-							<div className="loading-container"> Loading ... </div>
-						)}
-
-						<div className="column-format-new-options" ref={listRef}>
-							{addColumn?.options?.map(({ label, value }, index) => (
-								<div
-									className={cx('column-name', value === selectedValue && 'selected')}
-									key={value}
-									onMouseEnter={(): void => {
-										if (!initialMouseEnterRef.current) {
-											setSelectedValue(value as string | null);
-										}
-
-										initialMouseEnterRef.current = true;
-									}}
-									onMouseMove={(): void => {
-										// this is added to handle the mouse move explicit event and not the re-rendered on mouse enter event
-										setSelectedValue(value as string | null);
-									}}
-									onClick={(eve): void => {
-										eve.stopPropagation();
-										handleColumnSelection(index, addColumn?.options || []);
-									}}
-								>
-									<div className="name">
-										<Tooltip placement="left" title={label}>
-											{label}
-										</Tooltip>
-									</div>
-								</div>
-							))}
-						</div>
-					</div>
-				</div>
-			)}
-
-			{!isFontSizeOptionsOpen && !showAddNewColumnContainer && (
+			) : (
 				<div>
 					<div className="font-size-container">
 						<div className="title">Font Size</div>
@@ -373,72 +195,52 @@ function OptionsMenu({
 
 					{selectedItem && (
 						<>
-							<>
-								<div className="horizontal-line" />
-								<div className="max-lines-per-row">
-									<div className="title"> max lines per row </div>
-									<div className="raw-format max-lines-per-row-input">
-										<button
-											type="button"
-											className="periscope-btn"
-											onClick={decrementMaxLinesPerRow}
-										>
-											{' '}
-											<Minus size={12} />{' '}
-										</button>
-										<InputNumber
-											min={1}
-											max={10}
-											value={maxLinesPerRow}
-											onChange={handleLinesPerRowChange}
-										/>
-										<button
-											type="button"
-											className="periscope-btn"
-											onClick={incrementMaxLinesPerRow}
-										>
-											{' '}
-											<Plus size={12} />{' '}
-										</button>
-									</div>
+							<div className="horizontal-line" />
+							<div className="max-lines-per-row">
+								<div className="title"> max lines per row </div>
+								<div className="raw-format max-lines-per-row-input">
+									<button
+										type="button"
+										className="periscope-btn"
+										onClick={decrementMaxLinesPerRow}
+									>
+										{' '}
+										<Minus size={12} />{' '}
+									</button>
+									<InputNumber
+										min={1}
+										max={10}
+										value={maxLinesPerRow}
+										onChange={handleLinesPerRowChange}
+									/>
+									<button
+										type="button"
+										className="periscope-btn"
+										onClick={incrementMaxLinesPerRow}
+									>
+										{' '}
+										<Plus size={12} />{' '}
+									</button>
 								</div>
-							</>
+							</div>
+						</>
+					)}
 
-							<div className="selected-item-content-container active">
-								{!showAddNewColumnContainer && <div className="horizontal-line" />}
-
-								<div className="item-content">
-									{!showAddNewColumnContainer && (
-										<div className="title">
-											columns
-											<Plus size={14} onClick={handleToggleAddNewColumn} />{' '}
-										</div>
-									)}
-
-									<div className="column-format">
-										{addColumn?.value?.map(({ name }) => (
-											<div className="column-name" key={name}>
-												<div className="name">
-													<Tooltip placement="left" title={name}>
-														{name}
-													</Tooltip>
-												</div>
-												{addColumn?.value?.length > 1 && (
-													<X
-														className="delete-btn"
-														size={14}
-														onClick={(): void => addColumn.onRemove(name)}
-													/>
-												)}
-											</div>
-										))}
-										{addColumn && addColumn?.value?.length === 0 && (
-											<div className="column-name no-columns-selected">
-												No columns selected
-											</div>
-										)}
-									</div>
-								</div>
+					{selectedItem === 'table' && onOpenColumns && (
+						<>
+							<div className="horizontal-line" />
+							<div className="edit-columns-container">
+								<Button
+									className="edit-columns-btn"
+									type="text"
+									onClick={handleEditColumns}
+									data-testid="periscope-btn-edit-columns"
+								>
+									<Typography.Text className="edit-columns-text">
+										Edit columns
+									</Typography.Text>
+									<ChevronRight size={14} className="icon" />
+								</Button>
 							</div>
 						</>
 					)}
@@ -452,6 +254,7 @@ function LogsFormatOptionsMenu({
 	items,
 	selectedOptionFormat,
 	config,
+	onOpenColumns,
 }: LogsFormatOptionsMenuProps): JSX.Element {
 	const [isPopoverOpen, setIsPopoverOpen] = useState<boolean>(false);
 	return (
@@ -461,6 +264,8 @@ function LogsFormatOptionsMenu({
 					items={items}
 					selectedOptionFormat={selectedOptionFormat}
 					config={config}
+					onOpenColumns={onOpenColumns}
+					closePopover={(): void => setIsPopoverOpen(false)}
 				/>
 			}
 			trigger="click"

frontend/src/components/LogsFormatOptionsMenu/__tests__/LogsFormatOptionsMenu.test.tsx

@@ -155,4 +155,66 @@ describe('LogsFormatOptionsMenu (unit)', () => {
 			expect(fontSizeOnChange).toHaveBeenCalledWith(FontSize.MEDIUM);
 		});
 	});
+
+	function renderWithOnOpen(
+		onOpenColumns?: jest.Mock,
+		selectedOptionFormat: 'table' | 'raw' | 'list' = 'table',
+	): { getByTestId: ReturnType<typeof render>['getByTestId'] } {
+		const items = [
+			{ key: 'raw', label: 'Raw', data: { title: 'max lines per row' } },
+			{ key: 'list', label: 'Default' },
+			{ key: 'table', label: 'Column', data: { title: 'columns' } },
+		];
+
+		const { getByTestId } = render(
+			<LogsFormatOptionsMenu
+				items={items}
+				selectedOptionFormat={selectedOptionFormat}
+				config={{
+					format: { value: selectedOptionFormat, onChange: jest.fn() },
+					maxLines: { value: 1, onChange: jest.fn() },
+					fontSize: { value: FontSize.SMALL, onChange: jest.fn() },
+				}}
+				onOpenColumns={onOpenColumns}
+			/>,
+		);
+		fireEvent.click(getByTestId('periscope-btn-format-options'));
+		return { getByTestId };
+	}
+
+	it('renders "Edit columns" row when format=table and onOpenColumns provided', () => {
+		const onOpenColumns = jest.fn();
+		const { getByTestId } = renderWithOnOpen(onOpenColumns, 'table');
+
+		expect(getByTestId('periscope-btn-edit-columns')).toBeInTheDocument();
+	});
+
+	it('does not render "Edit columns" row when onOpenColumns is not provided', () => {
+		renderWithOnOpen(undefined, 'table');
+
+		expect(
+			document.querySelector('[data-testid="periscope-btn-edit-columns"]'),
+		).toBeNull();
+	});
+
+	it('does not render "Edit columns" row when format is not table', () => {
+		renderWithOnOpen(jest.fn(), 'raw');
+
+		expect(
+			document.querySelector('[data-testid="periscope-btn-edit-columns"]'),
+		).toBeNull();
+	});
+
+	it('fires onOpenColumns and closes the popover when "Edit columns" is clicked', async () => {
+		const onOpenColumns = jest.fn();
+		const { getByTestId } = renderWithOnOpen(onOpenColumns, 'table');
+
+		fireEvent.click(getByTestId('periscope-btn-edit-columns'));
+
+		expect(onOpenColumns).toHaveBeenCalledTimes(1);
+		await waitFor(() => {
+			// Popover content unmounts on close.
+			expect(document.querySelector('.menu-container')).toBeNull();
+		});
+	});
 });

frontend/src/components/TanStackTableView/TanStackTable.tsx

@@ -72,6 +72,7 @@ function TanStackTableInner<TData>(
 		data,
 		columns,
 		columnStorageKey,
+		respectColumnOrder = true,
 		columnSizing: columnSizingProp,
 		onColumnSizingChange,
 		onColumnOrderChange,
@@ -175,6 +176,7 @@ function TanStackTableInner<TData>(
 		storageKey: columnStorageKey,
 		columns,
 		isGrouped,
+		respectColumnOrder,
 	});
 
 	// Use store values when columnStorageKey is provided, otherwise fall back to props/defaults
@@ -206,6 +208,7 @@ function TanStackTableInner<TData>(
 		handleRemoveColumn,
 	} = useColumnHandlers({
 		columnStorageKey,
+		respectColumnOrder,
 		effectiveSizing,
 		storeSetSizing,
 		storeSetOrder,

frontend/src/components/TanStackTableView/__tests__/TanStackTableView.test.tsx

@@ -24,6 +24,8 @@ jest.mock('../TanStackTable.module.scss', () => ({
 }));
 
 beforeAll(() => {
+	// jsdom doesn't include ResizeObserver — must direct-assign rather than
+	// spyOn (spyOn requires the property to already exist).
 	window.ResizeObserver = jest.fn().mockImplementation(() => ({
 		disconnect: jest.fn(),
 		observe: jest.fn(),
@@ -867,4 +869,110 @@ describe('TanStackTableView Integration', () => {
 			expect(screen.queryByRole('navigation')).not.toBeInTheDocument();
 		});
 	});
+
+	describe('hasSingleColumn — gates the Remove popover per-column', () => {
+		const hasSingleColumnFlagPresent = (): boolean =>
+			Boolean(document.querySelector('th[data-single-column="true"]'));
+
+		it('is true when only one non-pinned column exists', async () => {
+			renderTanStackTable({
+				props: {
+					data: [{ id: '1', name: 'Item 1', value: 100 }],
+					columns: [
+						{
+							id: 'name',
+							header: 'Name',
+							accessorKey: 'name',
+							cell: ({ value }): string => String(value),
+						},
+					],
+				},
+			});
+
+			await waitFor(() => {
+				expect(screen.getByText('Item 1')).toBeInTheDocument();
+			});
+
+			expect(hasSingleColumnFlagPresent()).toBe(true);
+		});
+
+		it('is false when multiple non-pinned columns exist (all removable)', async () => {
+			renderTanStackTable({});
+
+			await waitFor(() => {
+				expect(screen.getByText('Item 1')).toBeInTheDocument();
+			});
+
+			// 3 default columns (id/name/value), none pinned, none non-removable
+			// → table is not single-column.
+			expect(hasSingleColumnFlagPresent()).toBe(false);
+		});
+
+		it('is false when removable + non-removable mix exists (the body/timestamp case)', async () => {
+			renderTanStackTable({
+				props: {
+					data: [{ id: '1', name: 'Item 1', value: 100 }],
+					columns: [
+						{
+							id: 'name',
+							header: 'Timestamp',
+							accessorKey: 'name',
+							enableRemove: false,
+							cell: ({ value }): string => String(value),
+						},
+						{
+							id: 'value',
+							header: 'Body',
+							accessorKey: 'value',
+							enableRemove: false,
+							cell: ({ value }): string => String(value),
+						},
+						{
+							id: 'id',
+							header: 'User',
+							accessorKey: 'id',
+							enableRemove: true,
+							cell: ({ value }): string => String(value),
+						},
+					],
+				},
+			});
+
+			await waitFor(() => {
+				expect(screen.getByText('Item 1')).toBeInTheDocument();
+			});
+
+			expect(hasSingleColumnFlagPresent()).toBe(false);
+		});
+
+		it('does not count pinned columns toward the total', async () => {
+			renderTanStackTable({
+				props: {
+					data: [{ id: '1', name: 'Item 1', value: 100 }],
+					columns: [
+						{
+							id: 'stateIndicator',
+							header: '',
+							pin: 'left',
+							accessorKey: 'id',
+							cell: ({ value }): string => String(value),
+						},
+						{
+							id: 'name',
+							header: 'Name',
+							accessorKey: 'name',
+							cell: ({ value }): string => String(value),
+						},
+					],
+				},
+			});
+
+			await waitFor(() => {
+				expect(screen.getByText('Item 1')).toBeInTheDocument();
+			});
+
+			// 1 pinned + 1 non-pinned → only the non-pinned counts → single-column.
+			expect(hasSingleColumnFlagPresent()).toBe(true);
+		});
+	});
 });

frontend/src/components/TanStackTableView/__tests__/useColumnState.test.tsx

@@ -168,6 +168,53 @@ describe('useColumnState', () => {
 				'a',
 			]);
 		});
+
+		it('ignores stored columnOrder when respectColumnOrder=false', () => {
+			const columns = [col('a'), col('b'), col('c')];
+
+			act(() => {
+				useColumnStore.getState().initializeFromDefaults(TEST_KEY, columns);
+				useColumnStore.getState().setColumnOrder(TEST_KEY, ['c', 'a', 'b']);
+			});
+
+			const { result } = renderHook(() =>
+				useColumnState({
+					storageKey: TEST_KEY,
+					columns,
+					respectColumnOrder: false,
+				}),
+			);
+
+			// Falls through to natural columns-array order; stored order is ignored.
+			expect(result.current.sortedColumns.map((c) => c.id)).toStrictEqual([
+				'a',
+				'b',
+				'c',
+			]);
+		});
+
+		it('honors stored columnOrder when respectColumnOrder=true (default)', () => {
+			const columns = [col('a'), col('b'), col('c')];
+
+			act(() => {
+				useColumnStore.getState().initializeFromDefaults(TEST_KEY, columns);
+				useColumnStore.getState().setColumnOrder(TEST_KEY, ['c', 'a', 'b']);
+			});
+
+			const { result } = renderHook(() =>
+				useColumnState({
+					storageKey: TEST_KEY,
+					columns,
+					respectColumnOrder: true,
+				}),
+			);
+
+			expect(result.current.sortedColumns.map((c) => c.id)).toStrictEqual([
+				'c',
+				'a',
+				'b',
+			]);
+		});
 	});
 
 	describe('actions', () => {

frontend/src/components/TanStackTableView/types.ts

@@ -152,6 +152,8 @@ export type TanStackTableProps<TData> = {
 	columns: TableColumnDef<TData>[];
 	/** Storage key for column state persistence (visibility, sizing, ordering). When set, enables unified column management. */
 	columnStorageKey?: string;
+	/** When false, the table renders columns in their natural array order and the persisted columnOrder slot is ignored on read and skipped on write. Use when an external source (e.g. preferences.selectColumns) is the canonical order. Default true. */
+	respectColumnOrder?: boolean;
 	columnSizing?: ColumnSizingState;
 	onColumnSizingChange?: Dispatch<SetStateAction<ColumnSizingState>>;
 	onColumnOrderChange?: (cols: TableColumnDef<TData>[]) => void;

frontend/src/components/TanStackTableView/useColumnHandlers.ts

@@ -7,6 +7,8 @@ import { TableColumnDef } from './types';
 export interface UseColumnHandlersOptions<TData> {
 	/** Storage key for persisting column state (enables store mode) */
 	columnStorageKey?: string;
+	/** When false, drag-reorder skips the persisted columnOrder write — order flows back via onColumnOrderChange only. */
+	respectColumnOrder?: boolean;
 	effectiveSizing: ColumnSizingState;
 	storeSetSizing: (sizing: ColumnSizingState) => void;
 	storeSetOrder: (columns: TableColumnDef<TData>[]) => void;
@@ -28,6 +30,7 @@ export interface UseColumnHandlersResult<TData> {
  */
 export function useColumnHandlers<TData>({
 	columnStorageKey,
+	respectColumnOrder = true,
 	effectiveSizing,
 	storeSetSizing,
 	storeSetOrder,
@@ -50,12 +53,12 @@ export function useColumnHandlers<TData>({
 
 	const handleColumnOrderChange = useCallback(
 		(cols: TableColumnDef<TData>[]) => {
-			if (columnStorageKey) {
+			if (columnStorageKey && respectColumnOrder) {
 				storeSetOrder(cols);
 			}
 			onColumnOrderChange?.(cols);
 		},
-		[columnStorageKey, storeSetOrder, onColumnOrderChange],
+		[columnStorageKey, respectColumnOrder, storeSetOrder, onColumnOrderChange],
 	);
 
 	const handleRemoveColumn = useCallback(

frontend/src/components/TanStackTableView/useColumnState.ts

@@ -20,6 +20,7 @@ type UseColumnStateOptions<TData> = {
 	storageKey?: string;
 	columns: TableColumnDef<TData>[];
 	isGrouped?: boolean;
+	respectColumnOrder?: boolean;
 };
 
 type UseColumnStateResult<TData> = {
@@ -40,6 +41,7 @@ export function useColumnState<TData>({
 	storageKey,
 	columns,
 	isGrouped = false,
+	respectColumnOrder = true,
 }: UseColumnStateOptions<TData>): UseColumnStateResult<TData> {
 	useEffect(() => {
 		if (storageKey) {
@@ -130,7 +132,7 @@ export function useColumnState<TData>({
 	}, [hiddenColumnIds, columns, isGrouped]);
 
 	const sortedColumns = useMemo((): TableColumnDef<TData>[] => {
-		if (columnOrder.length === 0) {
+		if (!respectColumnOrder || columnOrder.length === 0) {
 			return columns;
 		}
 
@@ -144,7 +146,7 @@ export function useColumnState<TData>({
 		});
 
 		return [...pinned, ...sortedRest];
-	}, [columns, columnOrder]);
+	}, [columns, columnOrder, respectColumnOrder]);
 
 	const hideColumn = useCallback(
 		(columnId: string) => {

frontend/src/container/LiveLogs/LiveLogsContainer/index.tsx

@@ -2,12 +2,14 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { useLocation } from 'react-router-dom';
 import { Switch } from '@signozhq/ui/switch';
 import { Typography } from '@signozhq/ui/typography';
+import FieldsSelector from 'components/FieldsSelector';
 import LogsFormatOptionsMenu from 'components/LogsFormatOptionsMenu/LogsFormatOptionsMenu';
 import { MAX_LOGS_LIST_SIZE } from 'constants/liveTail';
 import { LOCALSTORAGE } from 'constants/localStorage';
 import { ChangeViewFunctionType } from 'container/ExplorerOptions/types';
 import GoToTop from 'container/GoToTop';
 import { useOptionsMenu } from 'container/OptionsMenu';
+import { LOGS_REQUIRED_COLUMNS } from 'container/OptionsMenu/constants';
 import { useGetCompositeQueryParam } from 'hooks/queryBuilder/useGetCompositeQueryParam';
 import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
 import useDebouncedFn from 'hooks/useDebouncedFunction';
@@ -56,6 +58,8 @@ function LiveLogsContainer({
 		aggregateOperator: listQuery?.aggregateOperator || StringOperators.NOOP,
 	});
 
+	const [isFieldsSelectorOpen, setIsFieldsSelectorOpen] = useState(false);
+
 	const formatItems = [
 		{
 			key: 'raw',
@@ -238,6 +242,7 @@ function LiveLogsContainer({
 						items={formatItems}
 						selectedOptionFormat={options.format}
 						config={config}
+						onOpenColumns={(): void => setIsFieldsSelectorOpen(true)}
 					/>
 				</div>
 
@@ -261,6 +266,17 @@ function LiveLogsContainer({
 			</div>
 
 			<GoToTop />
+			{config.fieldsSelector && (
+				<FieldsSelector
+					isOpen={isFieldsSelectorOpen}
+					title="Edit columns"
+					fields={config.fieldsSelector.value}
+					onFieldsChange={config.fieldsSelector.onFieldsChange}
+					onClose={(): void => setIsFieldsSelectorOpen(false)}
+					signal={DataSource.LOGS}
+					requiredFields={LOGS_REQUIRED_COLUMNS}
+				/>
+			)}
 		</div>
 	);
 }

frontend/src/container/LiveLogs/LiveLogsList/index.tsx

@@ -82,7 +82,6 @@ function LiveLogsList({
 	const logsColumns = useLogsTableColumns({
 		fields: selectedFields,
 		fontSize: options.fontSize,
-		appendTo: 'end',
 	});
 
 	const makeOnLogCopy = useCallback(
@@ -198,6 +197,7 @@ function LiveLogsList({
 							ref={ref as React.Ref<TanStackTableHandle>}
 							columns={logsColumns}
 							columnStorageKey={LOCALSTORAGE.LOGS_LIST_COLUMNS}
+							respectColumnOrder={false}
 							onColumnRemove={config?.addColumn?.onRemove}
 							plainTextCellLineClamp={options.maxLines}
 							cellTypographySize={options.fontSize}

frontend/src/container/LogsExplorerList/index.tsx

@@ -105,7 +105,6 @@ function LogsExplorerList({
 	const logsColumns = useLogsTableColumns({
 		fields: selectedFields,
 		fontSize: options.fontSize,
-		appendTo: 'end',
 	});
 
 	const makeOnLogCopy = useCallback(
@@ -204,6 +203,7 @@ function LogsExplorerList({
 					ref={ref as React.Ref<TanStackTableHandle>}
 					columns={logsColumns}
 					columnStorageKey={LOCALSTORAGE.LOGS_LIST_COLUMNS}
+					respectColumnOrder={false}
 					onColumnRemove={config?.addColumn?.onRemove}
 					onColumnOrderChange={handleColumnOrderChange}
 					plainTextCellLineClamp={options.maxLines}

frontend/src/container/LogsExplorerViews/LogsActionsContainer.tsx

@@ -1,11 +1,14 @@
+import { useState } from 'react';
 import { Switch } from '@signozhq/ui/switch';
 import { Typography } from '@signozhq/ui/typography';
 import DownloadOptionsMenu from 'components/DownloadOptionsMenu/DownloadOptionsMenu';
+import FieldsSelector from 'components/FieldsSelector';
 import LogsFormatOptionsMenu from 'components/LogsFormatOptionsMenu/LogsFormatOptionsMenu';
 import ListViewOrderBy from 'components/OrderBy/ListViewOrderBy';
 import { LOCALSTORAGE } from 'constants/localStorage';
 import { PANEL_TYPES } from 'constants/queryBuilder';
 import { useOptionsMenu } from 'container/OptionsMenu';
+import { LOGS_REQUIRED_COLUMNS } from 'container/OptionsMenu/constants';
 import { ArrowUp10, Minus } from '@signozhq/icons';
 import { DataSource, StringOperators } from 'types/common/queryBuilder';
 
@@ -30,6 +33,8 @@ function LogsActionsContainer({
 		aggregateOperator: listQuery?.aggregateOperator || StringOperators.NOOP,
 	});
 
+	const [isFieldsSelectorOpen, setIsFieldsSelectorOpen] = useState(false);
+
 	const formatItems = [
 		{
 			key: 'raw',
@@ -92,12 +97,24 @@ function LogsActionsContainer({
 									items={formatItems}
 									selectedOptionFormat={options.format}
 									config={config}
+									onOpenColumns={(): void => setIsFieldsSelectorOpen(true)}
 								/>
 							</div>
 						</>
 					)}
 				</div>
 			</div>
+			{config.fieldsSelector && (
+				<FieldsSelector
+					isOpen={isFieldsSelectorOpen}
+					title="Edit columns"
+					fields={config.fieldsSelector.value}
+					onFieldsChange={config.fieldsSelector.onFieldsChange}
+					onClose={(): void => setIsFieldsSelectorOpen(false)}
+					signal={DataSource.LOGS}
+					requiredFields={LOGS_REQUIRED_COLUMNS}
+				/>
+			)}
 		</div>
 	);
 }

frontend/src/container/OptionsMenu/__test__/useOptionsMenu.test.ts

@@ -232,4 +232,159 @@ describe('useOptionsMenu', () => {
 		expect(optionNames).toContain('level');
 		expect(optionNames).toContain('timestamp');
 	});
+
+	describe('reorderSelectColumns / handleRemoveSelectedColumn — composite ID lookup', () => {
+		const baseFormatting = {
+			format: 'table',
+			maxLines: 1,
+			fontSize: 'small',
+		};
+
+		// Two entries share the same `name` but differ in `fieldContext` — the
+		// exact case composite IDs are meant to disambiguate.
+		const seedColumns = [
+			{
+				name: 'body',
+				fieldContext: 'log',
+				fieldDataType: 'string',
+				signal: 'logs',
+			},
+			{
+				name: 'service.name',
+				fieldContext: 'resource',
+				fieldDataType: 'string',
+				signal: 'logs',
+			},
+			{
+				name: 'service.name',
+				fieldContext: 'attribute',
+				fieldDataType: 'string',
+				signal: 'logs',
+			},
+			{
+				name: 'timestamp',
+				fieldContext: 'log',
+				fieldDataType: '',
+				signal: 'logs',
+			},
+		];
+
+		beforeEach(() => {
+			(useGetQueryKeySuggestions as jest.Mock).mockReturnValue({
+				data: { data: { data: { keys: {} } } },
+				isFetching: false,
+			});
+			(usePreferenceContext as jest.Mock).mockReturnValue({
+				traces: {
+					preferences: { columns: [], formatting: baseFormatting },
+					updateColumns: mockUpdateColumns,
+					updateFormatting: mockUpdateFormatting,
+				},
+				logs: {
+					preferences: { columns: seedColumns, formatting: baseFormatting },
+					updateColumns: mockUpdateColumns,
+					updateFormatting: mockUpdateFormatting,
+				},
+			});
+		});
+
+		it('reorders by composite IDs — preserves both same-name variants distinctly', () => {
+			const { result } = renderHook(() =>
+				useOptionsMenu({
+					dataSource: DataSource.LOGS,
+					aggregateOperator: 'count',
+				}),
+			);
+
+			// New order: [attribute.service.name, log.body, resource.service.name, log.timestamp]
+			result.current.config.addColumn?.onReorder([
+				'attribute.service.name',
+				'log.body',
+				'resource.service.name',
+				'log.timestamp',
+			]);
+
+			expect(mockUpdateColumns).toHaveBeenCalledTimes(1);
+			const reordered = mockUpdateColumns.mock.calls[0][0];
+			expect(
+				reordered.map(
+					(c: { name: string; fieldContext: string }) =>
+						`${c.fieldContext}.${c.name}`,
+				),
+			).toStrictEqual([
+				'attribute.service.name',
+				'log.body',
+				'resource.service.name',
+				'log.timestamp',
+			]);
+		});
+
+		it('reorder ignores ids that are not columns (e.g. state-indicator) and skips unknown ids', () => {
+			const { result } = renderHook(() =>
+				useOptionsMenu({
+					dataSource: DataSource.LOGS,
+					aggregateOperator: 'count',
+				}),
+			);
+
+			result.current.config.addColumn?.onReorder([
+				'state-indicator',
+				'log.timestamp',
+				'unknown.composite',
+				'log.body',
+				'resource.service.name',
+				'attribute.service.name',
+			]);
+
+			const reordered = mockUpdateColumns.mock.calls[0][0];
+			// state-indicator + unknown.composite filtered; rest preserved in given order.
+			expect(
+				reordered.map(
+					(c: { name: string; fieldContext: string }) =>
+						`${c.fieldContext}.${c.name}`,
+				),
+			).toStrictEqual([
+				'log.timestamp',
+				'log.body',
+				'resource.service.name',
+				'attribute.service.name',
+			]);
+		});
+
+		it('removes only the variant whose composite ID matches', () => {
+			const { result } = renderHook(() =>
+				useOptionsMenu({
+					dataSource: DataSource.LOGS,
+					aggregateOperator: 'count',
+				}),
+			);
+
+			// Removing 'resource.service.name' should drop ONLY the resource variant.
+			result.current.config.addColumn?.onRemove('resource.service.name');
+
+			expect(mockUpdateColumns).toHaveBeenCalledTimes(1);
+			const remaining = mockUpdateColumns.mock.calls[0][0];
+			expect(
+				remaining.map(
+					(c: { name: string; fieldContext: string }) =>
+						`${c.fieldContext}.${c.name}`,
+				),
+			).toStrictEqual(['log.body', 'attribute.service.name', 'log.timestamp']);
+		});
+
+		it('removing by a non-matching composite ID is a no-op (filter returns the full list)', () => {
+			const { result } = renderHook(() =>
+				useOptionsMenu({
+					dataSource: DataSource.LOGS,
+					aggregateOperator: 'count',
+				}),
+			);
+
+			result.current.config.addColumn?.onRemove('unknown.composite');
+
+			const remaining = mockUpdateColumns.mock.calls[0][0];
+			// No entries match → full list preserved.
+			expect(remaining).toHaveLength(seedColumns.length);
+		});
+	});
 });

frontend/src/container/OptionsMenu/constants.ts

@@ -35,7 +35,7 @@ export const defaultLogsSelectedColumns: TelemetryFieldKey[] = [
 	},
 ];
 
-const LOGS_REQUIRED_COLUMNS = ['timestamp', 'body'] as const;
+export const LOGS_REQUIRED_COLUMNS = ['timestamp', 'body'] as const;
 
 /**
  * Always-on invariant: every logs selectColumns array must contain `body` and

frontend/src/container/OptionsMenu/types.ts

@@ -42,4 +42,8 @@ export type OptionsMenuConfig = {
 		onRemove: (key: string) => void;
 		onReorder: (orderedIds: string[]) => void;
 	};
+	fieldsSelector?: {
+		value: TelemetryFieldKey[];
+		onFieldsChange: (next: TelemetryFieldKey[]) => void;
+	};
 };

frontend/src/container/OptionsMenu/useOptionsMenu.ts

@@ -36,7 +36,7 @@ import {
 	OptionsMenuConfig,
 	OptionsQuery,
 } from './types';
-import { getOptionsFromKeys } from './utils';
+import { buildCompositeKey, getOptionsFromKeys } from './utils';
 
 interface UseOptionsMenuProps {
 	storageKey?: string;
@@ -281,7 +281,7 @@ const useOptionsMenu = ({
 	const handleRemoveSelectedColumn = useCallback(
 		(columnKey: string) => {
 			const newSelectedColumns = preferences?.columns?.filter(
-				({ name }) => name !== columnKey,
+				(f) => buildCompositeKey(f.name, f.fieldContext) !== columnKey,
 			);
 
 			if (!newSelectedColumns?.length && dataSource !== DataSource.LOGS) {
@@ -363,9 +363,11 @@ const useOptionsMenu = ({
 	const reorderSelectColumns = useCallback(
 		(orderedIds: string[]): void => {
 			const current = preferences?.columns ?? [];
-			const byName = new Map(current.map((f) => [f.name, f]));
+			const byCompositeKey = new Map(
+				current.map((f) => [buildCompositeKey(f.name, f.fieldContext), f]),
+			);
 			const reordered = orderedIds
-				.map((id) => byName.get(id))
+				.map((id) => byCompositeKey.get(id))
 				.filter((f): f is TelemetryFieldKey => f !== undefined);
 			updateColumns(reordered);
 		},
@@ -396,6 +398,10 @@ const useOptionsMenu = ({
 				onSearch: handleSearchAttribute,
 				onReorder: reorderSelectColumns,
 			},
+			fieldsSelector: {
+				value: preferences?.columns ?? [],
+				onFieldsChange: updateColumns,
+			},
 			format: {
 				value: preferences?.formatting?.format || defaultOptionsQuery.format,
 				onChange: handleFormatChange,
@@ -417,6 +423,7 @@ const useOptionsMenu = ({
 			handleRemoveSelectedColumn,
 			handleSearchAttribute,
 			reorderSelectColumns,
+			updateColumns,
 			handleFormatChange,
 			handleMaxLinesChange,
 			handleFontSizeChange,

frontend/src/container/OptionsMenu/utils.ts

@@ -14,3 +14,9 @@ export const getOptionsFromKeys = (
 		({ value }) => !selectedKeys.find((key) => key === value),
 	);
 };
+
+// Composite identity for a column. Disambiguates same-name fields across
+// different fieldContexts (e.g. resource.service.name vs attribute.service.name).
+// Falls back to bare name when context is missing.
+export const buildCompositeKey = (name: string, context?: string): string =>
+	context ? `${context}.${name}` : name;

frontend/src/container/TracesExplorer/Controls/Controls.module.scss

@@ -0,0 +1,14 @@
+.container {
+	display: flex;
+	align-items: center;
+	justify-content: flex-end;
+	gap: 0.3rem;
+	margin: 8px 0;
+}
+
+.optionsTrigger {
+	display: flex;
+	align-items: center;
+	gap: 4px;
+	cursor: pointer;
+}

frontend/src/container/TracesExplorer/Controls/index.tsx

@@ -1,11 +1,13 @@
-import { memo } from 'react';
-import { OptionFormatTypes } from 'constants/optionsFormatTypes';
+import { memo, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Settings } from '@signozhq/icons';
+import FieldsSelector from 'components/FieldsSelector';
 import Controls, { ControlsProps } from 'container/Controls';
-import OptionsMenu from 'container/OptionsMenu';
 import { OptionsMenuConfig } from 'container/OptionsMenu/types';
 import useQueryPagination from 'hooks/queryPagination/useQueryPagination';
+import { DataSource } from 'types/common/queryBuilder';
 
-import { Container } from './styles';
+import styles from './Controls.module.scss';
 
 function TraceExplorerControls({
 	isLoading,
@@ -14,6 +16,9 @@ function TraceExplorerControls({
 	config,
 	showSizeChanger = true,
 }: TraceExplorerControlsProps): JSX.Element | null {
+	const { t } = useTranslation(['trace']);
+	const [isFieldsSelectorOpen, setIsFieldsSelectorOpen] = useState(false);
+
 	const {
 		pagination,
 		handleCountItemsPerPageChange,
@@ -22,12 +27,25 @@ function TraceExplorerControls({
 	} = useQueryPagination(totalCount, perPageOptions);
 
 	return (
-		<Container>
-			{config && (
-				<OptionsMenu
-					selectedOptionFormat={OptionFormatTypes.LIST} // Defaulting it to List view as options are shown only in the List view tab
-					config={{ addColumn: config?.addColumn }}
-				/>
+		<div className={styles.container}>
+			{config?.fieldsSelector && (
+				<>
+					<div
+						className={styles.optionsTrigger}
+						onClick={(): void => setIsFieldsSelectorOpen(true)}
+					>
+						{t('options_menu.options')}
+						<Settings size="md" />
+					</div>
+					<FieldsSelector
+						isOpen={isFieldsSelectorOpen}
+						title="Edit columns"
+						fields={config.fieldsSelector.value}
+						onFieldsChange={config.fieldsSelector.onFieldsChange}
+						onClose={(): void => setIsFieldsSelectorOpen(false)}
+						signal={DataSource.TRACES}
+					/>
+				</>
 			)}
 
 			<Controls
@@ -41,7 +59,7 @@ function TraceExplorerControls({
 				handleNavigatePrevious={handleNavigatePrevious}
 				showSizeChanger={showSizeChanger}
 			/>
-		</Container>
+		</div>
 	);
 }
 

frontend/src/container/TracesExplorer/Controls/styles.ts

@@ -1,9 +0,0 @@
-import styled from 'styled-components';
-
-export const Container = styled.div`
-	display: flex;
-	align-items: center;
-	justify-content: flex-end;
-	gap: 0.3rem;
-	margin: 8px 0;
-`;

frontend/src/container/TracesExplorer/ListView/index.tsx

@@ -207,8 +207,9 @@ function ListView({
 			const reordered = [...columns];
 			const [moved] = reordered.splice(fromIndex, 1);
 			reordered.splice(toIndex, 0, moved);
+			// `key` is the composite (fieldContext.name) — disambiguates same-name fields.
 			const orderedIds = reordered
-				.map((c) => String(('dataIndex' in c && c.dataIndex) || c.key || ''))
+				.map((c) => String(c.key || ('dataIndex' in c && c.dataIndex) || ''))
 				.filter(Boolean);
 			config?.addColumn?.onReorder(orderedIds);
 		},

frontend/src/container/TracesExplorer/ListView/utils.tsx

@@ -5,6 +5,7 @@ import { Typography } from '@signozhq/ui/typography';
 import { TelemetryFieldKey } from 'api/v5/v5';
 import { DATE_TIME_FORMATS } from 'constants/dateTimeFormats';
 import ROUTES from 'constants/routes';
+import { buildCompositeKey } from 'container/OptionsMenu/utils';
 import { getMs } from 'container/Trace/Filters/Panel/PanelBody/Duration/util';
 import { formUrlParams } from 'container/TraceDetail/utils';
 import { TimestampInput } from 'hooks/useTimezoneFormatter/useTimezoneFormatter';
@@ -83,12 +84,11 @@ export const getListColumns = (
 	const columns: ColumnsType<RowData> =
 		selectedColumns.map((props) => {
 			const name = props?.name || (props as any)?.key;
-			const fieldDataType = props?.fieldDataType || (props as any)?.dataType;
 			const fieldContext = props?.fieldContext || (props as any)?.type;
 			return {
 				title: name,
 				dataIndex: name,
-				key: `${name}-${fieldDataType}-${fieldContext}`,
+				key: buildCompositeKey(name, fieldContext),
 				width: 145,
 				render: (value, item): JSX.Element => {
 					if (value === '') {

frontend/src/utils/__tests__/queryContextUtils.test.ts

@@ -3,6 +3,7 @@ import {
 	extractQueryPairs,
 	getCurrentQueryPair,
 	getCurrentValueIndexAtCursor,
+	getQueryContextAtCursor,
 } from '../queryContextUtils';
 
 describe('extractQueryPairs', () => {
@@ -608,3 +609,42 @@ describe('getCurrentQueryPair', () => {
 		expect(result).toStrictEqual(queryPairs[0]);
 	});
 });
+
+describe('getQueryContextAtCursor - trailing dot in key/value', () => {
+	// A token ending with "." (e.g. "service.") is lexed as FREETEXT rather than
+	// KEY, which previously collapsed the context to "nothing" and made the
+	// suggestion dropdown render empty (appearing closed). These cases lock in
+	// that a partial key/value still resolves to the correct context.
+	it('keeps key context while typing a key that ends with a dot', () => {
+		['service.', 'k8s.', 'attribute.', 'k8s.pod.'].forEach((q) => {
+			const ctx = getQueryContextAtCursor(q, q.length);
+			expect(ctx.isInKey).toBe(true);
+			expect(ctx.isInValue).toBe(false);
+			expect(ctx.isInOperator).toBe(false);
+			expect(ctx.keyToken).toBe(q);
+		});
+	});
+
+	it('treats a new key after a conjunction as key context', () => {
+		const q = 'a = b AND k8s.';
+		const ctx = getQueryContextAtCursor(q, q.length);
+		expect(ctx.isInKey).toBe(true);
+		expect(ctx.isInValue).toBe(false);
+	});
+
+	it('keeps value context while typing a value that ends with a dot', () => {
+		const q = 'service.name = foo.';
+		const ctx = getQueryContextAtCursor(q, q.length);
+		expect(ctx.isInValue).toBe(true);
+		expect(ctx.isInKey).toBe(false);
+		expect(ctx.keyToken).toBe('service.name');
+		expect(ctx.operatorToken).toBe('=');
+	});
+
+	it('still resolves a complete dotted key to key context', () => {
+		const q = 'k8s.namespace';
+		const ctx = getQueryContextAtCursor(q, q.length);
+		expect(ctx.isInKey).toBe(true);
+		expect(ctx.keyToken).toBe('k8s.namespace');
+	});
+});

frontend/src/utils/queryContextUtils.ts

@@ -766,6 +766,58 @@ export function getQueryContextAtCursor(
 			}
 		}
 
+		// A token that ends with a '.' (e.g. "service." or "k8s.pod.") is lexed as
+		// FREETEXT, not KEY, because the KEY rule requires a character after the dot.
+		// While the user is mid-typing such a key/value the token stays FREETEXT, which
+		// is otherwise unclassified and would collapse the context to "nothing" — making
+		// the suggestion dropdown render empty (and appear closed). Classify the partial
+		// token by the slot it occupies so suggestions keep flowing until the next
+		// character turns it back into a valid KEY token.
+		const partialToken = exactToken ?? lastTokenBeforeCursor;
+		if (
+			partialToken &&
+			partialToken.channel === 0 &&
+			partialToken.type === FilterQueryLexer.FREETEXT
+		) {
+			// Closest meaningful token before the partial one determines the slot:
+			// right after an operator we are typing a value, otherwise a key.
+			const prevMeaningful =
+				allTokens
+					.filter(
+						(t) =>
+							t.channel === 0 &&
+							t.type !== FilterQueryLexer.EOF &&
+							t.stop < partialToken.start,
+					)
+					.pop() || null;
+
+			const prevContext = prevMeaningful
+				? determineTokenContext(prevMeaningful, input)
+				: null;
+			const isValuePosition = !!prevContext?.isInOperator;
+
+			return {
+				tokenType: partialToken.type,
+				text: partialToken.text,
+				start: partialToken.start,
+				stop: partialToken.stop,
+				currentToken: partialToken.text,
+				isInKey: !isValuePosition,
+				isInNegation: false,
+				isInOperator: false,
+				isInValue: isValuePosition,
+				isInFunction: false,
+				isInConjunction: false,
+				isInParenthesis: false,
+				isInBracketList: false,
+				keyToken: isValuePosition ? currentPair?.key || '' : partialToken.text,
+				operatorToken: isValuePosition ? currentPair?.operator || '' : undefined,
+				valueToken: isValuePosition ? partialToken.text : undefined,
+				queryPairs,
+				currentPair,
+			};
+		}
+
 		// If we don't have tokens yet, return default context
 		if (!previousToken && !nextToken && !exactToken && !lastTokenBeforeCursor) {
 			return {

pkg/modules/cloudintegration/implcloudintegration/fs/definitions/azure/appservice/icon.svg

@@ -0,0 +1 @@
+<svg id="b70acf0a-34b4-4bdf-9024-7496043ff915" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18 18"><defs><radialGradient id="e2cf8746-c9a8-4eee-86c2-4951983c6032" cx="13428.81" cy="3518.86" r="56.67" gradientTransform="translate(-2005.33 -518.83) scale(0.15)" gradientUnits="userSpaceOnUse"><stop offset="0.18" stop-color="#5ea0ef"/><stop offset="1" stop-color="#0078d4"/></radialGradient><linearGradient id="bdd213dd-d313-473c-8ff4-0133fd3a9033" x1="4.4" y1="11.48" x2="4.37" y2="7.53" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#ccc"/><stop offset="1" stop-color="#fcfcfc"/></linearGradient><linearGradient id="afcc63c5-3649-4476-a742-bcb53a569f3c" x1="10.13" y1="15.45" x2="10.13" y2="11.9" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#ccc"/><stop offset="1" stop-color="#fcfcfc"/></linearGradient><linearGradient id="bd873f0b-9954-4aa5-a3df-9f4c64e8729d" x1="14.18" y1="11.15" x2="14.18" y2="7.38" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#ccc"/><stop offset="1" stop-color="#fcfcfc"/></linearGradient></defs><title>Icon-web-41</title><path id="ee75dd06-1aca-4f76-9d11-d05a284020ad" d="M14.21,15.72A8.5,8.5,0,0,1,3.79,2.28l.09-.06a8.5,8.5,0,0,1,10.33,13.5" fill="url(#e2cf8746-c9a8-4eee-86c2-4951983c6032)"/><path d="M6.69,7.23A13,13,0,0,1,15.6,3.65a8.47,8.47,0,0,0-1.49-1.44,14.34,14.34,0,0,0-4.69,1.1A12.54,12.54,0,0,0,5.34,6.13,2.76,2.76,0,0,1,6.69,7.23Z" fill="#fff" opacity="0.6"/><path d="M2.48,10.65a17.86,17.86,0,0,0-.83,2.62,7.82,7.82,0,0,0,.62.92c.18.23.35.44.55.65A17.94,17.94,0,0,1,3.9,11.37,2.76,2.76,0,0,1,2.48,10.65Z" fill="#fff" opacity="0.6"/><path d="M3.46,6.11a12,12,0,0,1-.69-2.94,8.15,8.15,0,0,0-1.1,1.45A12.69,12.69,0,0,0,2.24,7,2.69,2.69,0,0,1,3.46,6.11Z" fill="#f2f2f2" opacity="0.55"/><circle cx="4.38" cy="8.68" r="2.73" fill="url(#bdd213dd-d313-473c-8ff4-0133fd3a9033)"/><path d="M8.36,13.67A1.77,1.77,0,0,1,8.9,12.4a11.88,11.88,0,0,1-2.53-1.86,2.74,2.74,0,0,1-1.49.83,13.1,13.1,0,0,0,1.45,1.28A12.12,12.12,0,0,0,8.38,13.9,1.79,1.79,0,0,1,8.36,13.67Z" fill="#f2f2f2" opacity="0.55"/><path d="M14.66,13.88a12,12,0,0,1-2.76-.32.41.41,0,0,1,0,.11,1.75,1.75,0,0,1-.51,1.24,13.69,13.69,0,0,0,3.42.24A8.21,8.21,0,0,0,16,13.81,11.5,11.5,0,0,1,14.66,13.88Z" fill="#f2f2f2" opacity="0.55"/><circle cx="10.13" cy="13.67" r="1.78" fill="url(#afcc63c5-3649-4476-a742-bcb53a569f3c)"/><path d="M12.32,8.93a1.83,1.83,0,0,1,.61-1A25.5,25.5,0,0,1,8.47,3.79a16.91,16.91,0,0,1-2-2.92,7.64,7.64,0,0,0-1.09.42A18.14,18.14,0,0,0,7.53,4.47,26.44,26.44,0,0,0,12.32,8.93Z" fill="#f2f2f2" opacity="0.7"/><circle cx="14.18" cy="9.27" r="1.89" fill="url(#bd873f0b-9954-4aa5-a3df-9f4c64e8729d)"/><path d="M17.35,10.54,17,10.37l0,0-.3-.16-.06,0L16.38,10l-.07,0L16,9.8a1.76,1.76,0,0,1-.64.92c.12.08.25.15.38.22l.08.05.35.19,0,0,.86.45h0a8.63,8.63,0,0,0,.29-1.11Z" fill="#f2f2f2" opacity="0.55"/><circle cx="4.38" cy="8.68" r="2.73" fill="url(#bdd213dd-d313-473c-8ff4-0133fd3a9033)"/><circle cx="10.13" cy="13.67" r="1.78" fill="url(#afcc63c5-3649-4476-a742-bcb53a569f3c)"/></svg>

pkg/modules/cloudintegration/implcloudintegration/fs/definitions/azure/appservice/integration.json

@@ -0,0 +1,185 @@
+{
+  "id": "appservice",
+  "title": "App Services",
+  "icon": "file://icon.svg",
+  "overview": "file://overview.md",
+  "supportedSignals": {
+    "metrics": true,
+    "logs": true
+  },
+  "dataCollected": {
+    "metrics": [
+      {
+        "name": "azure_activewebsocketconnections_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_averagewebsocketconnectionduration_average",
+        "unit": "Milliseconds",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_backendhealthpercentage_average",
+        "unit": "Percent",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_backendrequestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_backendrequestlatency_average",
+        "unit": "Milliseconds",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_billableresponsesize_total",
+        "unit": "Bytes",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_bytehitratio_average",
+        "unit": "Percent",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_originhealthpercentage_average",
+        "unit": "Percent",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_originlatency_average",
+        "unit": "Milliseconds",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_originrequestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_originshieldoriginrequestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_originshieldratelimitrequestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_originshieldrequestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_originshieldrequestsize_total",
+        "unit": "Bytes",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_percentage4xx_average",
+        "unit": "Percent",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_percentage5xx_average",
+        "unit": "Percent",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_requestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_requestsize_total",
+        "unit": "Bytes",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_responsesize_total",
+        "unit": "Bytes",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_totallatency_average",
+        "unit": "Milliseconds",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_webapplicationfirewallrequestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_webapplicationfirewallcaptcharequestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_webapplicationfirewalljsrequestcount_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      },
+      {
+        "name": "azure_websocketconnections_total",
+        "unit": "Count",
+        "type": "Gauge",
+        "description": ""
+      }
+    ],
+    "logs": [
+      {
+        "name": "Resource ID",
+        "path": "resources.azure.resource.id",
+        "type": "string"
+      }
+    ]
+  },
+  "telemetryCollectionStrategy": {
+    "azure": {
+      "resourceProvider": "Microsoft.Web",
+      "resourceType": "sites",
+      "metrics": {},
+      "logs": {
+        "categoryGroups": ["allLogs"]
+      }
+    }
+  },
+  "assets": {
+    "dashboards": [
+      {
+        "id": "overview",
+        "title": "App Services Overview",
+        "description": "Overview of App Services metrics",
+        "definition": "file://assets/dashboards/overview.json"
+      }
+    ]
+  }
+}

pkg/modules/cloudintegration/implcloudintegration/fs/definitions/azure/appservice/overview.md

@@ -0,0 +1,5 @@
+### Monitor Azure App Services with SigNoz
+
+Collect key App Services metrics and view them with an out of the box dashboard.
+
+Note: This integration DO NOT collect metrics for any database that was setup with your App Service (if any).

pkg/telemetrylogs/condition_builder.go

@@ -186,7 +186,7 @@ func (c *conditionBuilder) conditionFor(
 		column := columns[0]
 		if len(key.Evolutions) > 0 {
 			// we will use the corresponding column and its evolution entry for the query
-			newColumns, _, err := selectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
+			newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
 			if err != nil {
 				return "", err
 			}

pkg/telemetrylogs/field_mapper.go

@@ -3,11 +3,7 @@ package telemetrylogs
 import (
 	"context"
 	"fmt"
-	"slices"
-	"sort"
-	"strconv"
 	"strings"
-	"time"
 
 	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
 	"github.com/SigNoz/signoz-otel-collector/utils"
@@ -137,113 +133,6 @@ func (m *fieldMapper) getColumn(ctx context.Context, key *telemetrytypes.Telemet
 	return nil, qbtypes.ErrColumnNotFound
 }
 
-// selectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
-// Logic:
-//   - Finds the latest base evolution (<= tsStartTime) across ALL columns
-//   - Rejects all evolutions before this latest base evolution
-//   - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
-//   - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime
-//   - Results are sorted by ReleaseTime descending (newest first)
-func selectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) {
-
-	sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions))
-	copy(sortedEvolutions, evolutions)
-
-	// sort the evolutions by ReleaseTime ascending
-	sort.Slice(sortedEvolutions, func(i, j int) bool {
-		return sortedEvolutions[i].ReleaseTime.Before(sortedEvolutions[j].ReleaseTime)
-	})
-
-	tsStartTime := time.Unix(0, int64(tsStart))
-	tsEndTime := time.Unix(0, int64(tsEnd))
-
-	// Build evolution map: column name -> evolution
-	evolutionMap := make(map[string]*telemetrytypes.EvolutionEntry)
-	for _, evolution := range sortedEvolutions {
-		if _, exists := evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))]; exists {
-			// since if there is duplicate we would just use the oldest one.
-			continue
-		}
-		evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))] = evolution
-	}
-
-	// Find the latest base evolution (<= tsStartTime) across ALL columns
-	// Evolutions are sorted, so we can break early
-	var latestBaseEvolutionAcrossAll *telemetrytypes.EvolutionEntry
-	for _, evolution := range sortedEvolutions {
-		if evolution.ReleaseTime.After(tsStartTime) {
-			break
-		}
-		latestBaseEvolutionAcrossAll = evolution
-	}
-
-	// We shouldn't reach this, it basically means there is something wrong with the evolutions data
-	if latestBaseEvolutionAcrossAll == nil {
-		return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
-	}
-
-	columnLookUpMap := make(map[string]*schema.Column)
-	for _, column := range columns {
-		columnLookUpMap[column.Name] = column
-	}
-
-	// Collect column-evolution pairs
-	type colEvoPair struct {
-		column    *schema.Column
-		evolution *telemetrytypes.EvolutionEntry
-	}
-	pairs := []colEvoPair{}
-
-	for _, evolution := range evolutionMap {
-		// Reject evolutions before the latest base evolution
-		if evolution.ReleaseTime.Before(latestBaseEvolutionAcrossAll.ReleaseTime) {
-			continue
-		}
-		// skip evolutions after tsEndTime
-		if evolution.ReleaseTime.After(tsEndTime) || evolution.ReleaseTime.Equal(tsEndTime) {
-			continue
-		}
-
-		if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
-			return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
-		}
-
-		pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
-	}
-
-	// If no pairs found, fall back to latestBaseEvolutionAcrossAll for matching columns
-	if len(pairs) == 0 {
-		for _, column := range columns {
-			// Use latestBaseEvolutionAcrossAll if this column name matches its column name
-			if column.Name == latestBaseEvolutionAcrossAll.ColumnName {
-				pairs = append(pairs, colEvoPair{column, latestBaseEvolutionAcrossAll})
-			}
-		}
-	}
-
-	// Sort by ReleaseTime descending (newest first)
-	slices.SortFunc(pairs, func(a, b colEvoPair) int {
-		// Sort by ReleaseTime descending (newest first)
-		if a.evolution.ReleaseTime.After(b.evolution.ReleaseTime) {
-			return -1
-		}
-		if a.evolution.ReleaseTime.Before(b.evolution.ReleaseTime) {
-			return 1
-		}
-		return 0
-	})
-
-	// Extract results
-	newColumns := make([]*schema.Column, len(pairs))
-	evolutionsEntries := make([]*telemetrytypes.EvolutionEntry, len(pairs))
-	for i, pair := range pairs {
-		newColumns[i] = pair.column
-		evolutionsEntries[i] = pair.evolution
-	}
-
-	return newColumns, evolutionsEntries, nil
-}
-
 func (m *fieldMapper) FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
 	columns, err := m.getColumn(ctx, key)
 	if err != nil {
@@ -254,7 +143,7 @@ func (m *fieldMapper) FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *
 	var evolutionsEntries []*telemetrytypes.EvolutionEntry
 	if len(key.Evolutions) > 0 {
 		// we will use the corresponding column and its evolution entry for the query
-		newColumns, evolutionsEntries, err = selectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
+		newColumns, evolutionsEntries, err = qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
 		if err != nil {
 			return "", err
 		}

pkg/telemetrylogs/field_mapper_test.go

@@ -536,390 +536,6 @@ func TestFieldForWithEvolutions(t *testing.T) {
 	}
 }
 
-func TestSelectEvolutionsForColumns(t *testing.T) {
-	testCases := []struct {
-		name            string
-		columns         []*schema.Column
-		evolutions      []*telemetrytypes.EvolutionEntry
-		tsStart         uint64
-		tsEnd           uint64
-		expectedColumns []string // column names
-		expectedEvols   []string // evolution column names
-		expectedError   bool
-		errorStr        string
-	}{
-		{
-			name: "New evolutions at tsStartTime - should include latest evolution",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resource"},
-			expectedEvols:   []string{"resource"},
-		},
-		{
-			name: "New evolutions after tsStartTime but less than tsEndTime - should include both",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resource", "resources_string"}, // sorted by ReleaseTime desc
-			expectedEvols:   []string{"resource", "resources_string"},
-		},
-		{
-			name: "Columns without matching evolutions - should exclude them",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],          // no evolution for this
-				logsV2Columns["attributes_string"], // no evolution for this
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string"},
-			expectedEvols:   []string{"resources_string"},
-		},
-		{
-			name: "New evolutions at tsEndTime - should not include new evolution",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string"},
-			expectedEvols:   []string{"resources_string"},
-		},
-		{
-			name: "New evolutions after tsEndTime - should exclude new",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string"},
-			expectedEvols:   []string{"resources_string"},
-		},
-		{
-			name:    "Empty columns array",
-			columns: []*schema.Column{},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{},
-			expectedEvols:   []string{},
-			expectedError:   true,
-			errorStr:        "column resources_string not found",
-		},
-		{
-			name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
-			columns: []*schema.Column{
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resource"},
-			expectedEvols:   []string{"resource"}, // should use first one (older)
-		},
-		{
-			name: "Genuine Duplicate evolutions with new version- should consider both",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      0,
-					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      1,
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					Version:      2,
-					ReleaseTime:  time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string", "resource"},
-			expectedEvols:   []string{"resources_string", "resource"}, // should use first one (older)
-		},
-		{
-			name: "Evolution exactly at tsEndTime",
-			columns: []*schema.Column{
-				logsV2Columns["resources_string"],
-				logsV2Columns["resource"],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resources_string",
-					ColumnType:   "Map(LowCardinality(String), String)",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   "resource",
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextResource,
-					FieldName:    "__all__",
-					ReleaseTime:  time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), // exactly at tsEnd
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{"resources_string"}, // resource excluded because After(tsEnd) is true
-			expectedEvols:   []string{"resources_string"},
-		},
-		{
-			name: "Single evolution after tsStartTime - JSON body",
-			columns: []*schema.Column{
-				logsV2Columns[LogsV2BodyV2Column],
-				logsV2Columns[LogsV2BodyPromotedColumn],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   LogsV2BodyV2Column,
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextBody,
-					FieldName:    "__all__",
-					ReleaseTime:  time.Unix(0, 0),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   LogsV2BodyPromotedColumn,
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextBody,
-					FieldName:    "user.name",
-					ReleaseTime:  time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column}, // sorted by ReleaseTime desc (newest first)
-			expectedEvols:   []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column},
-		},
-		{
-			name: "No evolution after tsStartTime - JSON body",
-			columns: []*schema.Column{
-				logsV2Columns[LogsV2BodyV2Column],
-				logsV2Columns[LogsV2BodyPromotedColumn],
-			},
-			evolutions: []*telemetrytypes.EvolutionEntry{
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   LogsV2BodyV2Column,
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextBody,
-					FieldName:    "__all__",
-					ReleaseTime:  time.Unix(0, 0),
-				},
-				{
-					Signal:       telemetrytypes.SignalLogs,
-					ColumnName:   LogsV2BodyPromotedColumn,
-					ColumnType:   "JSON()",
-					FieldContext: telemetrytypes.FieldContextBody,
-					FieldName:    "user.name",
-					ReleaseTime:  time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
-				},
-			},
-			tsStart:         uint64(time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC).UnixNano()),
-			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
-			expectedColumns: []string{LogsV2BodyPromotedColumn},
-			expectedEvols:   []string{LogsV2BodyPromotedColumn},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			resultColumns, resultEvols, err := selectEvolutionsForColumns(tc.columns, tc.evolutions, tc.tsStart, tc.tsEnd)
-
-			if tc.expectedError {
-				assert.Contains(t, err.Error(), tc.errorStr)
-			} else {
-				require.NoError(t, err)
-				assert.Equal(t, len(tc.expectedColumns), len(resultColumns), "column count mismatch")
-				assert.Equal(t, len(tc.expectedEvols), len(resultEvols), "evolution count mismatch")
-
-				resultColumnNames := make([]string, len(resultColumns))
-				for i, col := range resultColumns {
-					resultColumnNames[i] = col.Name
-				}
-				resultEvolNames := make([]string, len(resultEvols))
-				for i, evol := range resultEvols {
-					resultEvolNames[i] = evol.ColumnName
-				}
-
-				for i := range tc.expectedColumns {
-					assert.Equal(t, resultColumnNames[i], tc.expectedColumns[i], "expected column missing: "+tc.expectedColumns[i])
-				}
-				for i := range tc.expectedEvols {
-					assert.Equal(t, resultEvolNames[i], tc.expectedEvols[i], "expected evolution missing: "+tc.expectedEvols[i])
-				}
-				// Verify sorting: should be descending by ReleaseTime
-				for i := 0; i < len(resultEvols)-1; i++ {
-					assert.True(t, !resultEvols[i].ReleaseTime.Before(resultEvols[i+1].ReleaseTime),
-						"evolutions should be sorted descending by ReleaseTime")
-				}
-			}
-		})
-	}
-}
-
 func TestFieldForWithMaterialized(t *testing.T) {
 	ctx := context.Background()
 

pkg/telemetrymetadata/metadata.go

@@ -344,6 +344,11 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
 			})
 		}
 	}
+
+	if err = t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
+		return nil, false, err
+	}
+
 	return keys, complete, nil
 }
 
@@ -689,7 +694,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
 		}
 	}
 
-	if _, err := t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
+	if err := t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
 		return nil, false, err
 	}
 
@@ -2370,8 +2375,8 @@ func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Conte
 	return entries, nil
 }
 
-// Get retrieves all evolutions for the given selectors from DB.
-func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) (map[string][]*telemetrytypes.EvolutionEntry, error) {
+// updateColumnEvolutionMetadataForKeys updates the evolution field for keys.
+func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) error {
 
 	var metadataKeySelectors []*telemetrytypes.EvolutionSelector
 	for _, keySelector := range keysToUpdate {
@@ -2385,7 +2390,7 @@ func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Co
 
 	evolutions, err := k.fetchEvolutionEntryFromClickHouse(ctx, metadataKeySelectors)
 	if err != nil {
-		return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to fetch evolution from clickhouse %s", err.Error())
+		return errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to fetch evolution from clickhouse %s", err.Error())
 	}
 
 	evolutionsByUniqueKey := make(map[string][]*telemetrytypes.EvolutionEntry)
@@ -2416,7 +2421,7 @@ func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Co
 			}
 		}
 	}
-	return evolutionsByUniqueKey, nil
+	return nil
 }
 
 // chunkSizeFirstSeenMetricMetadata limits the number of tuples per SQL query to avoid hitting the max_query_size limit.

pkg/telemetrymetadata/metadata_query_test.go

@@ -2,8 +2,12 @@ package telemetrymetadata
 
 import (
 	"context"
+	"regexp"
 	"testing"
 
+	cmock "github.com/SigNoz/clickhouse-go-mock"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
 	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/telemetryaudit"
 	"github.com/SigNoz/signoz/pkg/telemetrylogs"
@@ -13,12 +17,24 @@ import (
 	"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
 	"github.com/SigNoz/signoz/pkg/telemetrytraces"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-	cmock "github.com/SigNoz/clickhouse-go-mock"
 	"github.com/stretchr/testify/assert"
-	"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
 	"github.com/stretchr/testify/require"
 )
 
+type regexMatcher struct {
+}
+
+func (m *regexMatcher) Match(expectedSQL, actualSQL string) error {
+	re, err := regexp.Compile(expectedSQL)
+	if err != nil {
+		return err
+	}
+	if !re.MatchString(actualSQL) {
+		return errors.NewInvalidInputf(errors.CodeInvalidInput, "expected query to contain %s, got %s", expectedSQL, actualSQL)
+	}
+	return nil
+}
+
 func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
 	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
 	mock := mockTelemetryStore.Mock()

pkg/telemetrymetadata/metadata_test.go

@@ -1,582 +0,0 @@
-package telemetrymetadata
-
-import (
-	"context"
-	"regexp"
-	"testing"
-
-	"github.com/SigNoz/signoz/pkg/errors"
-	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
-	"github.com/SigNoz/signoz/pkg/telemetryaudit"
-	"github.com/SigNoz/signoz/pkg/telemetrylogs"
-	"github.com/SigNoz/signoz/pkg/telemetrymeter"
-	"github.com/SigNoz/signoz/pkg/telemetrymetrics"
-	"github.com/SigNoz/signoz/pkg/telemetrystore"
-	"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
-	"github.com/SigNoz/signoz/pkg/telemetrytraces"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
-	cmock "github.com/SigNoz/clickhouse-go-mock"
-	"github.com/stretchr/testify/assert"
-	"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
-	"github.com/stretchr/testify/require"
-)
-
-func newTestTelemetryMetaStoreTestHelper(t *testing.T, store telemetrystore.TelemetryStore) telemetrytypes.MetadataStore {
-	t.Helper()
-	return NewTelemetryMetaStore(
-		instrumentationtest.New().ToProviderSettings(),
-		store,
-		telemetrytraces.DBName,
-		telemetrytraces.TagAttributesV2TableName,
-		telemetrytraces.SpanAttributesKeysTblName,
-		telemetrytraces.SpanIndexV3TableName,
-		telemetrymetrics.DBName,
-		telemetrymetrics.AttributesMetadataTableName,
-		telemetrymeter.DBName,
-		telemetrymeter.SamplesAgg1dTableName,
-		telemetrylogs.DBName,
-		telemetrylogs.LogsV2TableName,
-		telemetrylogs.TagAttributesV2TableName,
-		telemetrylogs.LogAttributeKeysTblName,
-		telemetrylogs.LogResourceKeysTblName,
-		telemetryaudit.DBName,
-		telemetryaudit.AuditLogsTableName,
-		telemetryaudit.TagAttributesTableName,
-		telemetryaudit.LogAttributeKeysTblName,
-		telemetryaudit.LogResourceKeysTblName,
-		DBName,
-		AttributesMetadataLocalTableName,
-		ColumnEvolutionMetadataTableName,
-		flaggertest.New(t),
-	)
-}
-
-type regexMatcher struct {
-}
-
-func (m *regexMatcher) Match(expectedSQL, actualSQL string) error {
-	re, err := regexp.Compile(expectedSQL)
-	if err != nil {
-		return err
-	}
-	if !re.MatchString(actualSQL) {
-		return errors.NewInvalidInputf(errors.CodeInvalidInput, "expected query to contain %s, got %s", expectedSQL, actualSQL)
-	}
-	return nil
-}
-
-func TestGetKeys(t *testing.T) {
-	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-	mock := mockTelemetryStore.Mock()
-
-	metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-	rows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "statement", Type: "String"},
-	}, [][]any{{"CREATE TABLE signoz_traces.signoz_index_v3"}})
-
-	mock.
-		ExpectSelect("SHOW CREATE TABLE signoz_traces.distributed_signoz_index_v3").
-		WillReturnRows(rows)
-
-	query := `SELECT.*`
-
-	mock.ExpectQuery(query).
-		WithArgs("%http.method%", telemetrytypes.FieldDataTypeString.TagDataType(), 11).
-		WillReturnRows(cmock.NewRows([]cmock.ColumnType{
-			{Name: "tag_key", Type: "String"},
-			{Name: "tag_type", Type: "String"},
-			{Name: "tag_data_type", Type: "String"},
-			{Name: "priority", Type: "UInt8"},
-		}, [][]any{{"http.method", "tag", "String", 1}, {"http.method", "tag", "String", 1}}))
-	keys, _, err := metadata.GetKeys(context.Background(), &telemetrytypes.FieldKeySelector{
-		Signal:        telemetrytypes.SignalTraces,
-		FieldContext:  telemetrytypes.FieldContextSpan,
-		FieldDataType: telemetrytypes.FieldDataTypeString,
-		Name:          "http.method",
-		Limit:         10,
-	})
-
-	if err != nil {
-		t.Fatalf("Failed to get keys: %v", err)
-	}
-
-	t.Logf("Keys: %v", keys)
-}
-
-func TestApplyBackwardCompatibleKeys(t *testing.T) {
-	tests := []struct {
-		name           string
-		inputKeys      []*telemetrytypes.TelemetryFieldKey
-		expectedKeys   []string
-		notExpectedKey string
-	}{
-		{
-			name: "bidirectional mapping: net.peer.name -> server.address",
-			inputKeys: []*telemetrytypes.TelemetryFieldKey{
-				{
-					Name:          "net.peer.name",
-					Signal:        telemetrytypes.SignalTraces,
-					FieldContext:  telemetrytypes.FieldContextAttribute,
-					FieldDataType: telemetrytypes.FieldDataTypeString,
-				},
-			},
-			expectedKeys: []string{"net.peer.name", "server.address"},
-		},
-		{
-			name: "bidirectional mapping: server.address -> net.peer.name",
-			inputKeys: []*telemetrytypes.TelemetryFieldKey{
-				{
-					Name:          "server.address",
-					Signal:        telemetrytypes.SignalTraces,
-					FieldContext:  telemetrytypes.FieldContextResource,
-					FieldDataType: telemetrytypes.FieldDataTypeString,
-				},
-			},
-			expectedKeys: []string{"server.address", "net.peer.name"},
-		},
-		{
-			name: "bidirectional mapping: http.url -> url.full",
-			inputKeys: []*telemetrytypes.TelemetryFieldKey{
-				{
-					Name:          "http.url",
-					Signal:        telemetrytypes.SignalTraces,
-					FieldContext:  telemetrytypes.FieldContextAttribute,
-					FieldDataType: telemetrytypes.FieldDataTypeString,
-				},
-			},
-			expectedKeys: []string{"http.url", "url.full"},
-		},
-		{
-			name: "bidirectional mapping: url.full -> http.url",
-			inputKeys: []*telemetrytypes.TelemetryFieldKey{
-				{
-					Name:          "url.full",
-					Signal:        telemetrytypes.SignalTraces,
-					FieldContext:  telemetrytypes.FieldContextAttribute,
-					FieldDataType: telemetrytypes.FieldDataTypeString,
-				},
-			},
-			expectedKeys: []string{"url.full", "http.url"},
-		},
-		{
-			name: "key without alias",
-			inputKeys: []*telemetrytypes.TelemetryFieldKey{
-				{
-					Name:          "custom.attribute",
-					Signal:        telemetrytypes.SignalTraces,
-					FieldContext:  telemetrytypes.FieldContextAttribute,
-					FieldDataType: telemetrytypes.FieldDataTypeString,
-				},
-			},
-			expectedKeys:   []string{"custom.attribute"},
-			notExpectedKey: "server.address",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-			mock := mockTelemetryStore.Mock()
-
-			metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-			hasTraces := false
-			hasLogs := false
-			for _, key := range tt.inputKeys {
-				switch key.Signal {
-				case telemetrytypes.SignalTraces:
-					hasTraces = true
-				case telemetrytypes.SignalLogs:
-					hasLogs = true
-				}
-			}
-
-			if hasTraces {
-				mock.ExpectSelect("SHOW CREATE TABLE signoz_traces.distributed_signoz_index_v3").
-					WillReturnRows(cmock.NewRows([]cmock.ColumnType{
-						{Name: "statement", Type: "String"},
-					}, [][]any{{"CREATE TABLE signoz_traces.signoz_index_v3"}}))
-
-				var args []interface{}
-				var rows [][]any
-				for _, key := range tt.inputKeys {
-					if key.Signal == telemetrytypes.SignalTraces {
-						tagType := "tag"
-						if key.FieldContext == telemetrytypes.FieldContextResource {
-							tagType = "resource"
-						}
-						args = append(args, "%"+key.Name+"%", tagType, key.FieldDataType.TagDataType())
-						rows = append(rows, []any{key.Name, tagType, key.FieldDataType.TagDataType(), 1})
-					}
-				}
-				args = append(args, 11)
-
-				mock.ExpectQuery(`SELECT.*`).
-					WithArgs(args...).
-					WillReturnRows(cmock.NewRows([]cmock.ColumnType{
-						{Name: "tag_key", Type: "String"},
-						{Name: "tag_type", Type: "String"},
-						{Name: "tag_data_type", Type: "String"},
-						{Name: "priority", Type: "UInt8"},
-					}, rows))
-			}
-
-			if hasLogs {
-				var args []interface{}
-				var rows [][]any
-				for _, key := range tt.inputKeys {
-					if key.Signal == telemetrytypes.SignalLogs {
-						tagType := "tag"
-						if key.FieldContext == telemetrytypes.FieldContextResource {
-							tagType = "resource"
-						}
-						args = append(args, "%"+key.Name+"%", tagType, key.FieldDataType.TagDataType())
-						rows = append(rows, []any{key.Name, tagType, key.FieldDataType.TagDataType(), 1})
-					}
-				}
-				args = append(args, 11)
-
-				mock.ExpectQuery(`SELECT.*`).
-					WithArgs(args...).
-					WillReturnRows(cmock.NewRows([]cmock.ColumnType{
-						{Name: "tag_key", Type: "String"},
-						{Name: "tag_type", Type: "String"},
-						{Name: "tag_data_type", Type: "String"},
-						{Name: "priority", Type: "UInt8"},
-					}, rows))
-			}
-
-			selectors := []*telemetrytypes.FieldKeySelector{}
-			for _, key := range tt.inputKeys {
-				selectors = append(selectors, &telemetrytypes.FieldKeySelector{
-					Signal:        key.Signal,
-					FieldContext:  key.FieldContext,
-					FieldDataType: key.FieldDataType,
-					Name:          key.Name,
-					Limit:         10,
-				})
-			}
-
-			resultMap, _, err := metadata.GetKeysMulti(context.Background(), selectors)
-			require.NoError(t, err, "GetKeysMulti should not return an error")
-
-			for _, expectedKey := range tt.expectedKeys {
-				assert.Contains(t, resultMap, expectedKey, "Expected key %q to exist in result map", expectedKey)
-			}
-
-			if tt.notExpectedKey != "" {
-				assert.NotContains(t, resultMap, tt.notExpectedKey, "Did not expect key %q to exist in result map", tt.notExpectedKey)
-			}
-
-			for _, srcKey := range tt.inputKeys {
-				backwardCompatKeys := GetBackwardCompatKeysForSignal(srcKey.Signal)
-				if aliasKey, ok := backwardCompatKeys[srcKey.Name]; ok {
-					aliasExistedInInput := false
-					for _, inputKey := range tt.inputKeys {
-						if inputKey.Name == aliasKey {
-							aliasExistedInInput = true
-							break
-						}
-					}
-
-					if !aliasExistedInInput {
-						if aliasEntries, exists := resultMap[aliasKey]; exists && len(aliasEntries) > 0 {
-							aliasEntry := aliasEntries[0]
-							assert.Equal(t, srcKey.Signal, aliasEntry.Signal, "Alias %q should have same signal", aliasKey)
-							assert.Equal(t, srcKey.FieldContext, aliasEntry.FieldContext, "Alias %q should have same field context", aliasKey)
-							assert.Equal(t, srcKey.FieldDataType, aliasEntry.FieldDataType, "Alias %q should have same field data type", aliasKey)
-						}
-					}
-				}
-			}
-
-			require.NoError(t, mock.ExpectationsWereMet(), "All SQL expectations should be met")
-		})
-	}
-}
-
-func TestEnrichWithIntrinsicMetricKeys(t *testing.T) {
-	result := enrichWithIntrinsicMetricKeys(
-		map[string][]*telemetrytypes.TelemetryFieldKey{},
-		[]*telemetrytypes.FieldKeySelector{
-			{
-				Signal:            telemetrytypes.SignalMetrics,
-				Name:              "metric",
-				SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
-			},
-		},
-	)
-
-	require.Contains(t, result, "metric_name")
-	assert.Equal(t, telemetrytypes.FieldContextMetric, result["metric_name"][0].FieldContext)
-
-	result = enrichWithIntrinsicMetricKeys(
-		map[string][]*telemetrytypes.TelemetryFieldKey{},
-		[]*telemetrytypes.FieldKeySelector{
-			{
-				Signal:            telemetrytypes.SignalMetrics,
-				Name:              "metric",
-				FieldContext:      telemetrytypes.FieldContextAttribute,
-				SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
-			},
-		},
-	)
-	assert.NotContains(t, result, "metric_name")
-
-	result = enrichWithIntrinsicMetricKeys(
-		map[string][]*telemetrytypes.TelemetryFieldKey{},
-		[]*telemetrytypes.FieldKeySelector{
-			{
-				Signal: telemetrytypes.SignalMetrics,
-				MetricContext: &telemetrytypes.MetricContext{
-					MetricNamespace: "system.cpu",
-				},
-				SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
-			},
-		},
-	)
-	assert.NotContains(t, result, "metric_name")
-}
-
-func TestGetMetricFieldValuesIntrinsicMetricName(t *testing.T) {
-	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-	mock := mockTelemetryStore.Mock()
-
-	metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-	valueRows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "metric_name", Type: "String"},
-	}, [][]any{{"metric.a"}, {"metric.b"}})
-
-	query := `SELECT .*metric_name.*` + telemetrymetrics.TimeseriesV41weekTableName + `.*GROUP BY.*metric_name`
-
-	mock.ExpectQuery(query).
-		WithArgs(51).
-		WillReturnRows(valueRows)
-
-	metadataRows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "attr_string_value", Type: "String"},
-	}, [][]any{})
-
-	mock.ExpectQuery(regexp.QuoteMeta("SELECT DISTINCT attr_string_value FROM signoz_metrics.distributed_metadata WHERE attr_name = ? LIMIT ?")).
-		WithArgs("metric_name", 49).
-		WillReturnRows(metadataRows)
-
-	values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
-		FieldKeySelector: &telemetrytypes.FieldKeySelector{
-			Signal:            telemetrytypes.SignalMetrics,
-			Name:              "metric_name",
-			Limit:             50,
-			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
-		},
-		Limit: 50,
-	})
-	require.NoError(t, err)
-	assert.True(t, complete)
-	assert.ElementsMatch(t, []string{"metric.a", "metric.b"}, values.StringValues)
-	require.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestGetMetricFieldValuesIntrinsicBoolReturnsEmpty(t *testing.T) {
-	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-	mock := mockTelemetryStore.Mock()
-
-	metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-	metadataRows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "attr_string_value", Type: "String"},
-	}, [][]any{})
-
-	mock.ExpectQuery(regexp.QuoteMeta("SELECT DISTINCT attr_string_value FROM signoz_metrics.distributed_metadata WHERE attr_name = ? AND attr_datatype = ? AND attr_string_value = ? LIMIT ?")).
-		WithArgs("is_monotonic", telemetrytypes.FieldDataTypeBool.TagDataType(), "true", 11).
-		WillReturnRows(metadataRows)
-
-	values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
-		FieldKeySelector: &telemetrytypes.FieldKeySelector{
-			Signal:            telemetrytypes.SignalMetrics,
-			Name:              "is_monotonic",
-			FieldDataType:     telemetrytypes.FieldDataTypeBool,
-			Limit:             10,
-			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
-		},
-		Value: "true",
-		Limit: 10,
-	})
-	require.NoError(t, err)
-	assert.True(t, complete)
-	assert.Empty(t, values.StringValues)
-	assert.Empty(t, values.BoolValues)
-	require.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestGetMetricFieldValuesAppliesMetricNamespace(t *testing.T) {
-	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-	mock := mockTelemetryStore.Mock()
-
-	metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-	valueRows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "attr_string_value", Type: "String"},
-	}, [][]any{{"value.a"}})
-
-	mock.ExpectQuery(regexp.QuoteMeta("SELECT DISTINCT attr_string_value FROM signoz_metrics.distributed_metadata WHERE attr_name = ? AND metric_name LIKE ? LIMIT ?")).
-		WithArgs("custom_key", "system.cpu%", 11).
-		WillReturnRows(valueRows)
-
-	values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
-		FieldKeySelector: &telemetrytypes.FieldKeySelector{
-			Signal:            telemetrytypes.SignalMetrics,
-			Name:              "custom_key",
-			Limit:             10,
-			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
-			MetricContext: &telemetrytypes.MetricContext{
-				MetricNamespace: "system.cpu",
-			},
-		},
-		Limit: 10,
-	})
-	require.NoError(t, err)
-	assert.True(t, complete)
-	assert.ElementsMatch(t, []string{"value.a"}, values.StringValues)
-	require.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestGetMetricFieldValuesIntrinsicMetricNameAppliesMetricNamespace(t *testing.T) {
-	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-	mock := mockTelemetryStore.Mock()
-
-	metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-	valueRows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "metric_name", Type: "String"},
-	}, [][]any{{"system.cpu.utilization"}})
-
-	mock.ExpectQuery(regexp.QuoteMeta("SELECT metric_name FROM signoz_metrics.distributed_time_series_v4_1week WHERE metric_name LIKE ? GROUP BY metric_name LIMIT ?")).
-		WithArgs("system.cpu%", 51).
-		WillReturnRows(valueRows)
-
-	metadataRows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "attr_string_value", Type: "String"},
-	}, [][]any{})
-
-	mock.ExpectQuery(regexp.QuoteMeta("SELECT DISTINCT attr_string_value FROM signoz_metrics.distributed_metadata WHERE attr_name = ? AND metric_name LIKE ? LIMIT ?")).
-		WithArgs("metric_name", "system.cpu%", 50).
-		WillReturnRows(metadataRows)
-
-	values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
-		FieldKeySelector: &telemetrytypes.FieldKeySelector{
-			Signal:            telemetrytypes.SignalMetrics,
-			Name:              "metric_name",
-			Limit:             50,
-			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
-			MetricContext: &telemetrytypes.MetricContext{
-				MetricNamespace: "system.cpu",
-			},
-		},
-		Limit: 50,
-	})
-	require.NoError(t, err)
-	assert.True(t, complete)
-	assert.ElementsMatch(t, []string{"system.cpu.utilization"}, values.StringValues)
-	require.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestGetMeterSourceMetricFieldValuesAppliesMetricNamespace(t *testing.T) {
-	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-	mock := mockTelemetryStore.Mock()
-
-	metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-	rows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "attr", Type: "Array(String)"},
-	}, [][]any{{[]string{"service.name", "frontend"}}})
-
-	mock.ExpectQuery(`SELECT .*distributed_samples_agg_1d.*metric_name LIKE .*`).
-		WithArgs("service.name", "\\_\\_%", "system.cpu%", "", 11).
-		WillReturnRows(rows)
-
-	values, complete, err := metadata.(*telemetryMetaStore).getMeterSourceMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
-		FieldKeySelector: &telemetrytypes.FieldKeySelector{
-			Signal: telemetrytypes.SignalMetrics,
-			Source: telemetrytypes.SourceMeter,
-			Name:   "service.name",
-			MetricContext: &telemetrytypes.MetricContext{
-				MetricNamespace: "system.cpu",
-			},
-		},
-		Limit: 10,
-	})
-	require.NoError(t, err)
-	assert.True(t, complete)
-	assert.ElementsMatch(t, []string{"frontend"}, values.StringValues)
-	require.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestGetMetricsKeysAppliesMetricNamespace(t *testing.T) {
-	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-	mock := mockTelemetryStore.Mock()
-
-	metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-	rows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "name", Type: "String"},
-		{Name: "field_context", Type: "String"},
-		{Name: "field_data_type", Type: "String"},
-		{Name: "priority", Type: "UInt8"},
-	}, [][]any{{"service.name", "resource", "String", 1}})
-
-	mock.ExpectQuery(`(?s)SELECT.*distributed_metadata.*metric_name LIKE.*`).
-		WithArgs("%service%", "\\_\\_%", "system.cpu%", 11).
-		WillReturnRows(rows)
-
-	keys, complete, err := metadata.(*telemetryMetaStore).getMetricsKeys(context.Background(), []*telemetrytypes.FieldKeySelector{
-		{
-			Signal:            telemetrytypes.SignalMetrics,
-			Name:              "service",
-			Limit:             10,
-			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
-			MetricContext: &telemetrytypes.MetricContext{
-				MetricNamespace: "system.cpu",
-			},
-		},
-	})
-	require.NoError(t, err)
-	assert.True(t, complete)
-	assert.Len(t, keys, 1)
-	assert.Equal(t, "service.name", keys[0].Name)
-	require.NoError(t, mock.ExpectationsWereMet())
-}
-
-func TestGetMeterSourceMetricKeysAppliesMetricNamespace(t *testing.T) {
-	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
-	mock := mockTelemetryStore.Mock()
-
-	metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
-
-	rows := cmock.NewRows([]cmock.ColumnType{
-		{Name: "attr_name", Type: "String"},
-	}, [][]any{{"service.name"}})
-
-	mock.ExpectQuery(`SELECT.*distributed_samples_agg_1d.*metric_name LIKE.*`).
-		WithArgs("%service%", "\\_\\_%", "system.cpu%", 10).
-		WillReturnRows(rows)
-
-	keys, complete, err := metadata.(*telemetryMetaStore).getMeterSourceMetricKeys(context.Background(), []*telemetrytypes.FieldKeySelector{
-		{
-			Signal:            telemetrytypes.SignalMetrics,
-			Source:            telemetrytypes.SourceMeter,
-			Name:              "service",
-			Limit:             10,
-			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
-			MetricContext: &telemetrytypes.MetricContext{
-				MetricNamespace: "system.cpu",
-			},
-		},
-	})
-	require.NoError(t, err)
-	assert.True(t, complete)
-	assert.Len(t, keys, 1)
-	assert.Equal(t, "service.name", keys[0].Name)
-	require.NoError(t, mock.ExpectationsWereMet())
-}

pkg/telemetrytraces/condition_builder.go

@@ -161,7 +161,33 @@ func (c *conditionBuilder) conditionFor(
 	case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
 
 		var value any
-		switch columns[0].Type.GetType() {
+		column := columns[0]
+		if len(key.Evolutions) > 0 {
+			// we will use the corresponding column and its evolution entry for the query
+			newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
+			if err != nil {
+				return "", err
+			}
+
+			if len(newColumns) == 0 {
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no valid evolution found for field %s in the given time range", key.Name)
+			}
+
+			// Multiple columns means fieldExpression is a multiIf returning NULL when none match,
+			// so a simple null check is sufficient.
+			if len(newColumns) > 1 {
+				if operator == qbtypes.FilterOperatorExists {
+					return sb.IsNotNull(fieldExpression), nil
+				} else {
+					return sb.IsNull(fieldExpression), nil
+				}
+			}
+
+			// otherwise we have to find the correct exist operator based on the column type
+			column = newColumns[0]
+		}
+
+		switch column.Type.GetType() {
 		case schema.ColumnTypeEnumJSON:
 			if operator == qbtypes.FilterOperatorExists {
 				return sb.IsNotNull(fieldExpression), nil
@@ -178,7 +204,7 @@ func (c *conditionBuilder) conditionFor(
 				return sb.E(fieldExpression, value), nil
 			}
 		case schema.ColumnTypeEnumLowCardinality:
-			switch elementType := columns[0].Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
+			switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
 			case schema.ColumnTypeEnumString:
 				value = ""
 				if operator == qbtypes.FilterOperatorExists {
@@ -202,14 +228,14 @@ func (c *conditionBuilder) conditionFor(
 				return sb.E(fieldExpression, value), nil
 			}
 		case schema.ColumnTypeEnumMap:
-			keyType := columns[0].Type.(schema.MapColumnType).KeyType
+			keyType := column.Type.(schema.MapColumnType).KeyType
 			if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
-				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, columns[0].Type)
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
 			}
 
-			switch valueType := columns[0].Type.(schema.MapColumnType).ValueType; valueType.GetType() {
+			switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
 			case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
-				leftOperand := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
+				leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
 				if key.Materialized {
 					leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
 				}
@@ -222,7 +248,7 @@ func (c *conditionBuilder) conditionFor(
 				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
 			}
 		default:
-			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", columns[0].Type)
+			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type)
 		}
 	}
 	return "", nil

pkg/telemetrytraces/condition_builder_test.go

@@ -3,6 +3,7 @@ package telemetrytraces
 import (
 	"context"
 	"testing"
+	"time"
 
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -14,6 +15,7 @@ import (
 func TestConditionFor(t *testing.T) {
 	ctx := context.Background()
 
+	mockEvolution := mockEvolutionData(time.Date(2025, 10, 26, 0, 10, 0, 0, time.UTC))
 	testCases := []struct {
 		name          string
 		key           telemetrytypes.TelemetryFieldKey
@@ -213,6 +215,7 @@ func TestConditionFor(t *testing.T) {
 				Name:          "service.name",
 				FieldContext:  telemetrytypes.FieldContextResource,
 				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    mockEvolution,
 			},
 			operator:      qbtypes.FilterOperatorExists,
 			value:         nil,
@@ -225,6 +228,7 @@ func TestConditionFor(t *testing.T) {
 				Name:          "service.name",
 				FieldContext:  telemetrytypes.FieldContextResource,
 				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    mockEvolution,
 			},
 			operator:      qbtypes.FilterOperatorNotExists,
 			value:         nil,
@@ -302,3 +306,85 @@ func TestConditionFor(t *testing.T) {
 		})
 	}
 }
+
+func TestConditionForResourceWithEvolution(t *testing.T) {
+	ctx := context.Background()
+	releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
+	evolutions := mockEvolutionData(releaseTime)
+
+	testCases := []struct {
+		name        string
+		key         telemetrytypes.TelemetryFieldKey
+		operator    qbtypes.FilterOperator
+		tsStart     uint64
+		tsEnd       uint64
+		expectedSQL string
+	}{
+		{
+			name: "Exists - window after release - JSON only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    evolutions,
+			},
+			operator:    qbtypes.FilterOperatorExists,
+			tsStart:     uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:       uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedSQL: "WHERE resource.`service.name`::String IS NOT NULL",
+		},
+		{
+			name: "NotExists - window after release - JSON only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    evolutions,
+			},
+			operator:    qbtypes.FilterOperatorNotExists,
+			tsStart:     uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:       uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedSQL: "WHERE resource.`service.name`::String IS NULL",
+		},
+		{
+			name: "Exists - window before release - map only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    evolutions,
+			},
+			operator:    qbtypes.FilterOperatorExists,
+			tsStart:     uint64(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:       uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedSQL: "WHERE mapContains(resources_string, 'service.name') = ?",
+		},
+		{
+			name: "Exists - window straddles release - multiIf null check",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "service.name",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Evolutions:    evolutions,
+			},
+			operator:    qbtypes.FilterOperatorExists,
+			tsStart:     uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:       uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL",
+		},
+	}
+
+	fm := NewFieldMapper()
+	conditionBuilder := NewConditionBuilder(fm)
+
+	for _, tc := range testCases {
+		sb := sqlbuilder.NewSelectBuilder()
+		t.Run(tc.name, func(t *testing.T) {
+			cond, err := conditionBuilder.ConditionFor(ctx, tc.tsStart, tc.tsEnd, &tc.key, tc.operator, nil, sb)
+			require.NoError(t, err)
+			sb.Where(cond)
+			sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+			assert.Contains(t, sql, tc.expectedSQL)
+		})
+	}
+}

pkg/telemetrytraces/const.go

@@ -1,6 +1,8 @@
 package telemetrytraces
 
-import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+import (
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
 
 var (
 	IntrinsicFields = map[string]telemetrytypes.TelemetryFieldKey{

pkg/telemetrytraces/field_mapper.go

@@ -174,7 +174,7 @@ func (m *defaultFieldMapper) getColumn(
 ) ([]*schema.Column, error) {
 	switch key.FieldContext {
 	case telemetrytypes.FieldContextResource:
-		return []*schema.Column{indexV3Columns["resource"]}, nil
+		return []*schema.Column{indexV3Columns["resources_string"], indexV3Columns["resource"]}, nil
 	case telemetrytypes.FieldContextScope:
 		return []*schema.Column{}, qbtypes.ErrColumnNotFound
 	case telemetrytypes.FieldContextAttribute:
@@ -254,63 +254,92 @@ func (m *defaultFieldMapper) FieldFor(
 	if err != nil {
 		return "", err
 	}
-	if len(columns) != 1 {
-		return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "expected exactly 1 column, got %d", len(columns))
+
+	var newColumns []*schema.Column
+	var evolutionsEntries []*telemetrytypes.EvolutionEntry
+	if len(key.Evolutions) > 0 {
+		// we will use the corresponding column and its evolution entry for the query
+		newColumns, evolutionsEntries, err = qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		newColumns = columns
 	}
-	column := columns[0]
 
-	switch column.Type.GetType() {
-	case schema.ColumnTypeEnumJSON:
-		// json is only supported for resource context as of now
-		if key.FieldContext != telemetrytypes.FieldContextResource {
-			return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
-		}
-		oldColumn := indexV3Columns["resources_string"]
-		oldKeyName := fmt.Sprintf("%s['%s']", oldColumn.Name, key.Name)
-		// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
-		// once clickHouse dependency is updated, we need to check if we can remove it.
-		if key.Materialized {
-			oldKeyName = telemetrytypes.FieldKeyToMaterializedColumnName(key)
-			oldKeyNameExists := telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
-			return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, %s==true, %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldKeyNameExists, oldKeyName), nil
-		} else {
-			return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
-		}
-	case schema.ColumnTypeEnumString,
-		schema.ColumnTypeEnumUInt64,
-		schema.ColumnTypeEnumUInt32,
-		schema.ColumnTypeEnumInt8,
-		schema.ColumnTypeEnumInt16,
-		schema.ColumnTypeEnumBool,
-		schema.ColumnTypeEnumDateTime64,
-		schema.ColumnTypeEnumFixedString:
-		return column.Name, nil
-	case schema.ColumnTypeEnumLowCardinality:
-		switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
-		case schema.ColumnTypeEnumString:
-			return column.Name, nil
-		default:
-			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for low cardinality column type %s", elementType, column.Type)
-		}
-	case schema.ColumnTypeEnumMap:
-		keyType := column.Type.(schema.MapColumnType).KeyType
-		if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
-			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
+	exprs := []string{}
+	existExpr := []string{}
+	for i, column := range newColumns {
+		// Use evolution column name if available, otherwise use the column name
+		columnName := column.Name
+		if evolutionsEntries != nil && evolutionsEntries[i] != nil {
+			columnName = evolutionsEntries[i].ColumnName
 		}
 
-		switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
-		case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
-			// a key could have been materialized, if so return the materialized column name
-			if key.Materialized {
-				return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
+		switch column.Type.GetType() {
+		case schema.ColumnTypeEnumJSON:
+			// json is only supported for resource context as of now
+			if key.FieldContext != telemetrytypes.FieldContextResource {
+				return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
+			}
+			// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
+			// once clickHouse dependency is updated, we need to check if we can remove it.
+			exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
+			existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
+		case schema.ColumnTypeEnumString,
+			schema.ColumnTypeEnumUInt64,
+			schema.ColumnTypeEnumUInt32,
+			schema.ColumnTypeEnumInt8,
+			schema.ColumnTypeEnumInt16,
+			schema.ColumnTypeEnumBool,
+			schema.ColumnTypeEnumDateTime64,
+			schema.ColumnTypeEnumFixedString:
+			exprs = append(exprs, column.Name)
+		case schema.ColumnTypeEnumLowCardinality:
+			switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
+			case schema.ColumnTypeEnumString:
+				exprs = append(exprs, column.Name)
+			default:
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for low cardinality column type %s", elementType, column.Type)
+			}
+		case schema.ColumnTypeEnumMap:
+			keyType := column.Type.(schema.MapColumnType).KeyType
+			if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
+			}
+
+			switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
+			case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
+				// a key could have been materialized, if so return the materialized column name
+				if key.Materialized {
+					exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
+					existExpr = append(existExpr, fmt.Sprintf("%s==true", telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)))
+				} else {
+					exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
+					existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
+				}
+			default:
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
 			}
-			return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
-		default:
-			return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
 		}
 	}
+
+	if len(exprs) == 1 {
+		return exprs[0], nil
+	} else if len(exprs) > 1 {
+		// Ensure existExpr has the same length as exprs
+		if len(existExpr) != len(exprs) {
+			return "", errors.New(errors.TypeInternal, errors.CodeInternal, "length of exist exprs doesn't match to that of exprs")
+		}
+		finalExprs := []string{}
+		for i, expr := range exprs {
+			finalExprs = append(finalExprs, fmt.Sprintf("%s, %s", existExpr[i], expr))
+		}
+		return "multiIf(" + strings.Join(finalExprs, ", ") + ", NULL)", nil
+	}
+
 	// should not reach here
-	return column.Name, nil
+	return columns[0].Name, nil
 }
 
 // ColumnExpressionFor returns the column expression for the given field

pkg/telemetrytraces/field_mapper_test.go

@@ -3,6 +3,7 @@ package telemetrytraces
 import (
 	"context"
 	"testing"
+	"time"
 
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -13,6 +14,7 @@ import (
 func TestGetFieldKeyName(t *testing.T) {
 	ctx := context.Background()
 
+	mockEvolution := mockEvolutionData(time.Date(2024, 6, 2, 0, 0, 0, 0, time.UTC))
 	testCases := []struct {
 		name           string
 		key            telemetrytypes.TelemetryFieldKey
@@ -63,6 +65,7 @@ func TestGetFieldKeyName(t *testing.T) {
 			key: telemetrytypes.TelemetryFieldKey{
 				Name:         "service.name",
 				FieldContext: telemetrytypes.FieldContextResource,
+				Evolutions:   mockEvolution,
 			},
 			expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
 			expectedError:  nil,
@@ -74,6 +77,7 @@ func TestGetFieldKeyName(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextResource,
 				FieldDataType: telemetrytypes.FieldDataTypeString,
 				Materialized:  true,
+				Evolutions:    mockEvolution,
 			},
 			expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`==true, `resource_string_deployment$$environment`, NULL)",
 			expectedError:  nil,
@@ -92,7 +96,7 @@ func TestGetFieldKeyName(t *testing.T) {
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			fm := NewFieldMapper()
-			result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
+			result, err := fm.FieldFor(ctx, uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), uint64(time.Date(2024, 6, 5, 0, 0, 0, 0, time.UTC).UnixNano()), &tc.key)
 
 			if tc.expectedError != nil {
 				assert.Equal(t, tc.expectedError, err)
@@ -103,3 +107,86 @@ func TestGetFieldKeyName(t *testing.T) {
 		})
 	}
 }
+
+func TestFieldForResourceWithEvolution(t *testing.T) {
+	ctx := context.Background()
+	releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
+	evolutions := mockEvolutionData(releaseTime)
+
+	testCases := []struct {
+		name           string
+		key            telemetrytypes.TelemetryFieldKey
+		tsStart        uint64
+		tsEnd          uint64
+		expectedResult string
+	}{
+		{
+			name: "Window straddles release - both columns",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+				Evolutions:   evolutions,
+			},
+			tsStart:        uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
+		},
+		{
+			name: "Window fully after release - JSON column only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+				Evolutions:   evolutions,
+			},
+			tsStart:        uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "resource.`service.name`::String",
+		},
+		{
+			name: "Window fully before release - map column only",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:         "service.name",
+				FieldContext: telemetrytypes.FieldContextResource,
+				Evolutions:   evolutions,
+			},
+			tsStart:        uint64(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "resources_string['service.name']",
+		},
+		{
+			name: "Window fully after release - materialized resource",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "deployment.environment",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Materialized:  true,
+				Evolutions:    evolutions,
+			},
+			tsStart:        uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "resource.`deployment.environment`::String",
+		},
+		{
+			name: "Window straddles release - materialized resource",
+			key: telemetrytypes.TelemetryFieldKey{
+				Name:          "deployment.environment",
+				FieldContext:  telemetrytypes.FieldContextResource,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				Materialized:  true,
+				Evolutions:    evolutions,
+			},
+			tsStart:        uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:          uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`==true, `resource_string_deployment$$environment`, NULL)",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			fm := NewFieldMapper()
+			result, err := fm.FieldFor(ctx, tc.tsStart, tc.tsEnd, &tc.key)
+			require.NoError(t, err)
+			assert.Equal(t, tc.expectedResult, result)
+		})
+	}
+}

pkg/telemetrytraces/statement_builder.go

@@ -82,13 +82,6 @@ func (b *traceQueryStatementBuilder) Build(
 	start = querybuilder.ToNanoSecs(start)
 	end = querybuilder.ToNanoSecs(end)
 
-	keySelectors := getKeySelectors(query)
-
-	keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
-	if err != nil {
-		return nil, err
-	}
-
 	/*
 		Adding a tech debt note here:
 		This piece of code is a hot fix and should be removed once we close issue: engineering-pod/issues/3622
@@ -124,6 +117,15 @@ func (b *traceQueryStatementBuilder) Build(
 		-------------------------------- End of tech debt ----------------------------
 	*/
 
+	// We modify SelectFields above (injecting default fields), and those default
+	// fields can carry keys that need evolutions, so fetch keys after that.
+	keySelectors := getKeySelectors(query)
+
+	keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
+	if err != nil {
+		return nil, err
+	}
+
 	for _, action := range adjustTraceKeys(keys, &query, requestType) {
 		// TODO: change to debug level once we are confident about the behavior
 		b.logger.InfoContext(ctx, "key adjustment action", slog.String("action", action))

pkg/telemetrytraces/stmt_builder_test.go

@@ -16,6 +16,9 @@ import (
 )
 
 func TestStatementBuilder(t *testing.T) {
+	// releaseTime is chosen so it lands inside the standard [1747947419000, 1747983448000]ms
+	// test window, keeping the multiIf SQL form for resource fields.
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		requestType qbtypes.RequestType
@@ -355,7 +358,7 @@ func TestStatementBuilder(t *testing.T) {
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 
@@ -394,6 +397,7 @@ func TestStatementBuilder(t *testing.T) {
 }
 
 func TestStatementBuilderListQuery(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		requestType qbtypes.RequestType
@@ -650,7 +654,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 
@@ -683,6 +687,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 }
 
 func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		requestType qbtypes.RequestType
@@ -703,6 +708,15 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
 						FieldDataType: telemetrytypes.FieldDataTypeString,
 					},
 				},
+				"service.name": {
+					{
+						Name:          "service.name",
+						Signal:        telemetrytypes.SignalTraces,
+						FieldContext:  telemetrytypes.FieldContextResource,
+						FieldDataType: telemetrytypes.FieldDataTypeString,
+						Evolutions:    mockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
+					},
+				},
 			},
 			query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
 				Signal:       telemetrytypes.SignalTraces,
@@ -728,6 +742,15 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
 						FieldDataType: telemetrytypes.FieldDataTypeString,
 					},
 				},
+				"service.name": {
+					{
+						Name:          "service.name",
+						Signal:        telemetrytypes.SignalTraces,
+						FieldContext:  telemetrytypes.FieldContextResource,
+						FieldDataType: telemetrytypes.FieldDataTypeString,
+						Evolutions:    mockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
+					},
+				},
 			},
 			query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
 				Signal:       telemetrytypes.SignalTraces,
@@ -758,7 +781,7 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
 			mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 			mockMetadataStore.KeysMap = c.keysMap
 			if mockMetadataStore.KeysMap == nil {
-				mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+				mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 			}
 			fl := flaggertest.New(t)
 			aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
@@ -788,7 +811,90 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
 	}
 }
 
+func TestStatementBuilderGroupByResourceEvolution(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
+
+	cases := []struct {
+		name     string
+		startMs  uint64
+		endMs    uint64
+		expected qbtypes.Statement
+	}{
+		{
+			name:    "window straddles release - both JSON and map branches",
+			startMs: 1747947419000, // 2025-05-22 21:56:59 UTC, ~3m before release
+			endMs:   1747983448000, // 2025-05-23 07:57:28 UTC, ~10h after release
+			expected: qbtypes.Statement{
+				Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
+				Args:  []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
+			},
+		},
+		{
+			name:    "window after release - JSON column only",
+			startMs: 1747960000000, // 2025-05-23 00:26:40 UTC, ~2.5h after release
+			endMs:   1747983448000, // 2025-05-23 07:57:28 UTC
+			expected: qbtypes.Statement{
+				Query: "WITH __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
+				Args:  []any{"1747960000000000000", "1747983448000000000", uint64(1747958200), uint64(1747983448), 10, "1747960000000000000", "1747983448000000000", uint64(1747958200), uint64(1747983448)},
+			},
+		},
+		{
+			name:    "window before release - map column only",
+			startMs: 1747900000000, // 2025-05-22 08:26:40 UTC, ~13.5h before release
+			endMs:   1747947000000, // 2025-05-22 21:50:00 UTC, ~10m before release
+			expected: qbtypes.Statement{
+				Query: "WITH __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
+				Args:  []any{true, "1747900000000000000", "1747947000000000000", uint64(1747898200), uint64(1747947000), 10, true, "1747900000000000000", "1747947000000000000", uint64(1747898200), uint64(1747947000)},
+			},
+		},
+	}
+
+	fm := NewFieldMapper()
+	cb := NewConditionBuilder(fm)
+	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
+	fl := flaggertest.New(t)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
+
+	statementBuilder := NewTraceQueryStatementBuilder(
+		instrumentationtest.New().ToProviderSettings(),
+		mockMetadataStore,
+		fm,
+		cb,
+		aggExprRewriter,
+		nil,
+		fl,
+	)
+
+	query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
+		Signal:       telemetrytypes.SignalTraces,
+		StepInterval: qbtypes.Step{Duration: 30 * time.Second},
+		Aggregations: []qbtypes.TraceAggregation{
+			{Expression: "count()"},
+		},
+		Filter: &qbtypes.Filter{},
+		Limit:  10,
+		GroupBy: []qbtypes.GroupByKey{
+			{
+				TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+					Name: "service.name",
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			q, err := statementBuilder.Build(context.Background(), c.startMs, c.endMs, qbtypes.RequestTypeTimeSeries, query, nil)
+			require.NoError(t, err)
+			require.Equal(t, c.expected.Query, q.Query)
+			require.Equal(t, c.expected.Args, q.Args)
+		})
+	}
+}
+
 func TestStatementBuilderTraceQuery(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		requestType qbtypes.RequestType
@@ -911,7 +1017,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 
@@ -944,6 +1050,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
 }
 
 func TestAdjustKey(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name        string
 		inputKey    telemetrytypes.TelemetryFieldKey
@@ -957,7 +1064,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
+			keysMap:     buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: IntrinsicFields["trace_id"],
 		},
 		{
@@ -967,7 +1074,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextBody, // incorrect context
 				FieldDataType: telemetrytypes.FieldDataTypeInt64,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "duration_nano",
 				FieldContext:  telemetrytypes.FieldContextSpan,    // should be corrected
@@ -981,7 +1088,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextSpan, // correct context
 				FieldDataType: telemetrytypes.FieldDataTypeInt64,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "duration_nano",
 				FieldContext:  telemetrytypes.FieldContextSpan,   // should be corrected
@@ -995,8 +1102,8 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
+			keysMap:     buildCompleteFieldKeyMap(releaseTime),
+			expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
 		},
 		{
 			name: "single matching key with context specified - override",
@@ -1005,8 +1112,8 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextAttribute,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["cart.items_count"][0],
+			keysMap:     buildCompleteFieldKeyMap(releaseTime),
+			expectedKey: *buildCompleteFieldKeyMap(releaseTime)["cart.items_count"][0],
 		},
 		{
 			name: "multiple matching keys - all materialized",
@@ -1043,7 +1150,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "mixed.materialization.key",
 				FieldDataType: telemetrytypes.FieldDataTypeString,
@@ -1057,7 +1164,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextAttribute,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "mixed.materialization.key",
 				FieldContext:  telemetrytypes.FieldContextAttribute,
@@ -1072,7 +1179,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:         "unknown.field",
 				Materialized: false,
@@ -1085,7 +1192,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextAttribute,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "service.name",
 				FieldContext:  telemetrytypes.FieldContextAttribute,
@@ -1100,7 +1207,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "cart.items_count",
 				FieldContext:  telemetrytypes.FieldContextAttribute,
@@ -1115,7 +1222,7 @@ func TestAdjustKey(t *testing.T) {
 				FieldContext:  telemetrytypes.FieldContextUnspecified,
 				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedKey: telemetrytypes.TelemetryFieldKey{
 				Name:          "user.id",
 				FieldContext:  telemetrytypes.FieldContextAttribute,
@@ -1143,6 +1250,7 @@ func TestAdjustKey(t *testing.T) {
 }
 
 func TestAdjustKeys(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	cases := []struct {
 		name                      string
 		query                     qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]
@@ -1168,7 +1276,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedSelectFields: []telemetrytypes.TelemetryFieldKey{
 				{
 					Name:          "service.name",
@@ -1205,7 +1313,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedGroupBy: []qbtypes.GroupByKey{
 				{
 					TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
@@ -1252,7 +1360,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedOrder: []qbtypes.OrderBy{
 				{
 					Key: qbtypes.OrderByKey{
@@ -1311,7 +1419,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			expectedSelectFields: []telemetrytypes.TelemetryFieldKey{
 				{
 					Name:          "trace_id",
@@ -1366,7 +1474,7 @@ func TestAdjustKeys(t *testing.T) {
 					},
 				},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
+			keysMap: buildCompleteFieldKeyMap(releaseTime),
 			// After alias adjustment, name becomes "span.duration" with FieldContextUnspecified
 			// "span.duration" is not in keysMap, so context stays unspecified
 			expectedOrder: []qbtypes.OrderBy{

pkg/telemetrytraces/test_data.go

@@ -1,10 +1,12 @@
 package telemetrytraces
 
 import (
+	"time"
+
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 )
 
-func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
+func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
 	keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
 		"service.name": {
 			{
@@ -115,7 +117,33 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
 	for _, keys := range keysMap {
 		for _, key := range keys {
 			key.Signal = telemetrytypes.SignalTraces
+			if key.FieldContext == telemetrytypes.FieldContextResource {
+				key.Evolutions = mockEvolutionData(releaseTime)
+			}
 		}
 	}
 	return keysMap
 }
+
+// mockEvolutionData returns the canonical resource-column evolution timeline used in tests:
+// the legacy resources_string map at epoch 0 and the JSON resource column released at releaseTime.
+func mockEvolutionData(releaseTime time.Time) []*telemetrytypes.EvolutionEntry {
+	return []*telemetrytypes.EvolutionEntry{
+		{
+			Signal:       telemetrytypes.SignalTraces,
+			ColumnName:   "resources_string",
+			FieldContext: telemetrytypes.FieldContextResource,
+			ColumnType:   "Map(LowCardinality(String), String)",
+			FieldName:    "__all__",
+			ReleaseTime:  time.Unix(0, 0),
+		},
+		{
+			Signal:       telemetrytypes.SignalTraces,
+			ColumnName:   "resource",
+			ColumnType:   "JSON()",
+			FieldContext: telemetrytypes.FieldContextResource,
+			FieldName:    "__all__",
+			ReleaseTime:  releaseTime,
+		},
+	}
+}

pkg/telemetrytraces/trace_operator_cte_builder_test.go

@@ -16,10 +16,11 @@ import (
 
 func newTestTraceOperatorStatementBuilder(t *testing.T) *traceOperatorStatementBuilder {
 	t.Helper()
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	fl := flaggertest.New(t)
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
 	traceStmtBuilder := NewTraceQueryStatementBuilder(

pkg/telemetrytraces/trace_time_range_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -16,12 +17,13 @@ import (
 )
 
 func TestTraceTimeRangeOptimization(t *testing.T) {
+	releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
 
 	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
 	mockMetadataStore.KeysMap["trace_id"] = []*telemetrytypes.TelemetryFieldKey{{
 		Name:          "trace_id",
 		FieldContext:  telemetrytypes.FieldContextSpan,

pkg/types/cloudintegrationtypes/serviceid.go

@@ -27,6 +27,7 @@ var (
 	// Azure services.
 	AzureServiceStorageAccountsBlob = ServiceID{valuer.NewString("storageaccountsblob")}
 	AzureServiceCDNProfile          = ServiceID{valuer.NewString("cdnprofile")}
+	AzureServiceAppService          = ServiceID{valuer.NewString("appservice")}
 )
 
 func (ServiceID) Enum() []any {
@@ -46,6 +47,7 @@ func (ServiceID) Enum() []any {
 		AWSServiceSQS,
 		AzureServiceStorageAccountsBlob,
 		AzureServiceCDNProfile,
+		AzureServiceAppService,
 	}
 }
 
@@ -69,6 +71,7 @@ var SupportedServices = map[CloudProviderType][]ServiceID{
 	CloudProviderTypeAzure: {
 		AzureServiceStorageAccountsBlob,
 		AzureServiceCDNProfile,
+		AzureServiceAppService,
 	},
 }
 

pkg/types/querybuildertypes/querybuildertypesv5/evolution.go

@@ -0,0 +1,119 @@
+package querybuildertypesv5
+
+import (
+	"slices"
+	"sort"
+	"strconv"
+	"time"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// SelectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
+// Logic:
+//   - Finds the latest base evolution (<= tsStartTime) across ALL columns
+//   - Rejects all evolutions before this latest base evolution
+//   - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
+//   - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime
+//   - Results are sorted by ReleaseTime descending (newest first)
+func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) {
+
+	sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions))
+	copy(sortedEvolutions, evolutions)
+
+	// sort the evolutions by ReleaseTime ascending
+	sort.Slice(sortedEvolutions, func(i, j int) bool {
+		return sortedEvolutions[i].ReleaseTime.Before(sortedEvolutions[j].ReleaseTime)
+	})
+
+	tsStartTime := time.Unix(0, int64(tsStart))
+	tsEndTime := time.Unix(0, int64(tsEnd))
+
+	// Build evolution map: column name -> evolution
+	evolutionMap := make(map[string]*telemetrytypes.EvolutionEntry)
+	for _, evolution := range sortedEvolutions {
+		if _, exists := evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))]; exists {
+			// since if there is duplicate we would just use the oldest one.
+			continue
+		}
+		evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))] = evolution
+	}
+
+	// Find the latest base evolution (<= tsStartTime) across ALL columns
+	// Evolutions are sorted, so we can break early
+	var latestBaseEvolutionAcrossAll *telemetrytypes.EvolutionEntry
+	for _, evolution := range sortedEvolutions {
+		if evolution.ReleaseTime.After(tsStartTime) {
+			break
+		}
+		latestBaseEvolutionAcrossAll = evolution
+	}
+
+	// We shouldn't reach this, it basically means there is something wrong with the evolutions data
+	if latestBaseEvolutionAcrossAll == nil {
+		return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
+	}
+
+	columnLookUpMap := make(map[string]*schema.Column)
+	for _, column := range columns {
+		columnLookUpMap[column.Name] = column
+	}
+
+	// Collect column-evolution pairs
+	type colEvoPair struct {
+		column    *schema.Column
+		evolution *telemetrytypes.EvolutionEntry
+	}
+	pairs := []colEvoPair{}
+
+	for _, evolution := range evolutionMap {
+		// Reject evolutions before the latest base evolution
+		if evolution.ReleaseTime.Before(latestBaseEvolutionAcrossAll.ReleaseTime) {
+			continue
+		}
+		// skip evolutions after tsEndTime
+		if evolution.ReleaseTime.After(tsEndTime) || evolution.ReleaseTime.Equal(tsEndTime) {
+			continue
+		}
+
+		if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
+			return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
+		}
+
+		pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
+	}
+
+	// If no pairs found, fall back to latestBaseEvolutionAcrossAll for matching columns
+	if len(pairs) == 0 {
+		for _, column := range columns {
+			// Use latestBaseEvolutionAcrossAll if this column name matches its column name
+			if column.Name == latestBaseEvolutionAcrossAll.ColumnName {
+				pairs = append(pairs, colEvoPair{column, latestBaseEvolutionAcrossAll})
+			}
+		}
+	}
+
+	// Sort by ReleaseTime descending (newest first)
+	slices.SortFunc(pairs, func(a, b colEvoPair) int {
+		// Sort by ReleaseTime descending (newest first)
+		if a.evolution.ReleaseTime.After(b.evolution.ReleaseTime) {
+			return -1
+		}
+		if a.evolution.ReleaseTime.Before(b.evolution.ReleaseTime) {
+			return 1
+		}
+		return 0
+	})
+
+	// Extract results
+	newColumns := make([]*schema.Column, len(pairs))
+	evolutionsEntries := make([]*telemetrytypes.EvolutionEntry, len(pairs))
+	for i, pair := range pairs {
+		newColumns[i] = pair.column
+		evolutionsEntries[i] = pair.evolution
+	}
+
+	return newColumns, evolutionsEntries, nil
+}

pkg/types/querybuildertypes/querybuildertypesv5/evolution_test.go

@@ -0,0 +1,414 @@
+package querybuildertypesv5
+
+import (
+	"testing"
+	"time"
+
+	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	LogsV2BodyV2Column       = "body_v2"
+	LogsV2BodyPromotedColumn = "body_promoted"
+)
+
+var (
+	resources_string = &schema.Column{Name: "resources_string", Type: schema.MapColumnType{
+		KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		ValueType: schema.ColumnTypeString,
+	}}
+	resource          = &schema.Column{Name: "resource", Type: schema.JSONColumnType{}}
+	attributes_string = &schema.Column{Name: "attributes_string", Type: schema.MapColumnType{
+		KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
+		ValueType: schema.ColumnTypeString,
+	}}
+	body_v2       = &schema.Column{Name: LogsV2BodyV2Column, Type: schema.JSONColumnType{}}
+	body_promoted = &schema.Column{Name: LogsV2BodyPromotedColumn, Type: schema.JSONColumnType{}}
+)
+
+func TestSelectEvolutionsForColumns(t *testing.T) {
+	testCases := []struct {
+		name            string
+		columns         []*schema.Column
+		evolutions      []*telemetrytypes.EvolutionEntry
+		tsStart         uint64
+		tsEnd           uint64
+		expectedColumns []string // column names
+		expectedEvols   []string // evolution column names
+		expectedError   bool
+		errorStr        string
+	}{
+		{
+			name: "New evolutions at tsStartTime - should include latest evolution",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resource"},
+			expectedEvols:   []string{"resource"},
+		},
+		{
+			name: "New evolutions after tsStartTime but less than tsEndTime - should include both",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resource", "resources_string"}, // sorted by ReleaseTime desc
+			expectedEvols:   []string{"resource", "resources_string"},
+		},
+		{
+			name: "Columns without matching evolutions - should exclude them",
+			columns: []*schema.Column{
+				resources_string,
+				resource,          // no evolution for this
+				attributes_string, // no evolution for this
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string"},
+			expectedEvols:   []string{"resources_string"},
+		},
+		{
+			name: "New evolutions at tsEndTime - should not include new evolution",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string"},
+			expectedEvols:   []string{"resources_string"},
+		},
+		{
+			name: "New evolutions after tsEndTime - should exclude new",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string"},
+			expectedEvols:   []string{"resources_string"},
+		},
+		{
+			name:    "Empty columns array",
+			columns: []*schema.Column{},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{},
+			expectedEvols:   []string{},
+			expectedError:   true,
+			errorStr:        "column resources_string not found",
+		},
+		{
+			name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
+			columns: []*schema.Column{
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resource"},
+			expectedEvols:   []string{"resource"}, // should use first one (older)
+		},
+		{
+			name: "Genuine Duplicate evolutions with new version- should consider both",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      0,
+					ReleaseTime:  time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      1,
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					Version:      2,
+					ReleaseTime:  time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string", "resource"},
+			expectedEvols:   []string{"resources_string", "resource"}, // should use first one (older)
+		},
+		{
+			name: "Evolution exactly at tsEndTime",
+			columns: []*schema.Column{
+				resources_string,
+				resource,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resources_string",
+					ColumnType:   "Map(LowCardinality(String), String)",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					ReleaseTime:  time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   "resource",
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextResource,
+					FieldName:    "__all__",
+					ReleaseTime:  time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), // exactly at tsEnd
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{"resources_string"}, // resource excluded because After(tsEnd) is true
+			expectedEvols:   []string{"resources_string"},
+		},
+		{
+			name: "Single evolution after tsStartTime - JSON body",
+			columns: []*schema.Column{
+				body_v2,
+				body_promoted,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   LogsV2BodyV2Column,
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextBody,
+					FieldName:    "__all__",
+					ReleaseTime:  time.Unix(0, 0),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   LogsV2BodyPromotedColumn,
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextBody,
+					FieldName:    "user.name",
+					ReleaseTime:  time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column}, // sorted by ReleaseTime desc (newest first)
+			expectedEvols:   []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column},
+		},
+		{
+			name: "No evolution after tsStartTime - JSON body",
+			columns: []*schema.Column{
+				body_v2,
+				body_promoted,
+			},
+			evolutions: []*telemetrytypes.EvolutionEntry{
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   LogsV2BodyV2Column,
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextBody,
+					FieldName:    "__all__",
+					ReleaseTime:  time.Unix(0, 0),
+				},
+				{
+					Signal:       telemetrytypes.SignalLogs,
+					ColumnName:   LogsV2BodyPromotedColumn,
+					ColumnType:   "JSON()",
+					FieldContext: telemetrytypes.FieldContextBody,
+					FieldName:    "user.name",
+					ReleaseTime:  time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
+				},
+			},
+			tsStart:         uint64(time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC).UnixNano()),
+			tsEnd:           uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
+			expectedColumns: []string{LogsV2BodyPromotedColumn},
+			expectedEvols:   []string{LogsV2BodyPromotedColumn},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			resultColumns, resultEvols, err := SelectEvolutionsForColumns(tc.columns, tc.evolutions, tc.tsStart, tc.tsEnd)
+
+			if tc.expectedError {
+				assert.Contains(t, err.Error(), tc.errorStr)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, len(tc.expectedColumns), len(resultColumns), "column count mismatch")
+				assert.Equal(t, len(tc.expectedEvols), len(resultEvols), "evolution count mismatch")
+
+				resultColumnNames := make([]string, len(resultColumns))
+				for i, col := range resultColumns {
+					resultColumnNames[i] = col.Name
+				}
+				resultEvolNames := make([]string, len(resultEvols))
+				for i, evol := range resultEvols {
+					resultEvolNames[i] = evol.ColumnName
+				}
+
+				for i := range tc.expectedColumns {
+					assert.Equal(t, resultColumnNames[i], tc.expectedColumns[i], "expected column missing: "+tc.expectedColumns[i])
+				}
+				for i := range tc.expectedEvols {
+					assert.Equal(t, resultEvolNames[i], tc.expectedEvols[i], "expected evolution missing: "+tc.expectedEvols[i])
+				}
+				// Verify sorting: should be descending by ReleaseTime
+				for i := 0; i < len(resultEvols)-1; i++ {
+					assert.True(t, !resultEvols[i].ReleaseTime.Before(resultEvols[i+1].ReleaseTime),
+						"evolutions should be sorted descending by ReleaseTime")
+				}
+			}
+		})
+	}
+}

pkg/web/config.go

@@ -24,7 +24,7 @@ type SettingsConfig struct {
 	Posthog PosthogConfig `mapstructure:"posthog"`
 	Appcues AppcuesConfig `mapstructure:"appcues"`
 	Sentry  SentryConfig  `mapstructure:"sentry"`
-	Pylon   PylonConfig   `mapstructure:"sentry"`
+	Pylon   PylonConfig   `mapstructure:"pylon"`
 }
 
 type PosthogConfig struct {

pkg/web/config_test.go

@@ -43,3 +43,42 @@ func TestNewWithEnvProvider(t *testing.T) {
 
 	assert.Equal(t, expected, actual)
 }
+
+func TestSettingsConfigWithEnvProvider(t *testing.T) {
+	testCases := []struct {
+		name     string
+		env      string
+		expected SettingsConfig
+	}{
+		{name: "posthog", env: "SIGNOZ_WEB_SETTINGS_POSTHOG_ENABLED", expected: SettingsConfig{Posthog: PosthogConfig{Enabled: true}}},
+		{name: "appcues", env: "SIGNOZ_WEB_SETTINGS_APPCUES_ENABLED", expected: SettingsConfig{Appcues: AppcuesConfig{Enabled: true}}},
+		{name: "sentry", env: "SIGNOZ_WEB_SETTINGS_SENTRY_ENABLED", expected: SettingsConfig{Sentry: SentryConfig{Enabled: true}}},
+		{name: "pylon", env: "SIGNOZ_WEB_SETTINGS_PYLON_ENABLED", expected: SettingsConfig{Pylon: PylonConfig{Enabled: true}}},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			t.Setenv(testCase.env, "true")
+
+			conf, err := config.New(
+				context.Background(),
+				config.ResolverConfig{
+					Uris: []string{"env:"},
+					ProviderFactories: []config.ProviderFactory{
+						envprovider.NewFactory(),
+					},
+				},
+				[]factory.ConfigFactory{
+					NewConfigFactory(),
+				},
+			)
+			require.NoError(t, err)
+
+			actual := &Config{}
+			err = conf.Unmarshal("web", actual)
+			require.NoError(t, err)
+
+			assert.Equal(t, testCase.expected, actual.Settings)
+		})
+	}
+}

tests/fixtures/querier.py

@@ -449,6 +449,21 @@ def index_series_by_label(
     return series_by_label
 
 
+def assert_grouped_series(
+    series_by_group: dict[str, dict],
+    expected_values_by_group: dict[str, dict[int, int]],
+) -> None:
+    assert set(series_by_group.keys()) == set(expected_values_by_group.keys())
+
+    for group_name, expected_by_ts in expected_values_by_group.items():
+        actual_values = sorted(
+            series_by_group[group_name]["values"],
+            key=lambda value: value["timestamp"],
+        )
+        expected_values = [{"timestamp": timestamp, "value": value} for timestamp, value in sorted(expected_by_ts.items())]
+        assert actual_values == expected_values
+
+
 def find_named_result(
     results: list[dict[str, Any]],
     name: str,
@@ -698,6 +713,28 @@ def assert_identical_query_response(response1: requests.Response, response2: req
         assert response1.json()["data"]["data"]["results"] == response2.json()["data"]["data"]["results"], "Response data do not match"
 
 
+# we already create the evolution for resource during schema migration
+# since we have to create test data around it, we need to get the evolution time
+def get_resource_evolution_time(signoz: types.SigNoz, signal: str) -> datetime:
+    result = signoz.telemetrystore.conn.query(
+        """
+        SELECT release_time
+        FROM signoz_metadata.distributed_column_evolution_metadata
+        WHERE signal = %(signal)s
+          AND field_context = 'resource'
+          AND field_name = '__all__'
+          AND column_name = 'resource'
+        LIMIT 1
+        """,
+        parameters={"signal": signal},
+    ).result_rows
+
+    assert result, f"Expected {signal} resource evolution metadata to exist"
+
+    release_time_ns = int(result[0][0])
+    return datetime.fromtimestamp(release_time_ns / 1e9, tz=UTC)
+
+
 def generate_logs_with_corrupt_metadata() -> list[Logs]:
     """
     Specifically, entries with 'id', 'timestamp', 'severity_text', 'severity_number' and 'body' fields in metadata

tests/fixtures/traces.py

@@ -6,7 +6,7 @@ import uuid
 from abc import ABC
 from collections.abc import Callable, Generator
 from enum import Enum
-from typing import Any
+from typing import Any, Literal
 from urllib.parse import urlparse
 
 import numpy as np
@@ -236,6 +236,7 @@ class Traces(ABC):
     attributes_number: dict[str, np.float64]
     attributes_bool: dict[str, bool]
     resources_string: dict[str, str]
+    resource_json: dict[str, str]
     events: list[str]
     links: str
     response_status_code: str
@@ -273,6 +274,7 @@ class Traces(ABC):
         links: list[TracesLink] = [],
         trace_state: str = "",
         flags: np.uint32 = 0,
+        resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
     ) -> None:
         if timestamp is None:
             timestamp = datetime.datetime.now()
@@ -322,8 +324,11 @@ class Traces(ABC):
         self.db_name = ""
         self.db_operation = ""
 
-        # Process resources and derive service_name
+        # Process resources and derive service_name. Spans written before the
+        # JSON-resource evolution time only populate resources_string (legacy_only);
+        # spans at or after the evolution time dual-write to both columns.
         self.resources_string = {k: str(v) for k, v in resources.items()}
+        self.resource_json = {} if resource_write_mode == "legacy_only" else dict(self.resources_string)
         self.service_name = self.resources_string.get("service.name", "default-service")
 
         for k, v in self.resources_string.items():
@@ -575,7 +580,7 @@ class Traces(ABC):
                 self.db_operation,
                 self.has_error,
                 self.is_remote,
-                self.resources_string,
+                self.resource_json,
             ],
             dtype=object,
         )

tests/integration/tests/querier/12_logs_resource_evolution.py

@@ -1,39 +1,20 @@
 from collections.abc import Callable
-from datetime import UTC, datetime, timedelta
+from datetime import datetime, timedelta
 from http import HTTPStatus
 
 from fixtures import types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.logs import Logs
 from fixtures.querier import (
+    assert_grouped_series,
     build_group_by_field,
     build_logs_aggregation,
+    get_resource_evolution_time,
     index_series_by_label,
     make_query_request,
 )
 
 
-# we already create the evolution for resource during schema migration
-# since we have to create test data around it, we need to get the evolution time
-def _get_logs_resource_evolution_time_json(signoz: types.SigNoz) -> datetime:
-    result = signoz.telemetrystore.conn.query(
-        """
-        SELECT release_time
-        FROM signoz_metadata.distributed_column_evolution_metadata
-        WHERE signal = 'logs'
-          AND field_context = 'resource'
-          AND field_name = '__all__'
-          AND column_name = 'resource'
-        LIMIT 1
-        """
-    ).result_rows
-
-    assert result, "Expected logs resource evolution metadata to exist"
-
-    release_time_ns = int(result[0][0])
-    return datetime.fromtimestamp(release_time_ns / 1e9, tz=UTC)
-
-
 # Logs with timestamps before the evolution time will have resources written only to resources_string.
 # Logs with timestamps at or after the evolution time will have resources written to both resources_string and resource_json.
 def _build_evolved_log(
@@ -97,21 +78,6 @@ def _query_grouped_log_series(
     return index_series_by_label(aggregations[0]["series"], group_by)
 
 
-def _assert_grouped_series(
-    series_by_group: dict[str, dict],
-    expected_values_by_group: dict[str, dict[int, int]],
-) -> None:
-    assert set(series_by_group.keys()) == set(expected_values_by_group.keys())
-
-    for group_name, expected_by_ts in expected_values_by_group.items():
-        actual_values = sorted(
-            series_by_group[group_name]["values"],
-            key=lambda value: value["timestamp"],
-        )
-        expected_values = [{"timestamp": timestamp, "value": value} for timestamp, value in sorted(expected_by_ts.items())]
-        assert actual_values == expected_values
-
-
 def _test_logs_resource_evolution(
     signoz: types.SigNoz,
     token: str,
@@ -125,7 +91,7 @@ def _test_logs_resource_evolution(
     # 5. Query the logs after the evolution time.
     # Both aggregation and group by should be checked.
     """
-    evolution_time = _get_logs_resource_evolution_time_json(signoz)
+    evolution_time = get_resource_evolution_time(signoz, "logs")
     evolution_time = evolution_time.replace(second=0, microsecond=0)
 
     before_2 = evolution_time - timedelta(minutes=10)
@@ -163,7 +129,7 @@ def _test_logs_resource_evolution(
     )
 
     before_series = _query_grouped_log_series(signoz, token, before_2 - timedelta(minutes=1), before_1 + timedelta(minutes=1))
-    _assert_grouped_series(
+    assert_grouped_series(
         before_series,
         expected_values_by_group={
             "svc-before-2": {
@@ -176,7 +142,7 @@ def _test_logs_resource_evolution(
     )
 
     after_series = _query_grouped_log_series(signoz, token, after_1 - timedelta(minutes=1), after_2 + timedelta(minutes=1))
-    _assert_grouped_series(
+    assert_grouped_series(
         after_series,
         expected_values_by_group={
             "svc-after-1": {
@@ -189,7 +155,7 @@ def _test_logs_resource_evolution(
     )
 
     spanning_series = _query_grouped_log_series(signoz, token, before_2, after_2 + timedelta(minutes=1))
-    _assert_grouped_series(
+    assert_grouped_series(
         spanning_series,
         expected_values_by_group={
             "svc-before-2": {
@@ -216,7 +182,7 @@ def _test_logs_resource_evolution(
         group_by="deployment.environment",
         aggregation="count_distinct(service.name)",
     )
-    _assert_grouped_series(
+    assert_grouped_series(
         aggregation_series,
         expected_values_by_group={
             "integration": {

tests/integration/tests/querier/13_traces_resource_evolution.py

@@ -0,0 +1,206 @@
+from collections.abc import Callable
+from datetime import datetime, timedelta
+from http import HTTPStatus
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.querier import (
+    assert_grouped_series,
+    build_group_by_field,
+    build_logs_aggregation,
+    get_resource_evolution_time,
+    index_series_by_label,
+    make_query_request,
+)
+from fixtures.traces import TraceIdGenerator, Traces
+
+
+# Spans with timestamps before the evolution time will have resources written only to resources_string.
+# Spans with timestamps at or after the evolution time will have resources written to both resources_string and resource (JSON).
+def _build_evolved_span(
+    timestamp: datetime,
+    evolution_time: datetime,
+    service_name: str,
+    name: str,
+) -> Traces:
+    resource_write_mode = "legacy_only" if timestamp < evolution_time else "dual_write"
+    return Traces(
+        timestamp=timestamp,
+        trace_id=TraceIdGenerator.trace_id(),
+        span_id=TraceIdGenerator.span_id(),
+        name=name,
+        resources={
+            "service.name": service_name,
+            "deployment.environment": "integration",
+        },
+        resource_write_mode=resource_write_mode,
+    )
+
+
+def _query_grouped_trace_series(
+    signoz: types.SigNoz,
+    token: str,
+    start: datetime,
+    end: datetime,
+    group_by: str = "service.name",
+    aggregation: str = "count()",
+) -> dict[str, list[dict]]:
+    response = make_query_request(
+        signoz,
+        token,
+        start_ms=int(start.timestamp() * 1000),
+        end_ms=int(end.timestamp() * 1000),
+        request_type="time_series",
+        queries=[
+            {
+                "type": "builder_query",
+                "spec": {
+                    "name": "A",
+                    "signal": "traces",
+                    "stepInterval": 60,
+                    "disabled": False,
+                    "groupBy": [build_group_by_field(group_by)],
+                    "having": {"expression": ""},
+                    "aggregations": [build_logs_aggregation(aggregation)],
+                },
+            }
+        ],
+    )
+
+    assert response.status_code == HTTPStatus.OK
+    assert response.json()["status"] == "success"
+
+    results = response.json()["data"]["data"]["results"]
+    assert len(results) == 1
+
+    aggregations = results[0]["aggregations"]
+    assert len(aggregations) == 1
+
+    return index_series_by_label(aggregations[0]["series"], group_by)
+
+
+def _test_traces_resource_evolution(
+    signoz: types.SigNoz,
+    token: str,
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    """
+    # 1. Get the evolution time.
+    # 2. Ingest spans before the evolution time.
+    # 3. Ingest spans after the evolution time.
+    # 4. Query the spans before the evolution time.
+    # 5. Query the spans after the evolution time.
+    # Both aggregation and group by should be checked.
+    """
+    evolution_time = get_resource_evolution_time(signoz, "traces")
+    evolution_time = evolution_time.replace(second=0, microsecond=0)
+
+    before_2 = evolution_time - timedelta(minutes=10)
+    before_1 = evolution_time - timedelta(minutes=5)
+    after_1 = evolution_time + timedelta(minutes=5)
+    after_2 = evolution_time + timedelta(minutes=10)
+
+    insert_traces(
+        [
+            _build_evolved_span(
+                timestamp=before_2,
+                evolution_time=evolution_time,
+                service_name="svc-before-2",
+                name="span before evolution 2",
+            ),
+            _build_evolved_span(
+                timestamp=before_1,
+                evolution_time=evolution_time,
+                service_name="svc-before-1",
+                name="span before evolution 1",
+            ),
+            _build_evolved_span(
+                timestamp=after_1,
+                evolution_time=evolution_time,
+                service_name="svc-after-1",
+                name="span after evolution 1",
+            ),
+            _build_evolved_span(
+                timestamp=after_2,
+                evolution_time=evolution_time,
+                service_name="svc-after-2",
+                name="span after evolution 2",
+            ),
+        ]
+    )
+
+    before_series = _query_grouped_trace_series(signoz, token, before_2 - timedelta(minutes=1), before_1 + timedelta(minutes=1))
+    assert_grouped_series(
+        before_series,
+        expected_values_by_group={
+            "svc-before-2": {
+                int(before_2.timestamp() * 1000): 1,
+            },
+            "svc-before-1": {
+                int(before_1.timestamp() * 1000): 1,
+            },
+        },
+    )
+
+    after_series = _query_grouped_trace_series(signoz, token, after_1 - timedelta(minutes=1), after_2 + timedelta(minutes=1))
+    assert_grouped_series(
+        after_series,
+        expected_values_by_group={
+            "svc-after-1": {
+                int(after_1.timestamp() * 1000): 1,
+            },
+            "svc-after-2": {
+                int(after_2.timestamp() * 1000): 1,
+            },
+        },
+    )
+
+    spanning_series = _query_grouped_trace_series(signoz, token, before_2, after_2 + timedelta(minutes=1))
+    assert_grouped_series(
+        spanning_series,
+        expected_values_by_group={
+            "svc-before-2": {
+                int(before_2.timestamp() * 1000): 1,
+            },
+            "svc-before-1": {
+                int(before_1.timestamp() * 1000): 1,
+            },
+            "svc-after-1": {
+                int(after_1.timestamp() * 1000): 1,
+            },
+            "svc-after-2": {
+                int(after_2.timestamp() * 1000): 1,
+            },
+        },
+    )
+
+    # query to check aggregation on the resource field like count_distinct(service.name)
+    aggregation_series = _query_grouped_trace_series(
+        signoz,
+        token,
+        before_2,
+        after_2 + timedelta(minutes=1),
+        group_by="deployment.environment",
+        aggregation="count_distinct(service.name)",
+    )
+    assert_grouped_series(
+        aggregation_series,
+        expected_values_by_group={
+            "integration": {
+                int(before_2.timestamp() * 1000): 1,
+                int(before_1.timestamp() * 1000): 1,
+                int(after_1.timestamp() * 1000): 1,
+                int(after_2.timestamp() * 1000): 1,
+            },
+        },
+    )
+
+
+def test_traces_resource_evolution(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    _test_traces_resource_evolution(signoz, token, insert_traces)