chore: send all data for trace list api

* fix: variables-initial-url-state

* chore: add tests

.github/CODEOWNERS

@@ -127,12 +127,15 @@
 /frontend/src/pages/DashboardsListPage/ @SigNoz/pulse-frontend
 /frontend/src/container/ListOfDashboard/ @SigNoz/pulse-frontend
 
+# Dashboard Widget Page
+/frontend/src/pages/DashboardWidget/ @SigNoz/pulse-frontend
+/frontend/src/container/NewWidget/ @SigNoz/pulse-frontend
+
 ## Dashboard Page
 
 /frontend/src/pages/DashboardPage/ @SigNoz/pulse-frontend
 /frontend/src/container/DashboardContainer/ @SigNoz/pulse-frontend
 /frontend/src/container/GridCardLayout/ @SigNoz/pulse-frontend
-/frontend/src/container/NewWidget/ @SigNoz/pulse-frontend
 
 ## Public Dashboard Page
 

frontend/src/container/DashboardContainer/DashboardVariablesSelection/DashboardVariableSelection.tsx

@@ -9,7 +9,6 @@ import {
 } from 'hooks/dashboard/useDashboardVariables';
 import useVariablesFromUrl from 'hooks/dashboard/useVariablesFromUrl';
 import { useDashboard } from 'providers/Dashboard/Dashboard';
-import { initializeDefaultVariables } from 'providers/Dashboard/initializeDefaultVariables';
 import { updateDashboardVariablesStore } from 'providers/Dashboard/store/dashboardVariables/dashboardVariablesStore';
 import {
 	enqueueDescendantsOfVariable,
@@ -30,7 +29,7 @@ function DashboardVariableSelection(): JSX.Element | null {
 		updateLocalStorageDashboardVariables,
 	} = useDashboard();
 
-	const { updateUrlVariable, getUrlVariables } = useVariablesFromUrl();
+	const { updateUrlVariable } = useVariablesFromUrl();
 
 	const { dashboardVariables } = useDashboardVariables();
 	const dashboardId = useDashboardVariablesSelector(
@@ -50,15 +49,6 @@ function DashboardVariableSelection(): JSX.Element | null {
 		(state) => state.globalTime,
 	);
 
-	useEffect(() => {
-		// Initialize variables with default values if not in URL
-		initializeDefaultVariables(
-			dashboardVariables,
-			getUrlVariables,
-			updateUrlVariable,
-		);
-	}, [getUrlVariables, updateUrlVariable, dashboardVariables]);
-
 	// Memoize the order key to avoid unnecessary triggers
 	const variableOrderKey = useMemo(() => {
 		const queryVariableOrderKey = dependencyData?.order?.join(',') ?? '';

frontend/src/container/NewWidget/index.tsx

@@ -439,6 +439,19 @@ function NewWidget({
 		globalSelectedInterval,
 	]);
 
+	const navigateToDashboardPage = useCallback(() => {
+		const params = new URLSearchParams();
+
+		const urlVariablesQueryString = query.get(QueryParams.variables);
+		if (urlVariablesQueryString) {
+			params.set(QueryParams.variables, urlVariablesQueryString);
+		}
+
+		const search = params.toString() ? `?${params.toString()}` : '';
+
+		safeNavigate(generatePath(ROUTES.DASHBOARD, { dashboardId }) + search);
+	}, [dashboardId, query, safeNavigate]);
+
 	const onClickSaveHandler = useCallback(() => {
 		if (!selectedDashboard) {
 			return;
@@ -554,9 +567,7 @@ function NewWidget({
 		updateDashboardMutation.mutateAsync(dashboard, {
 			onSuccess: () => {
 				setToScrollWidgetId(selectedWidget?.id || '');
-				safeNavigate({
-					pathname: generatePath(ROUTES.DASHBOARD, { dashboardId }),
-				});
+				navigateToDashboardPage();
 			},
 		});
 	}, [
@@ -572,7 +583,7 @@ function NewWidget({
 		updateDashboardMutation,
 		widgets,
 		setToScrollWidgetId,
-		safeNavigate,
+		navigateToDashboardPage,
 		dashboardId,
 	]);
 
@@ -581,12 +592,12 @@ function NewWidget({
 			setDiscardModal(true);
 			return;
 		}
-		safeNavigate(generatePath(ROUTES.DASHBOARD, { dashboardId }));
-	}, [dashboardId, isQueryModified, safeNavigate]);
+		navigateToDashboardPage();
+	}, [isQueryModified, navigateToDashboardPage]);
 
 	const discardChanges = useCallback(() => {
-		safeNavigate(generatePath(ROUTES.DASHBOARD, { dashboardId }));
-	}, [dashboardId, safeNavigate]);
+		navigateToDashboardPage();
+	}, [navigateToDashboardPage]);
 
 	const setGraphHandler = (type: PANEL_TYPES): void => {
 		setIsLoadingPanelData(true);
@@ -728,12 +739,14 @@ function NewWidget({
 		}
 		const widgetId = query.get('widgetId') || '';
 		const graphType = query.get('graphType') || '';
+		const variables = query.get(QueryParams.variables) || '';
 		const queryParams = {
 			[QueryParams.expandedWidgetId]: widgetId,
 			[QueryParams.graphType]: graphType,
 			[QueryParams.compositeQuery]: encodeURIComponent(
 				JSON.stringify(currentQuery),
 			),
+			[QueryParams.variables]: variables,
 		};
 
 		const updatedSearch = createQueryParams(queryParams);

frontend/src/hooks/dashboard/__test__/useTransformDashboardVariables.test.tsx

@@ -0,0 +1,339 @@
+import { renderHook } from '@testing-library/react';
+import { useDashboardVariablesFromLocalStorage } from 'hooks/dashboard/useDashboardFromLocalStorage';
+import { useTransformDashboardVariables } from 'hooks/dashboard/useTransformDashboardVariables';
+import useVariablesFromUrl from 'hooks/dashboard/useVariablesFromUrl';
+import { Dashboard, IDashboardVariable } from 'types/api/dashboard/getAll';
+
+jest.mock('hooks/dashboard/useDashboardFromLocalStorage');
+jest.mock('hooks/dashboard/useVariablesFromUrl');
+
+const mockUseDashboardVariablesFromLocalStorage = useDashboardVariablesFromLocalStorage as jest.MockedFunction<
+	typeof useDashboardVariablesFromLocalStorage
+>;
+const mockUseVariablesFromUrl = useVariablesFromUrl as jest.MockedFunction<
+	typeof useVariablesFromUrl
+>;
+
+const makeVariable = (
+	overrides: Partial<IDashboardVariable> = {},
+): IDashboardVariable => ({
+	id: 'existing-id',
+	name: 'env',
+	description: '',
+	type: 'QUERY',
+	sort: 'DISABLED',
+	multiSelect: false,
+	showALLOption: false,
+	selectedValue: 'prod',
+	...overrides,
+});
+
+const makeDashboard = (
+	variables: Record<string, IDashboardVariable>,
+): Dashboard => ({
+	id: 'dash-1',
+	createdAt: '',
+	updatedAt: '',
+	createdBy: '',
+	updatedBy: '',
+	data: {
+		title: 'Test',
+		variables,
+	},
+});
+
+const setupHook = (
+	currentDashboard: Record<string, any> = {},
+	urlVariables: Record<string, any> = {},
+): ReturnType<typeof useTransformDashboardVariables> => {
+	mockUseDashboardVariablesFromLocalStorage.mockReturnValue({
+		currentDashboard,
+		updateLocalStorageDashboardVariables: jest.fn(),
+	});
+	mockUseVariablesFromUrl.mockReturnValue({
+		getUrlVariables: () => urlVariables,
+		setUrlVariables: jest.fn(),
+		updateUrlVariable: jest.fn(),
+	});
+
+	const { result } = renderHook(() => useTransformDashboardVariables('dash-1'));
+	return result.current;
+};
+
+describe('useTransformDashboardVariables', () => {
+	beforeEach(() => jest.clearAllMocks());
+
+	describe('order assignment', () => {
+		it('assigns order starting from 0 to variables that have none', () => {
+			const { transformDashboardVariables } = setupHook();
+			const dashboard = makeDashboard({
+				v1: makeVariable({ id: 'id1', name: 'v1', order: undefined }),
+				v2: makeVariable({ id: 'id2', name: 'v2', order: undefined }),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			const orders = Object.values(result.data.variables).map((v) => v.order);
+			expect(orders).toContain(0);
+			expect(orders).toContain(1);
+		});
+
+		it('preserves existing order values', () => {
+			const { transformDashboardVariables } = setupHook();
+			const dashboard = makeDashboard({
+				v1: makeVariable({ id: 'id1', name: 'v1', order: 5 }),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.order).toBe(5);
+		});
+
+		it('assigns unique orders across multiple variables that all lack an order', () => {
+			const { transformDashboardVariables } = setupHook();
+			const dashboard = makeDashboard({
+				v1: makeVariable({ id: 'id1', name: 'v1', order: undefined }),
+				v2: makeVariable({ id: 'id2', name: 'v2', order: undefined }),
+				v3: makeVariable({ id: 'id3', name: 'v3', order: undefined }),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			const orders = Object.values(result.data.variables).map((v) => v.order);
+			// All three newly assigned orders must be distinct
+			expect(new Set(orders).size).toBe(3);
+		});
+	});
+
+	describe('ID assignment', () => {
+		it('assigns a UUID to variables that have no id', () => {
+			const { transformDashboardVariables } = setupHook();
+			const variable = makeVariable({ name: 'v1' });
+			(variable as any).id = undefined;
+			const dashboard = makeDashboard({ v1: variable });
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.id).toMatch(
+				/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,
+			);
+		});
+
+		it('preserves existing IDs', () => {
+			const { transformDashboardVariables } = setupHook();
+			const dashboard = makeDashboard({
+				v1: makeVariable({ id: 'keep-me', name: 'v1' }),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.id).toBe('keep-me');
+		});
+	});
+
+	describe('TEXTBOX backward compatibility', () => {
+		it('copies textboxValue to defaultValue when defaultValue is missing', () => {
+			const { transformDashboardVariables } = setupHook();
+			const dashboard = makeDashboard({
+				v1: makeVariable({
+					id: 'id1',
+					name: 'v1',
+					type: 'TEXTBOX',
+					textboxValue: 'hello',
+					defaultValue: undefined,
+					order: undefined,
+				}),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.defaultValue).toBe('hello');
+		});
+
+		it('does not overwrite an existing defaultValue', () => {
+			const { transformDashboardVariables } = setupHook();
+			const dashboard = makeDashboard({
+				v1: makeVariable({
+					id: 'id1',
+					name: 'v1',
+					type: 'TEXTBOX',
+					textboxValue: 'old',
+					defaultValue: 'keep',
+					order: undefined,
+				}),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.defaultValue).toBe('keep');
+		});
+	});
+
+	describe('localStorage merge', () => {
+		it('applies localStorage selectedValue over DB value', () => {
+			const { transformDashboardVariables } = setupHook({
+				env: { selectedValue: 'staging', allSelected: false },
+			});
+			const dashboard = makeDashboard({
+				v1: makeVariable({ id: 'id1', name: 'env', selectedValue: 'prod' }),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.selectedValue).toBe('staging');
+		});
+
+		it('applies localStorage allSelected over DB value', () => {
+			const { transformDashboardVariables } = setupHook({
+				env: { selectedValue: undefined, allSelected: true },
+			});
+			const dashboard = makeDashboard({
+				v1: makeVariable({
+					id: 'id1',
+					name: 'env',
+					allSelected: false,
+					showALLOption: true,
+				}),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.allSelected).toBe(true);
+		});
+	});
+
+	describe('URL variable override', () => {
+		it('sets allSelected=true when URL value is __ALL__', () => {
+			const { transformDashboardVariables } = setupHook(
+				{ env: { selectedValue: 'prod', allSelected: false } },
+				{ env: '__ALL__' },
+			);
+			const dashboard = makeDashboard({
+				v1: makeVariable({
+					id: 'id1',
+					name: 'env',
+					showALLOption: true,
+					allSelected: false,
+				}),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.allSelected).toBe(true);
+		});
+
+		it('sets selectedValue from URL and clears allSelected when showALLOption is true', () => {
+			const { transformDashboardVariables } = setupHook(
+				{ env: { selectedValue: undefined, allSelected: true } },
+				{ env: 'dev' },
+			);
+			const dashboard = makeDashboard({
+				v1: makeVariable({
+					id: 'id1',
+					name: 'env',
+					showALLOption: true,
+					allSelected: true,
+					multiSelect: false,
+				}),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.selectedValue).toBe('dev');
+			expect(result.data.variables.v1.allSelected).toBe(false);
+		});
+
+		it('does not set allSelected=false when showALLOption is false', () => {
+			const { transformDashboardVariables } = setupHook(
+				{ env: { selectedValue: undefined, allSelected: true } },
+				{ env: 'dev' },
+			);
+			const dashboard = makeDashboard({
+				v1: makeVariable({
+					id: 'id1',
+					name: 'env',
+					showALLOption: false,
+					allSelected: true,
+					multiSelect: false,
+				}),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.selectedValue).toBe('dev');
+			expect(result.data.variables.v1.allSelected).toBe(true);
+		});
+
+		it('normalizes array URL value to single value for single-select variable', () => {
+			const { transformDashboardVariables } = setupHook(
+				{},
+				{ env: ['prod', 'dev'] },
+			);
+			const dashboard = makeDashboard({
+				v1: makeVariable({
+					id: 'id1',
+					name: 'env',
+					multiSelect: false,
+				}),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.selectedValue).toBe('prod');
+		});
+
+		it('wraps single URL value in array for multi-select variable', () => {
+			const { transformDashboardVariables } = setupHook({}, { env: 'prod' });
+			const dashboard = makeDashboard({
+				v1: makeVariable({
+					id: 'id1',
+					name: 'env',
+					multiSelect: true,
+				}),
+			});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.selectedValue).toEqual(['prod']);
+		});
+
+		it('looks up URL variable by variable id when name is absent', () => {
+			const { transformDashboardVariables } = setupHook(
+				{},
+				{ 'var-uuid': 'fallback' },
+			);
+			const variable = makeVariable({ id: 'var-uuid', multiSelect: false });
+			delete variable.name;
+			const dashboard = makeDashboard({ v1: variable });
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables.v1.selectedValue).toBe('fallback');
+		});
+	});
+
+	describe('edge cases', () => {
+		it('returns data unchanged when there are no variables', () => {
+			const { transformDashboardVariables } = setupHook();
+			const dashboard = makeDashboard({});
+
+			const result = transformDashboardVariables(dashboard);
+
+			expect(result.data.variables).toEqual({});
+		});
+
+		it('does not mutate the original dashboard', () => {
+			const { transformDashboardVariables } = setupHook({
+				env: { selectedValue: 'staging', allSelected: false },
+			});
+			const dashboard = makeDashboard({
+				v1: makeVariable({ id: 'id1', name: 'env', selectedValue: 'prod' }),
+			});
+			const originalValue = dashboard.data.variables.v1.selectedValue;
+
+			transformDashboardVariables(dashboard);
+
+			expect(dashboard.data.variables.v1.selectedValue).toBe(originalValue);
+		});
+	});
+});

frontend/src/hooks/dashboard/useDashboardFromLocalStorage.tsx

@@ -15,7 +15,7 @@ interface DashboardLocalStorageVariables {
 	[id: string]: LocalStoreDashboardVariables;
 }
 
-interface UseDashboardVariablesFromLocalStorageReturn {
+export interface UseDashboardVariablesFromLocalStorageReturn {
 	currentDashboard: LocalStoreDashboardVariables;
 	updateLocalStorageDashboardVariables: (
 		id: string,

frontend/src/hooks/dashboard/useTransformDashboardVariables.ts

@@ -0,0 +1,128 @@
+import { ALL_SELECTED_VALUE } from 'components/NewSelect/utils';
+import {
+	useDashboardVariablesFromLocalStorage,
+	UseDashboardVariablesFromLocalStorageReturn,
+} from 'hooks/dashboard/useDashboardFromLocalStorage';
+import useVariablesFromUrl, {
+	UseVariablesFromUrlReturn,
+} from 'hooks/dashboard/useVariablesFromUrl';
+import { isEmpty } from 'lodash-es';
+import { normalizeUrlValueForVariable } from 'providers/Dashboard/normalizeUrlValue';
+import { Dashboard, IDashboardVariable } from 'types/api/dashboard/getAll';
+import { v4 as generateUUID } from 'uuid';
+
+export function useTransformDashboardVariables(
+	dashboardId: string,
+): Pick<UseVariablesFromUrlReturn, 'getUrlVariables' | 'updateUrlVariable'> &
+	UseDashboardVariablesFromLocalStorageReturn & {
+		transformDashboardVariables: (data: Dashboard) => Dashboard;
+	} {
+	const {
+		currentDashboard,
+		updateLocalStorageDashboardVariables,
+	} = useDashboardVariablesFromLocalStorage(dashboardId);
+	const { getUrlVariables, updateUrlVariable } = useVariablesFromUrl();
+
+	const mergeDBWithLocalStorage = (
+		data: Dashboard,
+		localStorageVariables: any,
+	): Dashboard => {
+		const updatedData = data;
+		if (data && localStorageVariables) {
+			const updatedVariables = data.data.variables;
+			const variablesFromUrl = getUrlVariables();
+			Object.keys(data.data.variables).forEach((variable) => {
+				const variableData = data.data.variables[variable];
+
+				// values from url
+				const urlVariable = variableData?.name
+					? variablesFromUrl[variableData?.name] || variablesFromUrl[variableData.id]
+					: variablesFromUrl[variableData.id];
+
+				let updatedVariable = {
+					...data.data.variables[variable],
+					...localStorageVariables[variableData.name as any],
+				};
+
+				// respect the url variable if it is set, override the others
+				if (!isEmpty(urlVariable)) {
+					if (urlVariable === ALL_SELECTED_VALUE) {
+						updatedVariable = {
+							...updatedVariable,
+							allSelected: true,
+						};
+					} else {
+						// Normalize URL value to match variable's multiSelect configuration
+						const normalizedValue = normalizeUrlValueForVariable(
+							urlVariable,
+							variableData,
+						);
+
+						updatedVariable = {
+							...updatedVariable,
+							selectedValue: normalizedValue,
+							// Only set allSelected to false if showALLOption is available
+							...(updatedVariable?.showALLOption && { allSelected: false }),
+						};
+					}
+				}
+
+				updatedVariables[variable] = updatedVariable;
+			});
+			updatedData.data.variables = updatedVariables;
+		}
+		return updatedData;
+	};
+
+	// As we do not have order and ID's in the variables object, we have to process variables to add order and ID if they do not exist in the variables object
+	// eslint-disable-next-line sonarjs/cognitive-complexity
+	const transformDashboardVariables = (data: Dashboard): Dashboard => {
+		if (data && data.data && data.data.variables) {
+			const clonedDashboardData = mergeDBWithLocalStorage(
+				JSON.parse(JSON.stringify(data)),
+				currentDashboard,
+			);
+			const { variables } = clonedDashboardData.data;
+			const existingOrders: Set<number> = new Set();
+
+			for (const key in variables) {
+				// eslint-disable-next-line no-prototype-builtins
+				if (variables.hasOwnProperty(key)) {
+					const variable: IDashboardVariable = variables[key];
+
+					// Check if 'order' property doesn't exist or is undefined
+					if (variable.order === undefined) {
+						// Find a unique order starting from 0
+						let order = 0;
+						while (existingOrders.has(order)) {
+							order += 1;
+						}
+
+						variable.order = order;
+						existingOrders.add(order);
+						// ! BWC - Specific case for backward compatibility where textboxValue was used instead of defaultValue
+						if (variable.type === 'TEXTBOX' && !variable.defaultValue) {
+							variable.defaultValue = variable.textboxValue || '';
+						}
+					}
+
+					if (variable.id === undefined) {
+						variable.id = generateUUID();
+					}
+				}
+			}
+
+			return clonedDashboardData;
+		}
+
+		return data;
+	};
+
+	return {
+		transformDashboardVariables,
+		getUrlVariables,
+		updateUrlVariable,
+		currentDashboard,
+		updateLocalStorageDashboardVariables,
+	};
+}

frontend/src/hooks/dashboard/useVariablesFromUrl.tsx

@@ -11,7 +11,7 @@ export interface LocalStoreDashboardVariables {
 		| IDashboardVariable['selectedValue'];
 }
 
-interface UseVariablesFromUrlReturn {
+export interface UseVariablesFromUrlReturn {
 	getUrlVariables: () => LocalStoreDashboardVariables;
 	setUrlVariables: (variables: LocalStoreDashboardVariables) => void;
 	updateUrlVariable: (

frontend/src/pages/DashboardWidget/__tests__/DashboardWidget.test.tsx

@@ -0,0 +1,146 @@
+import { Route } from 'react-router-dom';
+import { PANEL_TYPES } from 'constants/queryBuilder';
+import { rest, server } from 'mocks-server/server';
+import { render, screen, waitFor } from 'tests/test-utils';
+
+import DashboardWidget from '../index';
+
+const DASHBOARD_ID = 'dash-1';
+const WIDGET_ID = 'widget-abc';
+
+const mockDashboardResponse = {
+	status: 'success',
+	data: {
+		id: DASHBOARD_ID,
+		createdAt: '2024-01-01T00:00:00Z',
+		createdBy: 'test',
+		updatedAt: '2024-01-01T00:00:00Z',
+		updatedBy: 'test',
+		isLocked: false,
+		data: {
+			collapsableRowsMigrated: true,
+			description: '',
+			name: '',
+			panelMap: {},
+			tags: [],
+			title: 'Test Dashboard',
+			uploadedGrafana: false,
+			uuid: '',
+			version: '',
+			variables: {},
+			widgets: [],
+			layout: [],
+		},
+	},
+};
+
+const mockSafeNavigate = jest.fn();
+
+jest.mock('hooks/useSafeNavigate', () => ({
+	useSafeNavigate: (): { safeNavigate: jest.Mock } => ({
+		safeNavigate: mockSafeNavigate,
+	}),
+}));
+
+jest.mock('container/NewWidget', () => ({
+	__esModule: true,
+	default: (): JSX.Element => <div data-testid="new-widget">NewWidget</div>,
+}));
+
+// nuqs's useQueryState doesn't read from MemoryRouter, so we mock it to return
+// controlled values via the `mockQueryState` map below.
+const mockQueryState: Record<string, string | null> = {};
+
+jest.mock('nuqs', () => ({
+	...jest.requireActual('nuqs'),
+	useQueryState: (key: string): [string | null, jest.Mock] => [
+		mockQueryState[key] ?? null,
+		jest.fn(),
+	],
+}));
+
+// Wrap component in a Route so useParams can resolve dashboardId
+function renderAtRoute(
+	queryState: Record<string, string | null> = {},
+): ReturnType<typeof render> {
+	Object.assign(mockQueryState, queryState);
+	return render(
+		<Route path="/dashboard/:dashboardId/new">
+			<DashboardWidget />
+		</Route>,
+		undefined,
+		{ initialRoute: `/dashboard/${DASHBOARD_ID}/new` },
+	);
+}
+
+beforeEach(() => {
+	mockSafeNavigate.mockClear();
+	Object.keys(mockQueryState).forEach((k) => delete mockQueryState[k]);
+});
+
+describe('DashboardWidget', () => {
+	it('redirects to dashboard when widgetId is missing', async () => {
+		renderAtRoute({ graphType: PANEL_TYPES.TIME_SERIES });
+
+		await waitFor(() => {
+			expect(mockSafeNavigate).toHaveBeenCalled();
+		});
+
+		const [navigatedTo] = mockSafeNavigate.mock.calls[0];
+		expect(navigatedTo).toContain(`/dashboard/${DASHBOARD_ID}`);
+	});
+
+	it('redirects to dashboard when graphType is missing', async () => {
+		renderAtRoute({ widgetId: WIDGET_ID });
+
+		await waitFor(() => {
+			expect(mockSafeNavigate).toHaveBeenCalled();
+		});
+
+		const [navigatedTo] = mockSafeNavigate.mock.calls[0];
+		expect(navigatedTo).toContain(`/dashboard/${DASHBOARD_ID}`);
+	});
+
+	it('shows spinner while dashboard is loading', () => {
+		server.use(
+			rest.get(
+				`http://localhost/api/v1/dashboards/${DASHBOARD_ID}`,
+				(_req, res, ctx) => res(ctx.delay('infinite')),
+			),
+		);
+
+		renderAtRoute({ widgetId: WIDGET_ID, graphType: PANEL_TYPES.TIME_SERIES });
+
+		expect(screen.getByRole('img', { name: 'loading' })).toBeInTheDocument();
+	});
+
+	it('shows error message when dashboard fetch fails', async () => {
+		server.use(
+			rest.get(
+				`http://localhost/api/v1/dashboards/${DASHBOARD_ID}`,
+				(_req, res, ctx) => res(ctx.status(500), ctx.json({ status: 'error' })),
+			),
+		);
+
+		renderAtRoute({ widgetId: WIDGET_ID, graphType: PANEL_TYPES.TIME_SERIES });
+
+		await waitFor(() => {
+			expect(screen.getByText('Something went wrong')).toBeInTheDocument();
+		});
+	});
+
+	it('renders NewWidget when dashboard loads successfully', async () => {
+		server.use(
+			rest.get(
+				`http://localhost/api/v1/dashboards/${DASHBOARD_ID}`,
+				(_req, res, ctx) => res(ctx.status(200), ctx.json(mockDashboardResponse)),
+			),
+		);
+
+		renderAtRoute({ widgetId: WIDGET_ID, graphType: PANEL_TYPES.TIME_SERIES });
+
+		await waitFor(() => {
+			expect(screen.getByTestId('new-widget')).toBeInTheDocument();
+		});
+	});
+});

frontend/src/pages/DashboardWidget/index.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useMemo } from 'react';
+import { useEffect, useState } from 'react';
 import { useQuery } from 'react-query';
 import { generatePath, useParams } from 'react-router-dom';
 import { Card, Typography } from 'antd';
@@ -11,9 +11,11 @@ import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
 import ROUTES from 'constants/routes';
 import NewWidget from 'container/NewWidget';
 import { isDrilldownEnabled } from 'container/QueryTable/Drilldown/drilldownUtils';
+import { useTransformDashboardVariables } from 'hooks/dashboard/useTransformDashboardVariables';
 import { useSafeNavigate } from 'hooks/useSafeNavigate';
 import { parseAsStringEnum, useQueryState } from 'nuqs';
 import { setDashboardVariablesStore } from 'providers/Dashboard/store/dashboardVariables/dashboardVariablesStore';
+import { Dashboard } from 'types/api/dashboard/getAll';
 
 function DashboardWidget(): JSX.Element | null {
 	const { dashboardId } = useParams<{
@@ -57,8 +59,15 @@ function DashboardWidgetInternal({
 	widgetId: string;
 	graphType: PANEL_TYPES;
 }): JSX.Element | null {
+	const [selectedDashboard, setSelectedDashboard] = useState<
+		Dashboard | undefined
+	>(undefined);
+
+	const { transformDashboardVariables } = useTransformDashboardVariables(
+		dashboardId,
+	);
+
 	const {
-		data: dashboardResponse,
 		isFetching: isFetchingDashboardResponse,
 		isError: isErrorDashboardResponse,
 	} = useQuery([REACT_QUERY_KEY.DASHBOARD_BY_ID, dashboardId, widgetId], {
@@ -70,17 +79,15 @@ function DashboardWidgetInternal({
 		refetchOnWindowFocus: false,
 		cacheTime: DASHBOARD_CACHE_TIME,
 		onSuccess: (response) => {
+			const updatedDashboardData = transformDashboardVariables(response.data);
+			setSelectedDashboard(updatedDashboardData);
 			setDashboardVariablesStore({
 				dashboardId,
-				variables: response.data.data.variables,
+				variables: updatedDashboardData.data.variables,
 			});
 		},
 	});
 
-	const selectedDashboard = useMemo(() => dashboardResponse?.data, [
-		dashboardResponse?.data,
-	]);
-
 	if (isFetchingDashboardResponse) {
 		return <Spinner tip="Loading.." />;
 	}

frontend/src/providers/Dashboard/Dashboard.tsx

@@ -17,21 +17,18 @@ import { useDispatch, useSelector } from 'react-redux';
 import { Modal } from 'antd';
 import getDashboard from 'api/v1/dashboards/id/get';
 import locked from 'api/v1/dashboards/id/lock';
-import { ALL_SELECTED_VALUE } from 'components/NewSelect/utils';
 import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
 import dayjs, { Dayjs } from 'dayjs';
-import { useDashboardVariablesFromLocalStorage } from 'hooks/dashboard/useDashboardFromLocalStorage';
-import useVariablesFromUrl from 'hooks/dashboard/useVariablesFromUrl';
+import { useTransformDashboardVariables } from 'hooks/dashboard/useTransformDashboardVariables';
 import useTabVisibility from 'hooks/useTabFocus';
 import { getUpdatedLayout } from 'lib/dashboard/getUpdatedLayout';
 import { getMinMaxForSelectedTime } from 'lib/getMinMax';
-import { defaultTo, isEmpty } from 'lodash-es';
+import { defaultTo } from 'lodash-es';
 import isEqual from 'lodash-es/isEqual';
 import isUndefined from 'lodash-es/isUndefined';
 import omitBy from 'lodash-es/omitBy';
 import { useAppContext } from 'providers/App/App';
 import { initializeDefaultVariables } from 'providers/Dashboard/initializeDefaultVariables';
-import { normalizeUrlValueForVariable } from 'providers/Dashboard/normalizeUrlValue';
 import { useErrorModal } from 'providers/ErrorModalProvider';
 // eslint-disable-next-line no-restricted-imports
 import { Dispatch } from 'redux';
@@ -39,10 +36,9 @@ import { AppState } from 'store/reducers';
 import AppActions from 'types/actions';
 import { UPDATE_TIME_INTERVAL } from 'types/actions/globalTime';
 import { SuccessResponseV2 } from 'types/api';
-import { Dashboard, IDashboardVariable } from 'types/api/dashboard/getAll';
+import { Dashboard } from 'types/api/dashboard/getAll';
 import APIError from 'types/api/error';
 import { GlobalReducer } from 'types/reducer/globalTime';
-import { v4 as generateUUID } from 'uuid';
 
 import {
 	DASHBOARD_CACHE_TIME,
@@ -137,9 +133,10 @@ export function DashboardProvider({
 	const {
 		currentDashboard,
 		updateLocalStorageDashboardVariables,
-	} = useDashboardVariablesFromLocalStorage(dashboardId);
-
-	const { getUrlVariables, updateUrlVariable } = useVariablesFromUrl();
+		getUrlVariables,
+		updateUrlVariable,
+		transformDashboardVariables,
+	} = useTransformDashboardVariables(dashboardId);
 
 	const updatedTimeRef = useRef<Dayjs | null>(null); // Using ref to store the updated time
 	const modalRef = useRef<any>(null);
@@ -151,99 +148,6 @@ export function DashboardProvider({
 
 	const [isDashboardFetching, setIsDashboardFetching] = useState<boolean>(false);
 
-	const mergeDBWithLocalStorage = (
-		data: Dashboard,
-		localStorageVariables: any,
-	): Dashboard => {
-		const updatedData = data;
-		if (data && localStorageVariables) {
-			const updatedVariables = data.data.variables;
-			const variablesFromUrl = getUrlVariables();
-			Object.keys(data.data.variables).forEach((variable) => {
-				const variableData = data.data.variables[variable];
-
-				// values from url
-				const urlVariable = variableData?.name
-					? variablesFromUrl[variableData?.name] || variablesFromUrl[variableData.id]
-					: variablesFromUrl[variableData.id];
-
-				let updatedVariable = {
-					...data.data.variables[variable],
-					...localStorageVariables[variableData.name as any],
-				};
-
-				// respect the url variable if it is set, override the others
-				if (!isEmpty(urlVariable)) {
-					if (urlVariable === ALL_SELECTED_VALUE) {
-						updatedVariable = {
-							...updatedVariable,
-							allSelected: true,
-						};
-					} else {
-						// Normalize URL value to match variable's multiSelect configuration
-						const normalizedValue = normalizeUrlValueForVariable(
-							urlVariable,
-							variableData,
-						);
-
-						updatedVariable = {
-							...updatedVariable,
-							selectedValue: normalizedValue,
-							// Only set allSelected to false if showALLOption is available
-							...(updatedVariable?.showALLOption && { allSelected: false }),
-						};
-					}
-				}
-
-				updatedVariables[variable] = updatedVariable;
-			});
-			updatedData.data.variables = updatedVariables;
-		}
-		return updatedData;
-	};
-	// As we do not have order and ID's in the variables object, we have to process variables to add order and ID if they do not exist in the variables object
-	// eslint-disable-next-line sonarjs/cognitive-complexity
-	const transformDashboardVariables = (data: Dashboard): Dashboard => {
-		if (data && data.data && data.data.variables) {
-			const clonedDashboardData = mergeDBWithLocalStorage(
-				JSON.parse(JSON.stringify(data)),
-				currentDashboard,
-			);
-			const { variables } = clonedDashboardData.data;
-			const existingOrders: Set<number> = new Set();
-
-			for (const key in variables) {
-				// eslint-disable-next-line no-prototype-builtins
-				if (variables.hasOwnProperty(key)) {
-					const variable: IDashboardVariable = variables[key];
-
-					// Check if 'order' property doesn't exist or is undefined
-					if (variable.order === undefined) {
-						// Find a unique order starting from 0
-						let order = 0;
-						while (existingOrders.has(order)) {
-							order += 1;
-						}
-
-						variable.order = order;
-						existingOrders.add(order);
-						// ! BWC - Specific case for backward compatibility where textboxValue was used instead of defaultValue
-						if (variable.type === 'TEXTBOX' && !variable.defaultValue) {
-							variable.defaultValue = variable.textboxValue || '';
-						}
-					}
-
-					if (variable.id === undefined) {
-						variable.id = generateUUID();
-					}
-				}
-			}
-
-			return clonedDashboardData;
-		}
-
-		return data;
-	};
 	const dashboardResponse = useQuery(
 		[
 			REACT_QUERY_KEY.DASHBOARD_BY_ID,
@@ -274,13 +178,14 @@ export function DashboardProvider({
 			},
 
 			onSuccess: (data: SuccessResponseV2<Dashboard>) => {
-				// if the url variable is not set for any variable, set it to the default value
-				const variables = data?.data?.data?.variables;
+				const updatedDashboardData = transformDashboardVariables(data?.data);
+
+				// initialize URL variables after dashboard state is set to avoid race conditions
+				const variables = updatedDashboardData?.data?.variables;
 				if (variables) {
 					initializeDefaultVariables(variables, getUrlVariables, updateUrlVariable);
 				}
 
-				const updatedDashboardData = transformDashboardVariables(data?.data);
 				const updatedDate = dayjs(updatedDashboardData?.updatedAt);
 
 				setIsDashboardLocked(updatedDashboardData?.locked || false);

frontend/src/providers/Dashboard/__tests__/Dashboard.test.tsx

@@ -381,6 +381,7 @@ describe('Dashboard Provider - URL Variables Integration', () => {
 							multiSelect: false,
 							allSelected: false,
 							showALLOption: true,
+							order: 0,
 						},
 						services: {
 							id: 'svc-id',
@@ -388,6 +389,7 @@ describe('Dashboard Provider - URL Variables Integration', () => {
 							multiSelect: true,
 							allSelected: false,
 							showALLOption: true,
+							order: 1,
 						},
 					},
 					mockGetUrlVariables,

pkg/telemetrytraces/const.go

@@ -1,6 +1,50 @@
 package telemetrytraces
 
-import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+import (
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+const (
+
+	// Internal Columns
+	SpanTimestampBucketStartColumn = "ts_bucket_start"
+	SpanResourceFingerPrintColumn  = "resource_fingerprint"
+
+	// Intrinsic Columns
+	SpanTimestampColumn        = "timestamp"
+	SpanTraceIDColumn          = "trace_id"
+	SpanSpanIDColumn           = "span_id"
+	SpanTraceStateColumn       = "trace_state"
+	SpanParentSpanIDColumn     = "parent_span_id"
+	SpanFlagsColumn            = "flags"
+	SpanNameColumn             = "name"
+	SpanKindColumn             = "kind"
+	SpanKindStringColumn       = "kind_string"
+	SpanDurationNanoColumn     = "duration_nano"
+	SpanStatusCodeColumn       = "status_code"
+	SpanStatusMessageColumn    = "status_message"
+	SpanStatusCodeStringColumn = "status_code_string"
+	SpanEventsColumn           = "events"
+	SpanLinksColumn            = "links"
+
+	// Calculated Columns
+	SpanResponseStatusCodeColumn = "response_status_code"
+	SpanExternalHTTPURLColumn    = "external_http_url"
+	SpanHTTPURLColumn            = "http_url"
+	SpanExternalHTTPMethodColumn = "external_http_method"
+	SpanHTTPMethodColumn         = "http_method"
+	SpanHTTPHostColumn           = "http_host"
+	SpanDBNameColumn             = "db_name"
+	SpanDBOperationColumn        = "db_operation"
+	SpanHasErrorColumn           = "has_error"
+	SpanIsRemoteColumn           = "is_remote"
+
+	// Contextual Columns
+	SpanAttributesStringColumn = "attributes_string"
+	SpanAttributesNumberColumn = "attributes_number"
+	SpanAttributesBoolColumn   = "attributes_bool"
+	SpanResourcesStringColumn  = "resources_string"
+)
 
 var (
 	IntrinsicFields = map[string]telemetrytypes.TelemetryFieldKey{

pkg/telemetrytraces/statement_builder.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
-	"slices"
 	"strings"
 
 	"github.com/SigNoz/signoz/pkg/errors"
@@ -14,7 +13,6 @@ import (
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/huandu/go-sqlbuilder"
-	"golang.org/x/exp/maps"
 )
 
 var (
@@ -74,41 +72,6 @@ func (b *traceQueryStatementBuilder) Build(
 		return nil, err
 	}
 
-	/*
-		Adding a tech debt note here:
-		This piece of code is a hot fix and should be removed once we close issue: engineering-pod/issues/3622
-	*/
-	/*
-		-------------------------------- Start of tech debt ----------------------------
-	*/
-	if requestType == qbtypes.RequestTypeRaw {
-
-		selectedFields := query.SelectFields
-
-		if len(selectedFields) == 0 {
-			sortedKeys := maps.Keys(DefaultFields)
-			slices.Sort(sortedKeys)
-			for _, key := range sortedKeys {
-				selectedFields = append(selectedFields, DefaultFields[key])
-			}
-			query.SelectFields = selectedFields
-		}
-
-		selectFieldKeys := []string{}
-		for _, field := range selectedFields {
-			selectFieldKeys = append(selectFieldKeys, field.Name)
-		}
-
-		for _, x := range []string{"timestamp", "span_id", "trace_id"} {
-			if !slices.Contains(selectFieldKeys, x) {
-				query.SelectFields = append(query.SelectFields, DefaultFields[x])
-			}
-		}
-	}
-	/*
-		-------------------------------- End of tech debt ----------------------------
-	*/
-
 	query = b.adjustKeys(ctx, keys, query, requestType)
 
 	// Check if filter contains trace_id(s) and optimize time range if needed
@@ -311,13 +274,53 @@ func (b *traceQueryStatementBuilder) buildListQuery(
 		cteArgs = append(cteArgs, args)
 	}
 
-	// TODO: should we deprecate `SelectFields` and return everything from a span like we do for logs?
-	for _, field := range query.SelectFields {
-		colExpr, err := b.fm.ColumnExpressionFor(ctx, &field, keys)
-		if err != nil {
-			return nil, err
+	sb.SelectMore(SpanTimestampColumn)
+	sb.SelectMore(SpanTraceIDColumn)
+	sb.SelectMore(SpanSpanIDColumn)
+	// By default select all
+	if len(query.SelectFields) == 0 {
+		// Select all intrinsic columns
+		sb.SelectMore(SpanTraceStateColumn)
+		sb.SelectMore(SpanParentSpanIDColumn)
+		sb.SelectMore(SpanFlagsColumn)
+		sb.SelectMore(SpanNameColumn)
+		sb.SelectMore(SpanKindColumn)
+		sb.SelectMore(SpanKindStringColumn)
+		sb.SelectMore(SpanDurationNanoColumn)
+		sb.SelectMore(SpanStatusCodeColumn)
+		sb.SelectMore(SpanStatusMessageColumn)
+		sb.SelectMore(SpanStatusCodeStringColumn)
+		sb.SelectMore(SpanEventsColumn)
+		sb.SelectMore(SpanLinksColumn)
+
+		// select all calculated columns
+		sb.SelectMore(SpanResponseStatusCodeColumn)
+		sb.SelectMore(SpanExternalHTTPURLColumn)
+		sb.SelectMore(SpanHTTPURLColumn)
+		sb.SelectMore(SpanExternalHTTPMethodColumn)
+		sb.SelectMore(SpanHTTPMethodColumn)
+		sb.SelectMore(SpanHTTPHostColumn)
+		sb.SelectMore(SpanDBNameColumn)
+		sb.SelectMore(SpanDBOperationColumn)
+		sb.SelectMore(SpanHasErrorColumn)
+		sb.SelectMore(SpanIsRemoteColumn)
+
+		// select all contextual columns
+		sb.SelectMore(SpanAttributesStringColumn)
+		sb.SelectMore(SpanAttributesNumberColumn)
+		sb.SelectMore(SpanAttributesBoolColumn)
+		sb.SelectMore(SpanResourcesStringColumn)
+	} else {
+		for _, field := range query.SelectFields {
+			if field.Name == SpanTimestampColumn || field.Name == SpanTraceIDColumn || field.Name == SpanSpanIDColumn {
+				continue
+			}
+			colExpr, err := b.fm.ColumnExpressionFor(ctx, &field, keys)
+			if err != nil {
+				return nil, err
+			}
+			sb.SelectMore(colExpr)
 		}
-		sb.SelectMore(colExpr)
 	}
 
 	// From table