Merge branch 'main' into temp/local_clickhouse_setup

* adding LLM Observability ref to README

* Update README.md

.devenv/docker/clickhouse/compose.yaml

@@ -1,15 +1,35 @@
 services:
+  init-clickhouse:
+    image: clickhouse/clickhouse-server:25.5.6
+    container_name: init-clickhouse
+    command:
+      - bash
+      - -c
+      - |
+        version="v0.0.1"
+        node_os=$$(uname -s | tr '[:upper:]' '[:lower:]')
+        node_arch=$$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/)
+        echo "Fetching histogram-binary for $${node_os}/$${node_arch}"
+        cd /tmp
+        wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F$${version}/histogram-quantile_$${node_os}_$${node_arch}.tar.gz"
+        tar -xvzf histogram-quantile.tar.gz
+        mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile
+        chmod +x /var/lib/clickhouse/user_scripts/histogramQuantile
+    volumes:
+      - ${PWD}/fs/tmp/var/lib/clickhouse/user_scripts/:/var/lib/clickhouse/user_scripts/
+    restart: on-failure
   clickhouse:
     image: clickhouse/clickhouse-server:25.5.6
     container_name: clickhouse
     volumes:
       - ${PWD}/fs/etc/clickhouse-server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml
+      - ${PWD}/fs/etc/clickhouse-server/config.d/custom-function.xml:/etc/clickhouse-server/custom-function.xml
       - ${PWD}/fs/etc/clickhouse-server/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml
       - ${PWD}/fs/tmp/var/lib/clickhouse/:/var/lib/clickhouse/
       - ${PWD}/fs/tmp/var/lib/clickhouse/user_scripts/:/var/lib/clickhouse/user_scripts/
     ports:
-      - '127.0.0.1:8123:8123'
-      - '127.0.0.1:9000:9000'
+      - '0.0.0.0:8123:8123'
+      - '0.0.0.0:9000:9000'
     tty: true
     healthcheck:
       test:
@@ -23,6 +43,7 @@ services:
       retries: 3
     depends_on:
       - zookeeper
+      - init-clickhouse
     environment:
       - CLICKHOUSE_SKIP_USER_SETUP=1
   zookeeper:

.devenv/docker/clickhouse/fs/etc/clickhouse-server/config.d/config.xml

@@ -44,4 +44,6 @@
         <shard>01</shard>
         <replica>01</replica>
     </macros>
+    <!-- Configuration of user defined executable functions -->
+    <user_defined_executable_functions_config>*function.xml</user_defined_executable_functions_config>
 </clickhouse>

.devenv/docker/clickhouse/fs/etc/clickhouse-server/config.d/custom-function.xml

@@ -0,0 +1,22 @@
+<functions>
+    <function>
+        <type>executable</type>
+        <name>histogramQuantile</name>
+        <return_type>Float64</return_type>
+        <argument>
+            <type>Array(Float64)</type>
+            <name>buckets</name>
+        </argument>
+        <argument>
+            <type>Array(Float64)</type>
+            <name>counts</name>
+        </argument>
+        <argument>
+            <type>Float64</type>
+            <name>quantile</name>
+        </argument>
+        <format>CSV</format>
+        <command>./histogramQuantile</command>
+    </function>
+</functions>
+

README.md

@@ -66,6 +66,17 @@ Read [more](https://signoz.io/metrics-and-dashboards/).
 
 ![metrics-n-dashboards-cover](https://github.com/user-attachments/assets/a536fd71-1d2c-4681-aa7e-516d754c47a5)
 
+### LLM Observability
+
+Monitor and debug your LLM applications with comprehensive observability. Track LLM calls, analyze token usage, monitor performance, and gain insights into your AI application's behavior in production.
+
+SigNoz LLM observability helps you understand how your language models are performing, identify issues with prompts and responses, track token usage and costs, and optimize your AI applications for better performance and reliability.
+
+[Get started with LLM Observability →](https://signoz.io/docs/llm-observability/)
+
+![llm-observability-cover](https://github.com/user-attachments/assets/a6cc0ca3-59df-48f9-9c16-7c843fccff96)
+
+
 ### Alerts
 
 Use alerts in SigNoz to get notified when anything unusual happens in your application. You can set alerts on any type of telemetry signal (logs, metrics, traces), create thresholds and set up a notification channel to get notified. Advanced features like alert history and anomaly detection can help you create smarter alerts.

frontend/package.json

@@ -105,6 +105,7 @@
 		"i18next": "^21.6.12",
 		"i18next-browser-languagedetector": "^6.1.3",
 		"i18next-http-backend": "^1.3.2",
+		"immer": "11.1.3",
 		"jest": "^27.5.1",
 		"js-base64": "^3.7.2",
 		"less": "^4.1.2",

frontend/src/container/OnboardingV2Container/AddDataSource/AddDataSource.tsx

@@ -15,6 +15,7 @@ import logEvent from 'api/common/logEvent';
 import LaunchChatSupport from 'components/LaunchChatSupport/LaunchChatSupport';
 import { DOCS_BASE_URL } from 'constants/app';
 import ROUTES from 'constants/routes';
+import { useGetGlobalConfig } from 'hooks/globalConfig/useGetGlobalConfig';
 import useDebouncedFn from 'hooks/useDebouncedFunction';
 import history from 'lib/history';
 import { isEmpty } from 'lodash-es';
@@ -148,6 +149,8 @@ function OnboardingAddDataSource(): JSX.Element {
 
 	const { org } = useAppContext();
 
+	const { data: globalConfig } = useGetGlobalConfig();
+
 	const [setupStepItems, setSetupStepItems] = useState(setupStepItemsBase);
 
 	const [searchQuery, setSearchQuery] = useState<string>('');
@@ -233,6 +236,16 @@ function OnboardingAddDataSource(): JSX.Element {
 			urlObj.searchParams.set('environment', selectedEnvironment);
 		}
 
+		const ingestionUrl = globalConfig?.data?.ingestion_url;
+
+		if (ingestionUrl) {
+			const parts = ingestionUrl.split('.');
+			if (parts?.length > 1 && parts[0]?.includes('ingest')) {
+				const region = parts[1];
+				urlObj.searchParams.set('region', region);
+			}
+		}
+
 		// Step 3: Return the updated URL as a string
 		const updatedUrl = urlObj.toString();
 

frontend/src/container/PipelinePage/tests/PipelineListsView.test.tsx

@@ -1,7 +1,17 @@
 /* eslint-disable sonarjs/no-duplicate-string */
-import { screen } from '@testing-library/react';
+import { screen, within } from '@testing-library/react';
+import { ENVIRONMENT } from 'constants/env';
+import { server } from 'mocks-server/server';
+import { rest } from 'msw';
 import { PreferenceContextProvider } from 'providers/preferences/context/PreferenceContextProvider';
-import { findByText, fireEvent, render, waitFor } from 'tests/test-utils';
+import {
+	findByText,
+	fireEvent,
+	render,
+	userEvent,
+	waitFor,
+} from 'tests/test-utils';
+import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
 
 import { pipelineApiResponseMockData } from '../mocks/pipeline';
 import PipelineListsView from '../PipelineListsView';
@@ -75,7 +85,20 @@ jest.mock('providers/preferences/sync/usePreferenceSync', () => ({
 	}),
 }));
 
+const BASE_URL = ENVIRONMENT.baseURL;
+const attributeKeysURL = `${BASE_URL}/api/v3/autocomplete/attribute_keys`;
+
 describe('PipelinePage container test', () => {
+	beforeAll(() => {
+		server.listen();
+	});
+	afterEach(() => {
+		server.resetHandlers();
+		jest.clearAllMocks();
+	});
+	afterAll(() => {
+		server.close();
+	});
 	it('should render PipelineListsView section', () => {
 		const { getByText, container } = render(
 			<PreferenceContextProvider>
@@ -272,6 +295,7 @@ describe('PipelinePage container test', () => {
 	});
 
 	it('should have populated form fields when edit pipeline is clicked', async () => {
+		const user = userEvent.setup({ pointerEventsCheck: 0 });
 		render(
 			<PreferenceContextProvider>
 				<PipelineListsView
@@ -301,5 +325,52 @@ describe('PipelinePage container test', () => {
 
 		// to have length 2
 		expect(screen.queryAllByText('source = nginx').length).toBe(2);
+
+		server.use(
+			rest.get(attributeKeysURL, (_req, res, ctx) =>
+				res(
+					ctx.status(200),
+					ctx.json({
+						status: 'success',
+						data: {
+							attributeKeys: [
+								{
+									key: 'otelServiceName',
+									dataType: DataTypes.String,
+									type: 'tag',
+								},
+								{
+									key: 'service.instance.id',
+									dataType: DataTypes.String,
+									type: 'resource',
+								},
+								{
+									key: 'service.name',
+									dataType: DataTypes.String,
+									type: 'resource',
+								},
+								{
+									key: 'service.name',
+									dataType: DataTypes.String,
+									type: 'tag',
+								},
+							],
+						},
+					}),
+				),
+			),
+		);
+
+		// Open Filter input and type to trigger suggestions
+		const filterSelect = screen.getByTestId('qb-search-select');
+		const input = within(filterSelect).getByRole('combobox') as HTMLInputElement;
+
+		await user.click(input);
+		await waitFor(() =>
+			expect(screen.getByText('otelServiceName')).toBeInTheDocument(),
+		);
+
+		const serviceNameOccurences = await screen.findAllByText('service.name');
+		expect(serviceNameOccurences.length).toBeGreaterThanOrEqual(2);
 	});
 });

frontend/src/container/QueryBuilder/filters/QueryBuilderSearch/index.tsx

@@ -356,7 +356,10 @@ function QueryBuilderSearch({
 
 	// conditional changes here to use a seperate component to render the example queries based on the option group label
 	const customRendererForLogsExplorer = options.map((option) => (
-		<Select.Option key={option.label} value={option.value}>
+		<Select.Option
+			key={`${option.label}-${option.type || ''}-${option.dataType || ''}`}
+			value={option.value}
+		>
 			<OptionRendererForLogs
 				label={option.label}
 				value={option.value}
@@ -371,6 +374,7 @@ function QueryBuilderSearch({
 	return (
 		<div className="query-builder-search-container">
 			<Select
+				data-testid={'qb-search-select'}
 				ref={selectRef}
 				getPopupContainer={popupContainer}
 				transitionName=""
@@ -488,7 +492,10 @@ function QueryBuilderSearch({
 				{isLogsExplorerPage
 					? customRendererForLogsExplorer
 					: options.map((option) => (
-							<Select.Option key={option.label} value={option.value}>
+							<Select.Option
+								key={`${option.label}-${option.type || ''}-${option.dataType || ''}`}
+								value={option.value}
+							>
 								<OptionRenderer
 									label={option.label}
 									value={option.value}

frontend/src/hooks/dashboard/useDashboardVariables.ts

@@ -0,0 +1,19 @@
+import { useSyncExternalStore } from 'react';
+
+import {
+	dashboardVariablesStore,
+	IDashboardVariables,
+} from '../../providers/Dashboard/store/dashboardVariablesStore';
+
+export const useDashboardVariables = (): {
+	dashboardVariables: IDashboardVariables;
+} => {
+	const dashboardVariables = useSyncExternalStore(
+		dashboardVariablesStore.subscribe,
+		dashboardVariablesStore.getSnapshot,
+	);
+
+	return {
+		dashboardVariables,
+	};
+};

frontend/src/hooks/dashboard/useDashboardVariablesByType.ts

@@ -0,0 +1,30 @@
+import { useMemo } from 'react';
+import {
+	IDashboardVariable,
+	TVariableQueryType,
+} from 'types/api/dashboard/getAll';
+
+import { useDashboardVariables } from './useDashboardVariables';
+
+export function useDashboardVariablesByType(
+	variableType: TVariableQueryType,
+	returnType: 'values',
+): IDashboardVariable[];
+export function useDashboardVariablesByType(
+	variableType: TVariableQueryType,
+	returnType?: 'entries',
+): [string, IDashboardVariable][];
+export function useDashboardVariablesByType(
+	variableType: TVariableQueryType,
+	returnType?: 'values' | 'entries',
+): IDashboardVariable[] | [string, IDashboardVariable][] {
+	const { dashboardVariables } = useDashboardVariables();
+
+	return useMemo(() => {
+		const entries = Object.entries(dashboardVariables || {}).filter(
+			(entry): entry is [string, IDashboardVariable] =>
+				Boolean(entry[1].name) && entry[1].type === variableType,
+		);
+		return returnType === 'values' ? entries.map(([, value]) => value) : entries;
+	}, [dashboardVariables, variableType, returnType]);
+}

frontend/src/hooks/dashboard/useWidgetsByDynamicVariableId.ts

@@ -0,0 +1,28 @@
+import { useMemo } from 'react';
+import { PANEL_GROUP_TYPES } from 'constants/queryBuilder';
+import { createDynamicVariableToWidgetsMap } from 'hooks/dashboard/utils';
+import { useDashboard } from 'providers/Dashboard/Dashboard';
+import { Widgets } from 'types/api/dashboard/getAll';
+
+import { useDashboardVariablesByType } from './useDashboardVariablesByType';
+
+/**
+ * Hook to get a map of dynamic variable IDs to widget IDs that use them.
+ * This is useful for determining which widgets need to be refreshed when a dynamic variable changes.
+ */
+export function useWidgetsByDynamicVariableId(): Record<string, string[]> {
+	const dynamicVariables = useDashboardVariablesByType('DYNAMIC', 'values');
+	const { selectedDashboard } = useDashboard();
+
+	return useMemo(() => {
+		const widgets =
+			selectedDashboard?.data?.widgets?.filter(
+				(widget) => widget.panelTypes !== PANEL_GROUP_TYPES.ROW,
+			) || [];
+
+		return createDynamicVariableToWidgetsMap(
+			dynamicVariables,
+			widgets as Widgets[],
+		);
+	}, [selectedDashboard, dynamicVariables]);
+}

frontend/src/hooks/queryBuilder/useOptions.ts

@@ -193,7 +193,11 @@ export const useOptions = (
 					(option, index, self) =>
 						index ===
 							self.findIndex(
-								(o) => o.label === option.label && o.value === option.value, // to remove duplicate & empty options from list
+								(o) =>
+									o.label === option.label &&
+									o.value === option.value &&
+									(o.type || '') === (option.type || '') &&
+									(o.dataType || '') === (option.dataType || ''), // keep entries with same key but different type/dataType
 							) && option.value !== '',
 				) || []
 			).map((option) => {

frontend/src/hooks/useDimensions.ts

@@ -16,13 +16,19 @@ export function useResizeObserver<T extends HTMLElement>(
 	});
 
 	useEffect(() => {
-		const handleResize = debounce((entries: ResizeObserverEntry[]) => {
-			const entry = entries[0];
-			if (entry) {
-				const { width, height } = entry.contentRect;
-				setSize({ width, height });
-			}
-		}, debounceTime);
+		const handleResize = debounce(
+			(entries: ResizeObserverEntry[]) => {
+				const entry = entries[0];
+				if (entry) {
+					const { width, height } = entry.contentRect;
+					setSize({ width, height });
+				}
+			},
+			debounceTime,
+			{
+				leading: true,
+			},
+		);
 
 		const ro = new ResizeObserver(handleResize);
 		const referenceNode = ref.current;

frontend/src/providers/Dashboard/Dashboard.tsx

@@ -45,6 +45,8 @@ import APIError from 'types/api/error';
 import { GlobalReducer } from 'types/reducer/globalTime';
 import { v4 as generateUUID } from 'uuid';
 
+import { useDashboardVariables } from '../../hooks/dashboard/useDashboardVariables';
+import { updateDashboardVariablesStore } from './store/dashboardVariablesStore';
 import {
 	DashboardSortOrder,
 	IDashboardContext,
@@ -196,6 +198,16 @@ export function DashboardProvider({
 			: isDashboardWidgetPage?.params.dashboardId) || '';
 
 	const [selectedDashboard, setSelectedDashboard] = useState<Dashboard>();
+	const dashboardVariables = useDashboardVariables();
+
+	useEffect(() => {
+		const existingVariables = dashboardVariables;
+		const updatedVariables = selectedDashboard?.data.variables || {};
+
+		if (!isEqual(existingVariables, updatedVariables)) {
+			updateDashboardVariablesStore(updatedVariables);
+		}
+	}, [selectedDashboard]);
 
 	const {
 		currentDashboard,

frontend/src/providers/Dashboard/__tests__/Dashboard.test.tsx

@@ -8,6 +8,7 @@ import ROUTES from 'constants/routes';
 import { DashboardProvider, useDashboard } from 'providers/Dashboard/Dashboard';
 import { IDashboardVariable } from 'types/api/dashboard/getAll';
 
+import { useDashboardVariables } from '../../../hooks/dashboard/useDashboardVariables';
 import { initializeDefaultVariables } from '../initializeDefaultVariables';
 import { normalizeUrlValueForVariable } from '../normalizeUrlValue';
 
@@ -55,6 +56,7 @@ jest.mock('uuid', () => ({ v4: jest.fn(() => 'mock-uuid') }));
 
 function TestComponent(): JSX.Element {
 	const { dashboardResponse, dashboardId, selectedDashboard } = useDashboard();
+	const { dashboardVariables } = useDashboardVariables();
 
 	return (
 		<div>
@@ -65,9 +67,7 @@ function TestComponent(): JSX.Element {
 				{dashboardResponse.isFetching.toString()}
 			</div>
 			<div data-testid="dashboard-variables">
-				{selectedDashboard?.data?.variables
-					? JSON.stringify(selectedDashboard.data.variables)
-					: 'null'}
+				{dashboardVariables ? JSON.stringify(dashboardVariables) : 'null'}
 			</div>
 			<div data-testid="dashboard-data">
 				{selectedDashboard?.data?.title || 'No Title'}

frontend/src/providers/Dashboard/store/dashboardVariablesStore.ts

@@ -0,0 +1,17 @@
+import { IDashboardVariable } from 'types/api/dashboard/getAll';
+
+import createStore from './store';
+
+// export type IDashboardVariables = DashboardData['variables'];
+export type IDashboardVariables = Record<string, IDashboardVariable>;
+
+export const dashboardVariablesStore = createStore<IDashboardVariables>({});
+
+export function updateDashboardVariablesStore(
+	variables: Partial<IDashboardVariables>,
+): void {
+	dashboardVariablesStore.update((currentVariables) => ({
+		...currentVariables,
+		...variables,
+	}));
+}

frontend/src/providers/Dashboard/store/store.ts

@@ -0,0 +1,44 @@
+import { produce } from 'immer';
+type ListenerFn = () => void;
+
+export default function createStore<T>(
+	init: T,
+): {
+	set: (setter: any) => void;
+	update: (updater: (draft: T) => void) => void;
+	subscribe: (listener: ListenerFn) => () => void;
+	getSnapshot: () => T;
+} {
+	let listeners: ListenerFn[] = [];
+	let state = init;
+
+	function emitChange(): void {
+		for (const listener of listeners) {
+			listener();
+		}
+	}
+
+	function set(setter: any): void {
+		state = produce(state, setter);
+		emitChange();
+	}
+
+	function update(updater: (draft: T) => void): void {
+		state = produce(state, updater);
+		emitChange();
+	}
+
+	return {
+		set,
+		update,
+		subscribe(listener: ListenerFn): () => void {
+			listeners = [...listeners, listener];
+			return (): void => {
+				listeners = listeners.filter((l) => l !== listener);
+			};
+		},
+		getSnapshot(): T {
+			return state;
+		},
+	};
+}

frontend/yarn.lock

@@ -12030,6 +12030,11 @@ immediate@~3.0.5:
   resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
   integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==
 
+immer@11.1.3:
+  version "11.1.3"
+  resolved "https://registry.yarnpkg.com/immer/-/immer-11.1.3.tgz#78681e1deb6cec39753acf04eb16d7576c04f4d6"
+  integrity sha512-6jQTc5z0KJFtr1UgFpIL3N9XSC3saRaI9PwWtzM2pSqkNGtiNkYY2OSwkOGDK2XcTRcLb1pi/aNkKZz0nxVH4Q==
+
 immer@^9.0.6:
   version "9.0.21"
   resolved "https://registry.yarnpkg.com/immer/-/immer-9.0.21.tgz#1e025ea31a40f24fb064f1fef23e931496330176"