feat(7274): added flags column to remove invalid nan values

* feat: new dropdown styles

* fix: added new tag

* feat: added endpoint name and port in endpoint details

* feat: endpoint details feedback

* feat: analytics added

* fix: title fixed

* fix: domain list breaking for non available data

* feat: added third party api feature flag

* fix: console removed

* feat: added traces corelation in api monitoring charts

* feat: added customondragselect in grid card full view to handle breaking flow

* fix: minor failsafes added:

* fix: minor ux fix

* feat: incorporated pr comments - 0

CONTRIBUTING.md

@@ -77,4 +77,4 @@ Need assistance? Join our Slack community:
 ## Where do I go from here?
 
 - Set up your [development environment](docs/contributing/development.md)
-- Deploy and observe [SigNoz in action with OpenTelemetry Demo Application](docs/otel-demo/otel-demo-docs.md)
+- Deploy and observe [SigNoz in action with OpenTelemetry Demo Application](docs/otel-demo-docs.md)

ee/query-service/app/api/api.go

@@ -11,8 +11,6 @@ import (
 	"github.com/SigNoz/signoz/ee/query-service/license"
 	"github.com/SigNoz/signoz/ee/query-service/usage"
 	"github.com/SigNoz/signoz/pkg/alertmanager"
-	"github.com/SigNoz/signoz/pkg/modules/preference"
-	preferencecore "github.com/SigNoz/signoz/pkg/modules/preference/core"
 	baseapp "github.com/SigNoz/signoz/pkg/query-service/app"
 	"github.com/SigNoz/signoz/pkg/query-service/app/cloudintegrations"
 	"github.com/SigNoz/signoz/pkg/query-service/app/integrations"
@@ -23,7 +21,6 @@ import (
 	rules "github.com/SigNoz/signoz/pkg/query-service/rules"
 	"github.com/SigNoz/signoz/pkg/signoz"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/types/preferencetypes"
 	"github.com/SigNoz/signoz/pkg/version"
 	"github.com/gorilla/mux"
 )
@@ -57,7 +54,6 @@ type APIHandler struct {
 
 // NewAPIHandler returns an APIHandler
 func NewAPIHandler(opts APIHandlerOptions, signoz *signoz.SigNoz) (*APIHandler, error) {
-	preference := preference.NewAPI(preferencecore.NewPreference(preferencecore.NewStore(signoz.SQLStore), preferencetypes.NewDefaultPreferenceMap()))
 
 	baseHandler, err := baseapp.NewAPIHandler(baseapp.APIHandlerOpts{
 		Reader:                        opts.DataConnector,
@@ -75,7 +71,6 @@ func NewAPIHandler(opts APIHandlerOptions, signoz *signoz.SigNoz) (*APIHandler,
 		UseTraceNewSchema:             opts.UseTraceNewSchema,
 		AlertmanagerAPI:               alertmanager.NewAPI(signoz.Alertmanager),
 		Signoz:                        signoz,
-		Preference:                    preference,
 	})
 
 	if err != nil {

ee/query-service/app/db/reader.go

@@ -5,20 +5,21 @@ import (
 
 	"github.com/ClickHouse/clickhouse-go/v2"
 
+	"github.com/jmoiron/sqlx"
+
 	"github.com/SigNoz/signoz/pkg/cache"
 	basechr "github.com/SigNoz/signoz/pkg/query-service/app/clickhouseReader"
 	"github.com/SigNoz/signoz/pkg/query-service/interfaces"
-	"github.com/SigNoz/signoz/pkg/sqlstore"
 )
 
 type ClickhouseReader struct {
 	conn  clickhouse.Conn
-	appdb sqlstore.SQLStore
+	appdb *sqlx.DB
 	*basechr.ClickHouseReader
 }
 
 func NewDataConnector(
-	sqlDB sqlstore.SQLStore,
+	localDB *sqlx.DB,
 	ch clickhouse.Conn,
 	promConfigPath string,
 	lm interfaces.FeatureLookup,
@@ -28,10 +29,10 @@ func NewDataConnector(
 	fluxIntervalForTraceDetail time.Duration,
 	cache cache.Cache,
 ) *ClickhouseReader {
-	chReader := basechr.NewReader(sqlDB, ch, promConfigPath, lm, cluster, useLogsNewSchema, useTraceNewSchema, fluxIntervalForTraceDetail, cache)
+	chReader := basechr.NewReader(localDB, ch, promConfigPath, lm, cluster, useLogsNewSchema, useTraceNewSchema, fluxIntervalForTraceDetail, cache)
 	return &ClickhouseReader{
 		conn:             ch,
-		appdb:            sqlDB,
+		appdb:            localDB,
 		ClickHouseReader: chReader,
 	}
 }

ee/query-service/app/server.go

@@ -43,6 +43,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/query-service/app/logparsingpipeline"
 	"github.com/SigNoz/signoz/pkg/query-service/app/opamp"
 	opAmpModel "github.com/SigNoz/signoz/pkg/query-service/app/opamp/model"
+	"github.com/SigNoz/signoz/pkg/query-service/app/preferences"
 	"github.com/SigNoz/signoz/pkg/query-service/cache"
 	baseconst "github.com/SigNoz/signoz/pkg/query-service/constants"
 	"github.com/SigNoz/signoz/pkg/query-service/healthcheck"
@@ -115,6 +116,10 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 		return nil, err
 	}
 
+	if err := preferences.InitDB(serverOptions.SigNoz.SQLStore.SQLxDB()); err != nil {
+		return nil, err
+	}
+
 	if err := dashboards.InitDB(serverOptions.SigNoz.SQLStore); err != nil {
 		return nil, err
 	}
@@ -141,7 +146,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 
 	var reader interfaces.DataConnector
 	qb := db.NewDataConnector(
-		serverOptions.SigNoz.SQLStore,
+		serverOptions.SigNoz.SQLStore.SQLxDB(),
 		serverOptions.SigNoz.TelemetryStore.ClickHouseDB(),
 		serverOptions.PromConfigPath,
 		lm,
@@ -191,7 +196,10 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 	}
 
 	// initiate opamp
-	opAmpModel.InitDB(serverOptions.SigNoz.SQLStore)
+	_, err = opAmpModel.InitDB(serverOptions.SigNoz.SQLStore.SQLxDB())
+	if err != nil {
+		return nil, err
+	}
 
 	integrationsController, err := integrations.NewController(serverOptions.SigNoz.SQLStore)
 	if err != nil {
@@ -217,7 +225,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 
 	// initiate agent config handler
 	agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{
-		Store:         serverOptions.SigNoz.SQLStore,
+		DB:            serverOptions.SigNoz.SQLStore.SQLxDB(),
 		AgentFeatures: []agentConf.AgentFeature{logParsingPipelineController},
 	})
 	if err != nil {

ee/query-service/model/plans.go

@@ -157,6 +157,13 @@ var BasicPlan = basemodel.FeatureSet{
 		UsageLimit: -1,
 		Route:      "",
 	},
+	basemodel.Feature{
+		Name:       basemodel.TraceFunnels,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
 }
 
 var ProPlan = basemodel.FeatureSet{
@@ -279,6 +286,13 @@ var ProPlan = basemodel.FeatureSet{
 		UsageLimit: -1,
 		Route:      "",
 	},
+	basemodel.Feature{
+		Name:       basemodel.TraceFunnels,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
 }
 
 var EnterprisePlan = basemodel.FeatureSet{
@@ -415,4 +429,11 @@ var EnterprisePlan = basemodel.FeatureSet{
 		UsageLimit: -1,
 		Route:      "",
 	},
+	basemodel.Feature{
+		Name:       basemodel.TraceFunnels,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
 }

ee/sqlstore/postgressqlstore/dialect.go

@@ -2,29 +2,11 @@ package postgressqlstore
 
 import (
 	"context"
-	"fmt"
 	"reflect"
 
-	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/uptrace/bun"
 )
 
-var (
-	Identity = "id"
-	Integer  = "bigint"
-	Text     = "text"
-)
-
-var (
-	Org  = "org"
-	User = "user"
-)
-
-var (
-	OrgReference  = `("org_id") REFERENCES "organizations" ("id")`
-	UserReference = `("user_id") REFERENCES "users" ("id") ON DELETE CASCADE ON UPDATE CASCADE`
-)
-
 type dialect struct {
 }
 
@@ -121,7 +103,7 @@ func (dialect *dialect) GetColumnType(ctx context.Context, bun bun.IDB, table st
 
 	err := bun.NewSelect().
 		ColumnExpr("data_type").
-		TableExpr("").
+		TableExpr("information_schema.columns").
 		Where("table_name = ?", table).
 		Where("column_name = ?", column).
 		Scan(ctx, &columnType)
@@ -148,22 +130,6 @@ func (dialect *dialect) ColumnExists(ctx context.Context, bun bun.IDB, table str
 	return count > 0, nil
 }
 
-func (dialect *dialect) IndexExists(ctx context.Context, bun bun.IDB, table string, index string) (bool, error) {
-	var count int
-	err := bun.NewSelect().
-		ColumnExpr("COUNT(*)").
-		TableExpr("pg_indexes").
-		Where("tablename = ?", table).
-		Where("indexname = ?", index).
-		Scan(ctx, &count)
-
-	if err != nil {
-		return false, err
-	}
-
-	return count > 0, nil
-}
-
 func (dialect *dialect) RenameColumn(ctx context.Context, bun bun.IDB, table string, oldColumnName string, newColumnName string) (bool, error) {
 	oldColumnExists, err := dialect.ColumnExists(ctx, bun, table, oldColumnName)
 	if err != nil {
@@ -208,10 +174,7 @@ func (dialect *dialect) TableExists(ctx context.Context, bun bun.IDB, table inte
 	return true, nil
 }
 
-func (dialect *dialect) RenameTableAndModifyModel(ctx context.Context, bun bun.IDB, oldModel interface{}, newModel interface{}, reference string, cb func(context.Context) error) error {
-	if reference == "" {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cannot run migration without reference")
-	}
+func (dialect *dialect) RenameTableAndModifyModel(ctx context.Context, bun bun.IDB, oldModel interface{}, newModel interface{}, cb func(context.Context) error) error {
 	exists, err := dialect.TableExists(ctx, bun, newModel)
 	if err != nil {
 		return err
@@ -220,18 +183,10 @@ func (dialect *dialect) RenameTableAndModifyModel(ctx context.Context, bun bun.I
 		return nil
 	}
 
-	fkReference := ""
-	if reference == Org {
-		fkReference = OrgReference
-	} else if reference == User {
-		fkReference = UserReference
-	}
-
 	_, err = bun.
 		NewCreateTable().
 		IfNotExists().
 		Model(newModel).
-		ForeignKey(fkReference).
 		Exec(ctx)
 
 	if err != nil {
@@ -254,115 +209,3 @@ func (dialect *dialect) RenameTableAndModifyModel(ctx context.Context, bun bun.I
 
 	return nil
 }
-
-func (dialect *dialect) UpdatePrimaryKey(ctx context.Context, bun bun.IDB, oldModel interface{}, newModel interface{}, reference string, cb func(context.Context) error) error {
-	if reference == "" {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cannot run migration without reference")
-	}
-	oldTableName := bun.Dialect().Tables().Get(reflect.TypeOf(oldModel)).Name
-	newTableName := bun.Dialect().Tables().Get(reflect.TypeOf(newModel)).Name
-
-	columnType, err := dialect.GetColumnType(ctx, bun, oldTableName, Identity)
-	if err != nil {
-		return err
-	}
-	if columnType == Text {
-		return nil
-	}
-
-	fkReference := ""
-	if reference == Org {
-		fkReference = OrgReference
-	} else if reference == User {
-		fkReference = UserReference
-	}
-
-	_, err = bun.
-		NewCreateTable().
-		IfNotExists().
-		Model(newModel).
-		ForeignKey(fkReference).
-		Exec(ctx)
-
-	if err != nil {
-		return err
-	}
-
-	err = cb(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = bun.
-		NewDropTable().
-		IfExists().
-		Model(oldModel).
-		Exec(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = bun.
-		ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s RENAME TO %s", newTableName, oldTableName))
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (dialect *dialect) AddPrimaryKey(ctx context.Context, bun bun.IDB, oldModel interface{}, newModel interface{}, reference string, cb func(context.Context) error) error {
-	if reference == "" {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cannot run migration without reference")
-	}
-	oldTableName := bun.Dialect().Tables().Get(reflect.TypeOf(oldModel)).Name
-	newTableName := bun.Dialect().Tables().Get(reflect.TypeOf(newModel)).Name
-
-	identityExists, err := dialect.ColumnExists(ctx, bun, oldTableName, Identity)
-	if err != nil {
-		return err
-	}
-	if identityExists {
-		return nil
-	}
-
-	fkReference := ""
-	if reference == Org {
-		fkReference = OrgReference
-	} else if reference == User {
-		fkReference = UserReference
-	}
-
-	_, err = bun.
-		NewCreateTable().
-		IfNotExists().
-		Model(newModel).
-		ForeignKey(fkReference).
-		Exec(ctx)
-
-	if err != nil {
-		return err
-	}
-
-	err = cb(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = bun.
-		NewDropTable().
-		IfExists().
-		Model(oldModel).
-		Exec(ctx)
-	if err != nil {
-		return err
-	}
-
-	_, err = bun.
-		ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s RENAME TO %s", newTableName, oldTableName))
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

frontend/public/locales/en/titles.json

@@ -60,10 +60,14 @@
 	"INTEGRATIONS": "SigNoz | Integrations",
 	"ALERT_HISTORY": "SigNoz | Alert Rule History",
 	"ALERT_OVERVIEW": "SigNoz | Alert Rule Overview",
-	"MESSAGING_QUEUES": "SigNoz | Messaging Queues",
+	"MESSAGING_QUEUES_OVERVIEW": "SigNoz | Messaging Queues",
+	"MESSAGING_QUEUES_KAFKA": "SigNoz | Messaging Queues | Kafka",
+	"MESSAGING_QUEUES_KAFKA_DETAIL": "SigNoz | Messaging Queues | Kafka",
+	"MESSAGING_QUEUES_CELERY_TASK": "SigNoz | Messaging Queues | Celery",
 	"INFRASTRUCTURE_MONITORING_HOSTS": "SigNoz | Infra Monitoring",
 	"INFRASTRUCTURE_MONITORING_KUBERNETES": "SigNoz | Infra Monitoring",
 	"METRICS_EXPLORER": "SigNoz | Metrics Explorer",
 	"METRICS_EXPLORER_EXPLORER": "SigNoz | Metrics Explorer",
-	"METRICS_EXPLORER_VIEWS": "SigNoz | Metrics Explorer"
+	"METRICS_EXPLORER_VIEWS": "SigNoz | Metrics Explorer",
+	"API_MONITORING": "SigNoz | API Monitoring"
 }

frontend/src/components/CeleryOverview/CeleryOverviewTable/CeleryOverviewTable.tsx

@@ -521,7 +521,7 @@ export default function CeleryOverviewTable({
 				locale={{
 					emptyText: isLoading ? null : <Typography.Text>No data</Typography.Text>,
 				}}
-				scroll={{ x: true }}
+				scroll={{ x: 'max-content' }}
 				showSorterTooltip
 				onDragColumn={handleDragColumn}
 				onRow={(record): { onClick: () => void; className: string } => ({

frontend/src/components/ResizeTable/ResizableHeader.tsx

@@ -1,3 +1,5 @@
+import './ResizeTable.styles.scss';
+
 import { SyntheticEvent, useMemo } from 'react';
 import { Resizable, ResizeCallbackData } from 'react-resizable';
 
@@ -10,8 +12,8 @@ function ResizableHeader(props: ResizableHeaderProps): JSX.Element {
 	const handle = useMemo(
 		() => (
 			<SpanStyle
-				className="react-resizable-handle"
 				onClick={(e): void => e.stopPropagation()}
+				className="resize-handle"
 			/>
 		),
 		[],
@@ -19,7 +21,7 @@ function ResizableHeader(props: ResizableHeaderProps): JSX.Element {
 
 	if (!width) {
 		// eslint-disable-next-line react/jsx-props-no-spreading
-		return <th {...restProps} />;
+		return <th {...restProps} className="resizable-header" />;
 	}
 
 	return (
@@ -29,9 +31,10 @@ function ResizableHeader(props: ResizableHeaderProps): JSX.Element {
 			handle={handle}
 			onResize={onResize}
 			draggableOpts={enableUserSelectHack}
+			minConstraints={[150, 0]}
 		>
 			{/* eslint-disable-next-line react/jsx-props-no-spreading */}
-			<th {...restProps} />
+			<th {...restProps} className="resizable-header" />
 		</Resizable>
 	);
 }

frontend/src/components/ResizeTable/ResizeTable.styles.scss

@@ -0,0 +1,53 @@
+.resizable-header {
+	user-select: none;
+	-webkit-user-select: none;
+	-moz-user-select: none;
+	-ms-user-select: none;
+	position: relative;
+
+	.ant-table-column-title {
+		white-space: normal;
+		overflow: hidden;
+		text-overflow: ellipsis;
+	}
+}
+
+.resize-main-table {
+	.ant-table-body {
+		.ant-table-tbody {
+			.ant-table-row {
+				.ant-table-cell {
+					.ant-typography {
+						white-space: unset;
+					}
+				}
+			}
+		}
+	}
+}
+
+.logs-table,
+.traces-table {
+	.resize-table {
+		.resize-handle {
+			position: absolute;
+			top: 0;
+			bottom: 0;
+			inset-inline-end: -5px;
+			width: 10px;
+			cursor: col-resize;
+
+			&::after {
+				content: '';
+				position: absolute;
+				top: 50%;
+				left: 50%;
+				transform: translate(-50%, -50%);
+				width: 1px;
+				height: 1.6em;
+				background-color: var(--bg-slate-200);
+				transition: background-color 0.2s;
+			}
+		}
+	}
+}

frontend/src/components/ResizeTable/ResizeTable.tsx

@@ -2,35 +2,63 @@
 
 import { Table } from 'antd';
 import { ColumnsType } from 'antd/lib/table';
+import cx from 'classnames';
 import { dragColumnParams } from 'hooks/useDragColumns/configs';
-import { set } from 'lodash-es';
+import { RowData } from 'lib/query/createTableColumnsFromQuery';
+import { debounce, set } from 'lodash-es';
+import { useDashboard } from 'providers/Dashboard/Dashboard';
 import {
 	SyntheticEvent,
 	useCallback,
 	useEffect,
 	useMemo,
+	useRef,
 	useState,
 } from 'react';
 import ReactDragListView from 'react-drag-listview';
 import { ResizeCallbackData } from 'react-resizable';
+import { Widgets } from 'types/api/dashboard/getAll';
 
 import ResizableHeader from './ResizableHeader';
 import { DragSpanStyle } from './styles';
 import { ResizeTableProps } from './types';
 
+// eslint-disable-next-line sonarjs/cognitive-complexity
 function ResizeTable({
 	columns,
 	onDragColumn,
 	pagination,
+	widgetId,
+	shouldPersistColumnWidths = false,
 	...restProps
 }: ResizeTableProps): JSX.Element {
 	const [columnsData, setColumns] = useState<ColumnsType>([]);
+	const { setColumnWidths, selectedDashboard } = useDashboard();
+
+	const columnWidths = shouldPersistColumnWidths
+		? (selectedDashboard?.data?.widgets?.find(
+				(widget) => widget.id === widgetId,
+		  ) as Widgets)?.columnWidths
+		: undefined;
+
+	const updateAllColumnWidths = useRef(
+		debounce((widthsConfig: Record<string, number>) => {
+			if (!widgetId || !shouldPersistColumnWidths) return;
+			setColumnWidths?.((prev) => ({
+				...prev,
+				[widgetId]: widthsConfig,
+			}));
+		}, 1000),
+	).current;
 
 	const handleResize = useCallback(
 		(index: number) => (
-			_e: SyntheticEvent<Element>,
+			e: SyntheticEvent<Element>,
 			{ size }: ResizeCallbackData,
 		): void => {
+			e.preventDefault();
+			e.stopPropagation();
+
 			const newColumns = [...columnsData];
 			newColumns[index] = {
 				...newColumns[index],
@@ -65,6 +93,7 @@ function ResizeTable({
 			...restProps,
 			components: { header: { cell: ResizableHeader } },
 			columns: mergedColumns,
+			className: cx('resize-main-table', restProps.className),
 		};
 
 		set(
@@ -78,9 +107,39 @@ function ResizeTable({
 
 	useEffect(() => {
 		if (columns) {
-			setColumns(columns);
+			// Apply stored column widths from widget configuration
+			const columnsWithStoredWidths = columns.map((col) => {
+				const dataIndex = (col as RowData).dataIndex as string;
+				if (dataIndex && columnWidths && columnWidths[dataIndex]) {
+					return {
+						...col,
+						width: columnWidths[dataIndex], // Apply stored width
+					};
+				}
+				return col;
+			});
+
+			setColumns(columnsWithStoredWidths);
 		}
-	}, [columns]);
+	}, [columns, columnWidths]);
+
+	useEffect(() => {
+		if (!shouldPersistColumnWidths) return;
+		// Collect all column widths in a single object
+		const newColumnWidths: Record<string, number> = {};
+
+		mergedColumns.forEach((col) => {
+			if (col.width && (col as RowData).dataIndex) {
+				const dataIndex = (col as RowData).dataIndex as string;
+				newColumnWidths[dataIndex] = col.width as number;
+			}
+		});
+
+		// Only update if there are actual widths to set
+		if (Object.keys(newColumnWidths).length > 0) {
+			updateAllColumnWidths(newColumnWidths);
+		}
+	}, [mergedColumns, updateAllColumnWidths, shouldPersistColumnWidths]);
 
 	return onDragColumn ? (
 		<ReactDragListView.DragColumn {...dragColumnParams} onDragEnd={onDragColumn}>

frontend/src/components/ResizeTable/styles.ts

@@ -8,6 +8,8 @@ export const SpanStyle = styled.span`
 	width: 0.625rem;
 	height: 100%;
 	cursor: col-resize;
+	margin-left: 4px;
+	margin-right: 4px;
 `;
 
 export const DragSpanStyle = styled.span`

frontend/src/components/ResizeTable/types.ts

@@ -9,6 +9,8 @@ import { TableDataSource } from './contants';
 
 export interface ResizeTableProps extends TableProps<any> {
 	onDragColumn?: (fromIndex: number, toIndex: number) => void;
+	widgetId?: string;
+	shouldPersistColumnWidths?: boolean;
 }
 export interface DynamicColumnTableProps extends TableProps<any> {
 	tablesource: typeof TableDataSource[keyof typeof TableDataSource];

frontend/src/constants/features.ts

@@ -25,4 +25,6 @@ export enum FeatureKeys {
 	ANOMALY_DETECTION = 'ANOMALY_DETECTION',
 	AWS_INTEGRATION = 'AWS_INTEGRATION',
 	ONBOARDING_V3 = 'ONBOARDING_V3',
+	THIRD_PARTY_API = 'THIRD_PARTY_API',
+	TRACE_FUNNELS = 'TRACE_FUNNELS',
 }

frontend/src/container/AllAlertChannels/__tests__/AlertChannels.test.tsx

@@ -1,3 +1,4 @@
+import ROUTES from 'constants/routes';
 import AlertChannels from 'container/AllAlertChannels';
 import { allAlertChannels } from 'mocks-server/__mockdata__/alerts';
 import { act, fireEvent, render, screen, waitFor } from 'tests/test-utils';
@@ -20,6 +21,13 @@ jest.mock('hooks/useNotifications', () => ({
 	})),
 }));
 
+jest.mock('react-router-dom', () => ({
+	...jest.requireActual('react-router-dom'),
+	useLocation: (): { pathname: string } => ({
+		pathname: `${process.env.FRONTEND_API_ENDPOINT}${ROUTES.ALL_CHANNELS}`,
+	}),
+}));
+
 describe('Alert Channels Settings List page', () => {
 	beforeEach(() => {
 		render(<AlertChannels />);

frontend/src/container/AllAlertChannels/__tests__/AlertChannelsNormalUser.test.tsx

@@ -1,3 +1,4 @@
+import ROUTES from 'constants/routes';
 import AlertChannels from 'container/AllAlertChannels';
 import { allAlertChannels } from 'mocks-server/__mockdata__/alerts';
 import { fireEvent, render, screen, waitFor } from 'tests/test-utils';
@@ -25,6 +26,13 @@ jest.mock('hooks/useComponentPermission', () => ({
 	default: jest.fn().mockImplementation(() => [false]),
 }));
 
+jest.mock('react-router-dom', () => ({
+	...jest.requireActual('react-router-dom'),
+	useLocation: (): { pathname: string } => ({
+		pathname: `${process.env.FRONTEND_API_ENDPOINT}${ROUTES.ALL_CHANNELS}`,
+	}),
+}));
+
 describe('Alert Channels Settings List page (Normal User)', () => {
 	beforeEach(() => {
 		render(<AlertChannels />);

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/AllEndPoints.tsx

@@ -1,5 +1,6 @@
 import { LoadingOutlined } from '@ant-design/icons';
 import { Select, Spin, Table, Typography } from 'antd';
+import logEvent from 'api/common/logEvent';
 import { ENTITY_VERSION_V4 } from 'constants/app';
 import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
 import {
@@ -151,6 +152,7 @@ function AllEndPoints({
 		if (groupBy.length === 0) {
 			setSelectedEndPointName(record.endpointName); // this will open up the endpoint details tab
 			setSelectedView(VIEW_TYPES.ENDPOINT_DETAILS);
+			logEvent('API Monitoring: Endpoint name row clicked', {});
 		} else {
 			handleGroupByRowClick(record); // this will prepare the nested query payload
 		}

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/DomainDetails.styles.scss

@@ -392,6 +392,39 @@
 	gap: 20px;
 	padding-top: 20px;
 
+	.endpoint-meta-data {
+		display: flex;
+		gap: 8px;
+		.endpoint-meta-data-pill {
+			display: flex;
+			align-items: flex-start;
+			border-radius: 4px;
+			border: 1px solid var(--bg-slate-300);
+			width: fit-content;
+			.endpoint-meta-data-label {
+				display: flex;
+				padding: 6px 8px;
+				align-items: center;
+				gap: 4px;
+				border-right: 1px solid var(--bg-slate-300);
+				color: var(--text-vanilla-100);
+				background: var(--bg-slate-500);
+				height: calc(100% - 12px);
+			}
+
+			.endpoint-meta-data-value {
+				display: flex;
+				padding: 6px 8px;
+				justify-content: center;
+				align-items: center;
+				gap: 10px;
+				color: var(--text-vanilla-400);
+				background: var(--bg-slate-400);
+				height: calc(100% - 12px);
+			}
+		}
+	}
+
 	.endpoint-details-filters-container {
 		display: flex;
 		flex-direction: row;
@@ -405,6 +438,13 @@
 		}
 	}
 
+	.ant-select-item,
+	.ant-select-item-option-content {
+		flex: auto;
+		white-space: normal;
+		overflow-wrap: break-word;
+	}
+
 	.status-code-table-container {
 		border-radius: 3px;
 		border: 1px solid var(--bg-slate-500);
@@ -809,6 +849,13 @@
 			width: 100%;
 		}
 	}
+
+	.ant-select-item,
+	.ant-select-item-option-content {
+		flex: auto;
+		white-space: normal;
+		overflow-wrap: break-word;
+	}
 }
 
 .lightMode {
@@ -917,6 +964,20 @@
 		}
 	}
 
+	.endpoint-meta-data {
+		.endpoint-meta-data-pill {
+			.endpoint-meta-data-label {
+				color: var(--text-ink-300);
+				background: var(--bg-vanilla-100);
+			}
+
+			.endpoint-meta-data-value {
+				color: var(--text-ink-300);
+				background: var(--bg-vanilla-100);
+			}
+		}
+	}
+
 	.status-code-table-container {
 		.ant-table {
 			.ant-table-thead > tr > th {

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/DomainDetails.tsx

@@ -19,12 +19,14 @@ function DomainDetails({
 	selectedDomainIndex,
 	setSelectedDomainIndex,
 	domainListLength,
+	domainListFilters,
 }: {
 	domainData: any;
 	handleClose: () => void;
 	selectedDomainIndex: number;
 	setSelectedDomainIndex: (index: number) => void;
 	domainListLength: number;
+	domainListFilters: IBuilderQuery['filters'];
 }): JSX.Element {
 	const [selectedView, setSelectedView] = useState<VIEWS>(VIEWS.ALL_ENDPOINTS);
 	const [selectedEndPointName, setSelectedEndPointName] = useState<string>('');
@@ -132,6 +134,7 @@ function DomainDetails({
 							domainName={domainData.domainName}
 							endPointName={selectedEndPointName}
 							setSelectedEndPointName={setSelectedEndPointName}
+							domainListFilters={domainListFilters}
 						/>
 					)}
 				</>

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/EndPointDetails.tsx

@@ -2,7 +2,10 @@ import { ENTITY_VERSION_V4 } from 'constants/app';
 import { initialQueriesMap } from 'constants/queryBuilder';
 import {
 	END_POINT_DETAILS_QUERY_KEYS_ARRAY,
+	extractPortAndEndpoint,
 	getEndPointDetailsQueryPayload,
+	getLatencyOverTimeWidgetData,
+	getRateOverTimeWidgetData,
 } from 'container/ApiMonitoring/utils';
 import QueryBuilderSearchV2 from 'container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2';
 import { GetMetricQueryRange } from 'lib/dashboard/getQueryResults';
@@ -27,10 +30,12 @@ function EndPointDetails({
 	domainName,
 	endPointName,
 	setSelectedEndPointName,
+	domainListFilters,
 }: {
 	domainName: string;
 	endPointName: string;
 	setSelectedEndPointName: (value: string) => void;
+	domainListFilters: IBuilderQuery['filters'];
 }): JSX.Element {
 	const { maxTime, minTime } = useSelector<AppState, GlobalReducer>(
 		(state) => state.globalTime,
@@ -101,8 +106,6 @@ function EndPointDetails({
 	const [
 		endPointMetricsDataQuery,
 		endPointStatusCodeDataQuery,
-		endPointRateOverTimeDataQuery,
-		endPointLatencyOverTimeDataQuery,
 		endPointDropDownDataQuery,
 		endPointDependentServicesDataQuery,
 		endPointStatusCodeBarChartsDataQuery,
@@ -115,12 +118,29 @@ function EndPointDetails({
 			endPointDetailsDataQueries[3],
 			endPointDetailsDataQueries[4],
 			endPointDetailsDataQueries[5],
-			endPointDetailsDataQueries[6],
-			endPointDetailsDataQueries[7],
 		],
 		[endPointDetailsDataQueries],
 	);
 
+	const { endpoint, port } = useMemo(
+		() => extractPortAndEndpoint(endPointName),
+		[endPointName],
+	);
+
+	const [rateOverTimeWidget, latencyOverTimeWidget] = useMemo(
+		() => [
+			getRateOverTimeWidgetData(domainName, endPointName, {
+				items: [...domainListFilters.items, ...filters.items],
+				op: filters.op,
+			}),
+			getLatencyOverTimeWidgetData(domainName, endPointName, {
+				items: [...domainListFilters.items, ...filters.items],
+				op: filters.op,
+			}),
+		],
+		[domainName, endPointName, filters, domainListFilters],
+	);
+
 	return (
 		<div className="endpoint-details-container">
 			<div className="endpoint-details-filters-container">
@@ -129,6 +149,8 @@ function EndPointDetails({
 						selectedEndPointName={endPointName}
 						setSelectedEndPointName={setSelectedEndPointName}
 						endPointDropDownDataQuery={endPointDropDownDataQuery}
+						parentContainerDiv=".endpoint-details-filters-container"
+						dropdownStyle={{ width: 'calc(100% - 36px)' }}
 					/>
 				</div>
 				<div className="endpoint-details-filters-container-search">
@@ -141,6 +163,16 @@ function EndPointDetails({
 					/>
 				</div>
 			</div>
+			<div className="endpoint-meta-data">
+				<div className="endpoint-meta-data-pill">
+					<div className="endpoint-meta-data-label">Endpoint</div>
+					<div className="endpoint-meta-data-value">{endpoint || '-'}</div>
+				</div>
+				<div className="endpoint-meta-data-pill">
+					<div className="endpoint-meta-data-label">Port</div>
+					<div className="endpoint-meta-data-value">{port || '-'}</div>
+				</div>
+			</div>
 			<EndPointMetrics endPointMetricsDataQuery={endPointMetricsDataQuery} />
 			{!isServicesFilterApplied && (
 				<DependentServices
@@ -152,18 +184,14 @@ function EndPointDetails({
 				endPointStatusCodeLatencyBarChartsDataQuery={
 					endPointStatusCodeLatencyBarChartsDataQuery
 				}
+				domainName={domainName}
+				endPointName={endPointName}
+				domainListFilters={domainListFilters}
+				filters={filters}
 			/>
 			<StatusCodeTable endPointStatusCodeDataQuery={endPointStatusCodeDataQuery} />
-			<MetricOverTimeGraph
-				metricOverTimeDataQuery={endPointRateOverTimeDataQuery}
-				widgetInfoIndex={0}
-				endPointName={endPointName}
-			/>
-			<MetricOverTimeGraph
-				metricOverTimeDataQuery={endPointLatencyOverTimeDataQuery}
-				widgetInfoIndex={1}
-				endPointName={endPointName}
-			/>
+			<MetricOverTimeGraph widget={rateOverTimeWidget} />
+			<MetricOverTimeGraph widget={latencyOverTimeWidget} />
 		</div>
 	);
 }

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/EndPointDetailsWrapper.tsx

@@ -8,6 +8,7 @@ import { useSelector } from 'react-redux';
 import { AppState } from 'store/reducers';
 import { SuccessResponse } from 'types/api';
 import { MetricRangePayloadProps } from 'types/api/metrics/getQueryRange';
+import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
 import { GlobalReducer } from 'types/reducer/globalTime';
 
 import EndPointDetailsZeroState from './components/EndPointDetailsZeroState';
@@ -17,10 +18,12 @@ function EndPointDetailsWrapper({
 	domainName,
 	endPointName,
 	setSelectedEndPointName,
+	domainListFilters,
 }: {
 	domainName: string;
 	endPointName: string;
 	setSelectedEndPointName: (value: string) => void;
+	domainListFilters: IBuilderQuery['filters'];
 }): JSX.Element {
 	const { maxTime, minTime } = useSelector<AppState, GlobalReducer>(
 		(state) => state.globalTime,
@@ -69,6 +72,7 @@ function EndPointDetailsWrapper({
 			domainName={domainName}
 			endPointName={endPointName}
 			setSelectedEndPointName={setSelectedEndPointName}
+			domainListFilters={domainListFilters}
 		/>
 	);
 }

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/components/EndPointDetailsZeroState.tsx

@@ -28,6 +28,8 @@ function EndPointDetailsZeroState({
 					<EndPointsDropDown
 						setSelectedEndPointName={setSelectedEndPointName}
 						endPointDropDownDataQuery={endPointDropDownDataQuery}
+						parentContainerDiv=".end-point-details-zero-state-wrapper"
+						dropdownStyle={{ width: '60%' }}
 					/>
 				</div>
 			</div>

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/components/EndPointMetrics.tsx

@@ -70,7 +70,7 @@ function EndPointMetrics({
 							<Skeleton.Button active size="small" />
 						) : (
 							<Tooltip title={metricsData?.rate}>
-								<span className="round-metric-tag">{metricsData?.rate}/sec</span>
+								<span className="round-metric-tag">{metricsData?.rate} ops/sec</span>
 							</Tooltip>
 						)}
 					</Typography.Text>

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/components/EndPointsDropDown.tsx

@@ -8,16 +8,22 @@ interface EndPointsDropDownProps {
 	selectedEndPointName?: string;
 	setSelectedEndPointName: (value: string) => void;
 	endPointDropDownDataQuery: UseQueryResult<SuccessResponse<any>, unknown>;
+	parentContainerDiv?: string;
+	dropdownStyle?: React.CSSProperties;
 }
 
 const defaultProps = {
 	selectedEndPointName: '',
+	parentContainerDiv: '',
+	dropdownStyle: {},
 };
 
 function EndPointsDropDown({
 	selectedEndPointName,
 	setSelectedEndPointName,
 	endPointDropDownDataQuery,
+	parentContainerDiv,
+	dropdownStyle,
 }: EndPointsDropDownProps): JSX.Element {
 	const { data, isLoading, isFetching } = endPointDropDownDataQuery;
 
@@ -39,6 +45,13 @@ function EndPointsDropDown({
 			style={{ width: '100%' }}
 			onChange={handleChange}
 			options={formattedData}
+			getPopupContainer={
+				parentContainerDiv
+					? (): HTMLElement =>
+							document.querySelector(parentContainerDiv) as HTMLElement
+					: (triggerNode): HTMLElement => triggerNode.parentNode as HTMLElement
+			}
+			dropdownStyle={dropdownStyle}
 		/>
 	);
 }

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/components/ExpandedRow.tsx

@@ -1,6 +1,7 @@
 import { LoadingOutlined } from '@ant-design/icons';
 import { Spin, Table } from 'antd';
 import { ColumnType } from 'antd/lib/table';
+import logEvent from 'api/common/logEvent';
 import { ENTITY_VERSION_V4 } from 'constants/app';
 import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
 import {
@@ -114,6 +115,7 @@ function ExpandedRow({
 							onClick: (): void => {
 								setSelectedEndPointName(record.endpointName);
 								setSelectedView(VIEW_TYPES.ENDPOINT_DETAILS);
+								logEvent('API Monitoring: Endpoint name row clicked', {});
 							},
 							className: 'expanded-clickable-row',
 						})}

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/components/MetricOverTimeGraph.tsx

@@ -1,110 +1,18 @@
-import { Card, Skeleton, Typography } from 'antd';
-import cx from 'classnames';
-import Uplot from 'components/Uplot';
-import { PANEL_TYPES } from 'constants/queryBuilder';
-import {
-	apiWidgetInfo,
-	extractPortAndEndpoint,
-	getFormattedChartData,
-} from 'container/ApiMonitoring/utils';
-import { useIsDarkMode } from 'hooks/useDarkMode';
-import { useResizeObserver } from 'hooks/useDimensions';
-import { getUPlotChartOptions } from 'lib/uPlotLib/getUplotChartOptions';
-import { getUPlotChartData } from 'lib/uPlotLib/utils/getUplotChartData';
-import { useCallback, useMemo, useRef } from 'react';
-import { UseQueryResult } from 'react-query';
-import { useSelector } from 'react-redux';
-import { AppState } from 'store/reducers';
-import { SuccessResponse } from 'types/api';
-import { GlobalReducer } from 'types/reducer/globalTime';
-import { Options } from 'uplot';
-
-import ErrorState from './ErrorState';
-
-function MetricOverTimeGraph({
-	metricOverTimeDataQuery,
-	widgetInfoIndex,
-	endPointName,
-}: {
-	metricOverTimeDataQuery: UseQueryResult<SuccessResponse<any>, unknown>;
-	widgetInfoIndex: number;
-	endPointName: string;
-}): JSX.Element {
-	const { data } = metricOverTimeDataQuery;
-
-	const { minTime, maxTime } = useSelector<AppState, GlobalReducer>(
-		(state) => state.globalTime,
-	);
-
-	const graphRef = useRef<HTMLDivElement>(null);
-	const dimensions = useResizeObserver(graphRef);
-
-	const { endpoint } = extractPortAndEndpoint(endPointName);
-
-	const formattedChartData = useMemo(
-		() => getFormattedChartData(data?.payload, [endpoint]),
-		[data?.payload, endpoint],
-	);
-
-	const chartData = useMemo(() => getUPlotChartData(formattedChartData), [
-		formattedChartData,
-	]);
-
-	const isDarkMode = useIsDarkMode();
-
-	const options = useMemo(
-		() =>
-			getUPlotChartOptions({
-				apiResponse: formattedChartData,
-				isDarkMode,
-				dimensions,
-				yAxisUnit: apiWidgetInfo[widgetInfoIndex].yAxisUnit,
-				softMax: null,
-				softMin: null,
-				minTimeScale: Math.floor(minTime / 1e9),
-				maxTimeScale: Math.floor(maxTime / 1e9),
-				panelType: PANEL_TYPES.TIME_SERIES,
-			}),
-		[
-			formattedChartData,
-			minTime,
-			maxTime,
-			widgetInfoIndex,
-			dimensions,
-			isDarkMode,
-		],
-	);
-
-	const renderCardContent = useCallback(
-		(query: UseQueryResult<SuccessResponse<any>, unknown>): JSX.Element => {
-			if (query.isLoading) {
-				return <Skeleton />;
-			}
-
-			if (query.error) {
-				return <ErrorState refetch={query.refetch} />;
-			}
-
-			return (
-				<div
-					className={cx('chart-container', {
-						'no-data-container':
-							!query.isLoading && !query?.data?.payload?.data?.result?.length,
-					})}
-				>
-					<Uplot options={options as Options} data={chartData} />
-				</div>
-			);
-		},
-		[options, chartData],
-	);
+import { Card } from 'antd';
+import GridCard from 'container/GridCardLayout/GridCard';
+import { Widgets } from 'types/api/dashboard/getAll';
 
+function MetricOverTimeGraph({ widget }: { widget: Widgets }): JSX.Element {
 	return (
 		<div>
 			<Card bordered className="endpoint-details-card">
-				<Typography.Text>{apiWidgetInfo[widgetInfoIndex].title}</Typography.Text>
-				<div className="graph-container" ref={graphRef}>
-					{renderCardContent(metricOverTimeDataQuery)}
+				<div className="graph-container">
+					<GridCard
+						widget={widget}
+						isQueryEnabled
+						onDragSelect={(): void => {}}
+						customOnDragSelect={(): void => {}}
+					/>
 				</div>
 			</Card>
 		</div>

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/components/StatusCodeBarCharts.tsx

@@ -1,13 +1,22 @@
+import { Color } from '@signozhq/design-tokens';
 import { Button, Card, Skeleton, Typography } from 'antd';
 import cx from 'classnames';
+import { useGetGraphCustomSeries } from 'components/CeleryTask/useGetGraphCustomSeries';
+import { useNavigateToExplorer } from 'components/CeleryTask/useNavigateToExplorer';
 import Uplot from 'components/Uplot';
 import { PANEL_TYPES } from 'constants/queryBuilder';
 import {
+	getCustomFiltersForBarChart,
 	getFormattedEndPointStatusCodeChartData,
+	getStatusCodeBarChartWidgetData,
 	statusCodeWidgetInfo,
 } from 'container/ApiMonitoring/utils';
+import { handleGraphClick } from 'container/GridCardLayout/GridCard/utils';
+import { useGraphClickToShowButton } from 'container/GridCardLayout/useGraphClickToShowButton';
+import useNavigateToExplorerPages from 'container/GridCardLayout/useNavigateToExplorerPages';
 import { useIsDarkMode } from 'hooks/useDarkMode';
 import { useResizeObserver } from 'hooks/useDimensions';
+import { useNotifications } from 'hooks/useNotifications';
 import { getUPlotChartOptions } from 'lib/uPlotLib/getUplotChartOptions';
 import { getUPlotChartData } from 'lib/uPlotLib/utils/getUplotChartData';
 import { useCallback, useMemo, useRef, useState } from 'react';
@@ -15,6 +24,8 @@ import { UseQueryResult } from 'react-query';
 import { useSelector } from 'react-redux';
 import { AppState } from 'store/reducers';
 import { SuccessResponse } from 'types/api';
+import { Widgets } from 'types/api/dashboard/getAll';
+import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
 import { GlobalReducer } from 'types/reducer/globalTime';
 import { Options } from 'uplot';
 
@@ -23,6 +34,10 @@ import ErrorState from './ErrorState';
 function StatusCodeBarCharts({
 	endPointStatusCodeBarChartsDataQuery,
 	endPointStatusCodeLatencyBarChartsDataQuery,
+	domainName,
+	endPointName,
+	domainListFilters,
+	filters,
 }: {
 	endPointStatusCodeBarChartsDataQuery: UseQueryResult<
 		SuccessResponse<any>,
@@ -32,6 +47,10 @@ function StatusCodeBarCharts({
 		SuccessResponse<any>,
 		unknown
 	>;
+	domainName: string;
+	endPointName: string;
+	domainListFilters: IBuilderQuery['filters'];
+	filters: IBuilderQuery['filters'];
 }): JSX.Element {
 	// 0 : Status Code Count
 	// 1 : Status Code Latency
@@ -85,6 +104,72 @@ function StatusCodeBarCharts({
 
 	const isDarkMode = useIsDarkMode();
 
+	const graphClick = useGraphClickToShowButton({
+		graphRef,
+		isButtonEnabled: true,
+		buttonClassName: 'view-onclick-show-button',
+	});
+
+	const navigateToExplorer = useNavigateToExplorer();
+
+	const navigateToExplorerPages = useNavigateToExplorerPages();
+	const { notifications } = useNotifications();
+
+	const { getCustomSeries } = useGetGraphCustomSeries({
+		isDarkMode,
+		drawStyle: 'bars',
+		colorMapping: {
+			'200-299': Color.BG_FOREST_500,
+			'300-399': Color.BG_AMBER_400,
+			'400-499': Color.BG_CHERRY_500,
+			'500-599': Color.BG_ROBIN_500,
+			Other: Color.BG_SIENNA_500,
+		},
+	});
+
+	const widget = useMemo<Widgets>(
+		() =>
+			getStatusCodeBarChartWidgetData(domainName, endPointName, {
+				items: [...domainListFilters.items, ...filters.items],
+				op: filters.op,
+			}),
+		[domainName, endPointName, domainListFilters, filters],
+	);
+
+	const graphClickHandler = useCallback(
+		(
+			xValue: number,
+			yValue: number,
+			mouseX: number,
+			mouseY: number,
+			metric?: { [key: string]: string },
+			queryData?: { queryName: string; inFocusOrNot: boolean },
+		): void => {
+			const customFilters = getCustomFiltersForBarChart(metric);
+			handleGraphClick({
+				xValue,
+				yValue,
+				mouseX,
+				mouseY,
+				metric,
+				queryData,
+				widget,
+				navigateToExplorerPages,
+				navigateToExplorer,
+				notifications,
+				graphClick,
+				customFilters,
+			});
+		},
+		[
+			widget,
+			navigateToExplorerPages,
+			navigateToExplorer,
+			notifications,
+			graphClick,
+		],
+	);
+
 	const options = useMemo(
 		() =>
 			getUPlotChartOptions({
@@ -100,6 +185,8 @@ function StatusCodeBarCharts({
 				minTimeScale: Math.floor(minTime / 1e9),
 				maxTimeScale: Math.floor(maxTime / 1e9),
 				panelType: PANEL_TYPES.BAR,
+				onClickHandler: graphClickHandler,
+				customSeries: getCustomSeries,
 			}),
 		[
 			minTime,
@@ -109,6 +196,8 @@ function StatusCodeBarCharts({
 			formattedEndPointStatusCodeBarChartsDataPayload,
 			formattedEndPointStatusCodeLatencyBarChartsDataPayload,
 			isDarkMode,
+			graphClickHandler,
+			getCustomSeries,
 		],
 	);
 

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainList.tsx

@@ -3,6 +3,7 @@ import '../Explorer.styles.scss';
 import { LoadingOutlined } from '@ant-design/icons';
 import { Spin, Table, Typography } from 'antd';
 import axios from 'api';
+import logEvent from 'api/common/logEvent';
 import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
 import { AxiosError } from 'axios';
 import cx from 'classnames';
@@ -130,6 +131,7 @@ function DomainList({
 								(item) => item.key === record.key,
 							);
 							setSelectedDomainIndex(dataIndex);
+							logEvent('API Monitoring: Domain name row clicked', {});
 						}
 					},
 					className: 'expanded-clickable-row',
@@ -147,6 +149,7 @@ function DomainList({
 					handleClose={(): void => {
 						setSelectedDomainIndex(-1);
 					}}
+					domainListFilters={query?.filters}
 				/>
 			)}
 		</section>

frontend/src/container/ApiMonitoring/Explorer/Explorer.tsx

@@ -3,13 +3,14 @@ import './Explorer.styles.scss';
 import { FilterOutlined } from '@ant-design/icons';
 import * as Sentry from '@sentry/react';
 import { Switch, Typography } from 'antd';
+import logEvent from 'api/common/logEvent';
 import cx from 'classnames';
 import QuickFilters from 'components/QuickFilters/QuickFilters';
 import { QuickFiltersSource } from 'components/QuickFilters/types';
 import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
 import { useQueryOperations } from 'hooks/queryBuilder/useQueryBuilderOperations';
 import ErrorBoundaryFallback from 'pages/ErrorBoundaryFallback/ErrorBoundaryFallback';
-import { useMemo, useState } from 'react';
+import { useEffect, useMemo, useState } from 'react';
 import { Query } from 'types/api/queryBuilder/queryBuilderData';
 import { DataSource } from 'types/common/queryBuilder';
 
@@ -21,6 +22,10 @@ function Explorer(): JSX.Element {
 
 	const { currentQuery } = useQueryBuilder();
 
+	useEffect(() => {
+		logEvent('API Monitoring: Landing page visited', {});
+	}, []);
+
 	const { handleChangeQueryData } = useQueryOperations({
 		index: 0,
 		query: currentQuery.builder.queryData[0],
@@ -64,7 +69,12 @@ function Explorer(): JSX.Element {
 							style={{ marginLeft: 'auto' }}
 							checked={showIP}
 							onClick={(): void => {
-								setShowIP((showIP) => !showIP);
+								setShowIP((showIP): boolean => {
+									logEvent('API Monitoring: Show IP addresses clicked', {
+										showIP: !showIP,
+									});
+									return !showIP;
+								});
 							}}
 						/>
 					</div>

frontend/src/container/ApiMonitoring/utils.tsx

@@ -8,16 +8,23 @@ import {
 } from 'components/QuickFilters/types';
 import { PANEL_TYPES } from 'constants/queryBuilder';
 import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
+import { GraphClickMetaData } from 'container/GridCardLayout/useNavigateToExplorerPages';
+import { getWidgetQueryBuilder } from 'container/MetricsApplication/MetricsApplication.factory';
 import dayjs from 'dayjs';
 import { GetQueryResultsProps } from 'lib/dashboard/getQueryResults';
 import { cloneDeep } from 'lodash-es';
 import { ArrowUpDown, ChevronDown, ChevronRight } from 'lucide-react';
+import { getWidgetQuery } from 'pages/MessagingQueues/MQDetails/MetricPage/MetricPageUtil';
+import { Widgets } from 'types/api/dashboard/getAll';
 import { MetricRangePayloadProps } from 'types/api/metrics/getQueryRange';
 import {
 	BaseAutocompleteData,
 	DataTypes,
 } from 'types/api/queryBuilder/queryAutocompleteResponse';
-import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
+import {
+	IBuilderQuery,
+	TagFilterItem,
+} from 'types/api/queryBuilder/queryBuilderData';
 import { QueryData } from 'types/api/widgets/getQuery';
 import { EQueryType } from 'types/common/dashboard';
 import { DataSource } from 'types/common/queryBuilder';
@@ -128,12 +135,15 @@ export const columnsConfig: ColumnType<APIDomainsRowData>[] = [
 		sorter: false,
 		align: 'right',
 		className: `column`,
-		render: (lastUsed: number): string => getLastUsedRelativeTime(lastUsed),
+		render: (lastUsed: number | string): string =>
+			lastUsed === 'n/a' || lastUsed === '-'
+				? '-'
+				: getLastUsedRelativeTime(lastUsed as number),
 	},
 	{
 		title: (
 			<div>
-				Rate <span className="round-metric-tag">/s</span>
+				Rate <span className="round-metric-tag">ops/s</span>
 			</div>
 		),
 		dataIndex: 'rate',
@@ -155,21 +165,26 @@ export const columnsConfig: ColumnType<APIDomainsRowData>[] = [
 		sorter: false,
 		align: 'right',
 		className: `column`,
-		render: (errorRate: number): React.ReactNode => (
-			<Progress
-				status="active"
-				percent={Number((errorRate * 100).toFixed(1))}
-				strokeLinecap="butt"
-				size="small"
-				strokeColor={((): string => {
-					const errorRatePercent = Number((errorRate * 100).toFixed(1));
-					if (errorRatePercent >= 90) return Color.BG_SAKURA_500;
-					if (errorRatePercent >= 60) return Color.BG_AMBER_500;
-					return Color.BG_FOREST_500;
-				})()}
-				className="progress-bar error-rate"
-			/>
-		),
+		render: (errorRate: number | string): React.ReactNode => {
+			if (errorRate === 'n/a' || errorRate === '-') {
+				return '-';
+			}
+			return (
+				<Progress
+					status="active"
+					percent={Number(((errorRate as number) * 100).toFixed(1))}
+					strokeLinecap="butt"
+					size="small"
+					strokeColor={((): string => {
+						const errorRatePercent = Number(((errorRate as number) * 100).toFixed(1));
+						if (errorRatePercent >= 90) return Color.BG_SAKURA_500;
+						if (errorRatePercent >= 60) return Color.BG_AMBER_500;
+						return Color.BG_FOREST_500;
+					})()}
+					className="progress-bar error-rate"
+				/>
+			);
+		},
 	},
 	{
 		title: (
@@ -217,9 +232,9 @@ interface APIMonitoringResponseRow {
 	data: {
 		endpoints: number;
 		error_rate: number;
-		lastseen: number;
+		lastseen: number | string;
 		[domainNameKey]: string;
-		p99: number;
+		p99: number | string;
 		rps: number;
 	};
 }
@@ -232,12 +247,12 @@ interface EndPointsResponseRow {
 
 export interface APIDomainsRowData {
 	key: string;
-	domainName: React.ReactNode;
-	endpointCount: React.ReactNode;
-	rate: React.ReactNode;
-	errorRate: React.ReactNode;
-	latency: React.ReactNode;
-	lastUsed: React.ReactNode;
+	domainName: string;
+	endpointCount: number | string;
+	rate: number | string;
+	errorRate: number | string;
+	latency: number | string;
+	lastUsed: string;
 }
 
 // Rename this to a proper name
@@ -246,12 +261,20 @@ export const formatDataForTable = (
 ): APIDomainsRowData[] =>
 	data?.map((domain) => ({
 		key: v4(),
-		domainName: domain.data[domainNameKey] || '',
-		endpointCount: domain.data.endpoints,
-		rate: domain.data.rps,
-		errorRate: domain.data.error_rate,
-		latency: Math.round(domain.data.p99 / 1000000), // Convert from nanoseconds to milliseconds
-		lastUsed: new Date(Math.floor(domain.data.lastseen / 1000000)).toISOString(), // Convert from nanoseconds to milliseconds
+		domainName: domain?.data[domainNameKey] || '-',
+		endpointCount: domain?.data?.endpoints || '-',
+		rate: domain.data.rps || '-',
+		errorRate: domain.data.error_rate || '-',
+		latency:
+			domain.data.p99 === 'n/a'
+				? '-'
+				: Math.round(Number(domain.data.p99) / 1000000), // Convert from nanoseconds to milliseconds
+		lastUsed:
+			domain.data.lastseen === 'n/a'
+				? '-'
+				: new Date(
+						Math.floor(Number(domain.data.lastseen) / 1000000),
+				  ).toISOString(), // Convert from nanoseconds to milliseconds
 	}));
 
 // Rename this to a proper name
@@ -468,7 +491,6 @@ export const extractPortAndEndpoint = (
 	}
 };
 
-// Add icons in the below column headers
 export const getEndPointsColumnsConfig = (
 	isGroupedByAttribute: boolean,
 	expandedRowKeys: React.Key[],
@@ -576,7 +598,7 @@ export const formatEndPointsDataForTable = (
 			);
 			return {
 				key: v4(),
-				endpointName: (endpoint.data['http.url'] as string) || '',
+				endpointName: (endpoint.data['http.url'] as string) || '-',
 				port,
 				callCount: endpoint.data.A || '-',
 				latency:
@@ -593,7 +615,6 @@ export const formatEndPointsDataForTable = (
 
 	const groupedByAttributeData = groupBy.map((attribute) => attribute.key);
 
-	// TODO: Use tags to show the concatenated attribute values
 	return data?.map((endpoint) => {
 		const newEndpointName = groupedByAttributeData
 			.map((attribute) => endpoint.data[attribute])
@@ -639,7 +660,7 @@ export const createFiltersForSelectedRowData = (
 				type: null,
 			},
 			op: '=',
-			value: groupedByMeta[key],
+			value: groupedByMeta[key] || '',
 			id: key,
 		})),
 	);
@@ -649,12 +670,10 @@ export const createFiltersForSelectedRowData = (
 
 // First query payload for endpoint metrics
 // Second query payload for endpoint status code
-// Third query payload for endpoint rate over time graph
-// Fourth query payload for endpoint latency over time graph
-// Fifth query payload for endpoint dropdown selection
-// Sixth query payload for endpoint dependant services
-// Seventh query payload for endpoint response status count bar chart
-// Eighth query payload for endpoint response status code latency bar chart
+// Third query payload for endpoint dropdown selection
+// Fourth query payload for endpoint dependant services
+// Fifth query payload for endpoint response status count bar chart
+// Sixth query payload for endpoint response status code latency bar chart
 export const getEndPointDetailsQueryPayload = (
 	domainName: string,
 	endPointName: string,
@@ -1101,205 +1120,6 @@ export const getEndPointDetailsQueryPayload = (
 		end,
 		step: 60,
 	},
-	{
-		selectedTime: 'GLOBAL_TIME',
-		graphType: PANEL_TYPES.TIME_SERIES,
-		query: {
-			builder: {
-				queryData: [
-					{
-						aggregateAttribute: {
-							dataType: DataTypes.String,
-							id: '------false',
-							isColumn: false,
-							key: '',
-							type: '',
-						},
-						aggregateOperator: 'rate',
-						dataSource: DataSource.TRACES,
-						disabled: false,
-						expression: 'B',
-						filters: {
-							items: [
-								{
-									id: '3c76fe0b',
-									key: {
-										dataType: DataTypes.String,
-										id: 'net.peer.name--string--tag--false',
-										isColumn: false,
-										isJSON: false,
-										key: 'net.peer.name',
-										type: 'tag',
-									},
-									op: '=',
-									value: domainName,
-								},
-								{
-									id: '30710f04',
-									key: {
-										dataType: DataTypes.String,
-										id: 'http.url--string--tag--false',
-										isColumn: false,
-										isJSON: false,
-										key: 'http.url',
-										type: 'tag',
-									},
-									op: '=',
-									value: endPointName,
-								},
-								...filters.items,
-							],
-							op: 'AND',
-						},
-						functions: [],
-						groupBy: [
-							{
-								dataType: DataTypes.String,
-								id: 'http.url--string--tag--false',
-								isColumn: false,
-								isJSON: false,
-								key: 'http.url',
-								type: 'tag',
-							},
-						],
-						having: [],
-						legend: '',
-						limit: null,
-						orderBy: [],
-						queryName: 'B',
-						reduceTo: 'avg',
-						spaceAggregation: 'sum',
-						stepInterval: 60,
-						timeAggregation: 'rate',
-					},
-				],
-				queryFormulas: [],
-			},
-			clickhouse_sql: [
-				{
-					disabled: false,
-					legend: '',
-					name: 'A',
-					query: '',
-				},
-			],
-			id: '315b15fa-ff0c-442f-89f8-2bf4fb1af2f2',
-			promql: [
-				{
-					disabled: false,
-					legend: '',
-					name: 'A',
-					query: '',
-				},
-			],
-			queryType: EQueryType.QUERY_BUILDER,
-		},
-		variables: {},
-		formatForWeb: false,
-		start,
-		end,
-		step: 60,
-	},
-	{
-		selectedTime: 'GLOBAL_TIME',
-		graphType: PANEL_TYPES.TIME_SERIES,
-		query: {
-			builder: {
-				queryData: [
-					{
-						aggregateAttribute: {
-							dataType: DataTypes.Float64,
-							id: 'duration_nano--float64----true',
-							isColumn: true,
-							isJSON: false,
-							key: 'duration_nano',
-							type: '',
-						},
-						aggregateOperator: 'p99',
-						dataSource: DataSource.TRACES,
-						disabled: false,
-						expression: 'B',
-						filters: {
-							items: [
-								{
-									id: '63adb3ff',
-									key: {
-										dataType: DataTypes.String,
-										id: 'net.peer.name--string--tag--false',
-										isColumn: false,
-										isJSON: false,
-										key: 'net.peer.name',
-										type: 'tag',
-									},
-									op: '=',
-									value: domainName,
-								},
-								{
-									id: '50142500',
-									key: {
-										dataType: DataTypes.String,
-										id: 'http.url--string--tag--false',
-										isColumn: false,
-										isJSON: false,
-										key: 'http.url',
-										type: 'tag',
-									},
-									op: '=',
-									value: endPointName,
-								},
-								...filters.items,
-							],
-							op: 'AND',
-						},
-						functions: [],
-						groupBy: [
-							{
-								dataType: DataTypes.String,
-								id: 'http.url--string--tag--false',
-								isColumn: false,
-								isJSON: false,
-								key: 'http.url',
-								type: 'tag',
-							},
-						],
-						having: [],
-						legend: '',
-						limit: null,
-						orderBy: [],
-						queryName: 'B',
-						reduceTo: 'avg',
-						spaceAggregation: 'sum',
-						stepInterval: 60,
-						timeAggregation: 'p99',
-					},
-				],
-				queryFormulas: [],
-			},
-			clickhouse_sql: [
-				{
-					disabled: false,
-					legend: '',
-					name: 'A',
-					query: '',
-				},
-			],
-			id: '315b15fa-ff0c-442f-89f8-2bf4fb1af2f2',
-			promql: [
-				{
-					disabled: false,
-					legend: '',
-					name: 'A',
-					query: '',
-				},
-			],
-			queryType: EQueryType.QUERY_BUILDER,
-		},
-		variables: {},
-		formatForWeb: false,
-		start,
-		end,
-		step: 60,
-	},
 	{
 		selectedTime: 'GLOBAL_TIME',
 		graphType: PANEL_TYPES.TABLE,
@@ -1801,7 +1621,7 @@ interface EndPointMetricsData {
 interface EndPointStatusCodeData {
 	key: string;
 	statusCode: string;
-	count: number;
+	count: number | string;
 	p99Latency: number | string;
 }
 
@@ -1824,8 +1644,8 @@ export const getFormattedEndPointStatusCodeData = (
 ): EndPointStatusCodeData[] =>
 	data?.map((row) => ({
 		key: v4(),
-		statusCode: row.data.response_status_code,
-		count: row.data.A,
+		statusCode: row.data.response_status_code || '-',
+		count: row.data.A || '-',
 		p99Latency:
 			row.data.B === 'n/a' ? '-' : Math.round(Number(row.data.B) / 1000000), // Convert from nanoseconds to milliseconds,
 	}));
@@ -1857,11 +1677,6 @@ export const endPointStatusCodeColumns: ColumnType<EndPointStatusCodeData>[] = [
 	},
 ];
 
-export const apiWidgetInfo = [
-	{ title: 'Rate over time', yAxisUnit: 'ops/s' },
-	{ title: 'Latency over time', yAxisUnit: 'ns' },
-];
-
 export const statusCodeWidgetInfo = [
 	{ yAxisUnit: 'calls' },
 	{ yAxisUnit: 'ns' },
@@ -1885,8 +1700,8 @@ export const getFormattedEndPointDropDownData = (
 ): EndPointDropDownData[] =>
 	data?.map((row) => ({
 		key: v4(),
-		label: row.data['http.url'],
-		value: row.data['http.url'],
+		label: row.data['http.url'] || '-',
+		value: row.data['http.url'] || '-',
 	}));
 
 interface DependentServicesResponseRow {
@@ -1903,6 +1718,7 @@ interface DependentServicesData {
 	percentage: number;
 }
 
+// Discuss once about type safety of this function
 export const getFormattedDependentServicesData = (
 	data: DependentServicesResponseRow[],
 ): DependentServicesData[] => {
@@ -1983,7 +1799,7 @@ export const groupStatusCodes = (
 
 		// Track all timestamps
 		series.values.forEach((value) => {
-			allTimestamps.add(value[0]);
+			allTimestamps.add(Number(value[0]));
 		});
 
 		// Initialize or update the grouped series
@@ -2049,8 +1865,114 @@ export const groupStatusCodes = (
 		});
 	});
 
-	return Object.values(groupedSeries);
+	// Define the order of status code ranges
+	const statusCodeOrder = ['200-299', '300-399', '400-499', '500-599', 'Other'];
+
+	// Return the grouped series in the specified order
+	return statusCodeOrder
+		.filter((code) => groupedSeries[code]) // Only include codes that exist in the data
+		.map((code) => groupedSeries[code]);
 };
+
+export const getStatusCodeBarChartWidgetData = (
+	domainName: string,
+	endPointName: string,
+	filters: IBuilderQuery['filters'],
+): Widgets => ({
+	query: {
+		builder: {
+			queryData: [
+				{
+					aggregateAttribute: {
+						dataType: DataTypes.String,
+						id: '------false',
+						isColumn: false,
+						key: '',
+						type: '',
+					},
+					aggregateOperator: 'count',
+					dataSource: DataSource.TRACES,
+					disabled: false,
+					expression: 'A',
+					filters: {
+						items: [
+							{
+								id: 'c6724407',
+								key: {
+									dataType: DataTypes.String,
+									id: 'net.peer.name--string--tag--false',
+									isColumn: false,
+									isJSON: false,
+									key: 'net.peer.name',
+									type: 'tag',
+								},
+								op: '=',
+								value: domainName,
+							},
+							{
+								id: '8b1be6f0',
+								key: {
+									dataType: DataTypes.String,
+									id: 'http.url--string--tag--false',
+									isColumn: false,
+									isJSON: false,
+									key: 'http.url',
+									type: 'tag',
+								},
+								op: '=',
+								value: endPointName,
+							},
+							...filters.items,
+						],
+						op: 'AND',
+					},
+					functions: [],
+					groupBy: [],
+					having: [],
+					legend: '',
+					limit: null,
+					orderBy: [],
+					queryName: 'A',
+					reduceTo: 'avg',
+					spaceAggregation: 'sum',
+					stepInterval: 60,
+					timeAggregation: 'rate',
+				},
+			],
+			queryFormulas: [],
+		},
+		clickhouse_sql: [
+			{
+				disabled: false,
+				legend: '',
+				name: 'A',
+				query: '',
+			},
+		],
+		id: '315b15fa-ff0c-442f-89f8-2bf4fb1af2f2',
+		promql: [
+			{
+				disabled: false,
+				legend: '',
+				name: 'A',
+				query: '',
+			},
+		],
+		queryType: EQueryType.QUERY_BUILDER,
+	},
+	description: '',
+	id: '315b15fa-ff0c-442f-89f8-2bf4fb1af2f2',
+	isStacked: false,
+	panelTypes: PANEL_TYPES.BAR,
+	title: '',
+	opacity: '',
+	nullZeroValues: '',
+	timePreferance: 'GLOBAL_TIME',
+	softMin: null,
+	softMax: null,
+	selectedLogFields: null,
+	selectedTracesFields: null,
+});
 interface EndPointStatusCodePayloadData {
 	data: {
 		result: QueryData[];
@@ -2085,3 +2007,277 @@ export const END_POINT_DETAILS_QUERY_KEYS_ARRAY = [
 	REACT_QUERY_KEY.GET_ENDPOINT_STATUS_CODE_BAR_CHARTS_DATA,
 	REACT_QUERY_KEY.GET_ENDPOINT_STATUS_CODE_LATENCY_BAR_CHARTS_DATA,
 ];
+
+export const getRateOverTimeWidgetData = (
+	domainName: string,
+	endPointName: string,
+	filters: IBuilderQuery['filters'],
+): Widgets => {
+	const { endpoint, port } = extractPortAndEndpoint(endPointName);
+	const legend = `${
+		port !== '-' && port !== 'n/a' ? `${port}:` : ''
+	}${endpoint}`;
+	return getWidgetQueryBuilder(
+		getWidgetQuery({
+			title: 'Rate Over Time',
+			description: 'Rate over time.',
+			queryData: [
+				{
+					aggregateAttribute: {
+						dataType: DataTypes.String,
+						id: '------false',
+						isColumn: false,
+						key: '',
+						type: '',
+					},
+					aggregateOperator: 'rate',
+					dataSource: DataSource.TRACES,
+					disabled: false,
+					expression: 'A',
+					filters: {
+						items: [
+							{
+								id: '3c76fe0b',
+								key: {
+									dataType: DataTypes.String,
+									id: 'net.peer.name--string--tag--false',
+									isColumn: false,
+									isJSON: false,
+									key: 'net.peer.name',
+									type: 'tag',
+								},
+								op: '=',
+								value: domainName,
+							},
+							{
+								id: '30710f04',
+								key: {
+									dataType: DataTypes.String,
+									id: 'http.url--string--tag--false',
+									isColumn: false,
+									isJSON: false,
+									key: 'http.url',
+									type: 'tag',
+								},
+								op: '=',
+								value: endPointName,
+							},
+							...filters.items,
+						],
+						op: 'AND',
+					},
+					functions: [],
+					groupBy: [
+						{
+							dataType: DataTypes.String,
+							id: 'http.url--string--tag--false',
+							isColumn: false,
+							isJSON: false,
+							key: 'http.url',
+							type: 'tag',
+						},
+					],
+					having: [],
+					legend,
+					limit: null,
+					orderBy: [],
+					queryName: 'A',
+					reduceTo: 'avg',
+					spaceAggregation: 'sum',
+					stepInterval: 60,
+					timeAggregation: 'rate',
+				},
+			],
+			yAxisUnit: 'ops/s',
+		}),
+	);
+};
+
+export const getLatencyOverTimeWidgetData = (
+	domainName: string,
+	endPointName: string,
+	filters: IBuilderQuery['filters'],
+): Widgets => {
+	const { endpoint, port } = extractPortAndEndpoint(endPointName);
+	const legend = `${port}:${endpoint}`;
+	return getWidgetQueryBuilder(
+		getWidgetQuery({
+			title: 'Latency Over Time',
+			description: 'Latency over time.',
+			queryData: [
+				{
+					aggregateAttribute: {
+						dataType: DataTypes.Float64,
+						id: 'duration_nano--float64----true',
+						isColumn: true,
+						isJSON: false,
+						key: 'duration_nano',
+						type: '',
+					},
+					aggregateOperator: 'p99',
+					dataSource: DataSource.TRACES,
+					disabled: false,
+					expression: 'A',
+					filters: {
+						items: [
+							{
+								id: '63adb3ff',
+								key: {
+									dataType: DataTypes.String,
+									id: 'net.peer.name--string--tag--false',
+									isColumn: false,
+									isJSON: false,
+									key: 'net.peer.name',
+									type: 'tag',
+								},
+								op: '=',
+								value: domainName,
+							},
+							{
+								id: '50142500',
+								key: {
+									dataType: DataTypes.String,
+									id: 'http.url--string--tag--false',
+									isColumn: false,
+									isJSON: false,
+									key: 'http.url',
+									type: 'tag',
+								},
+								op: '=',
+								value: endPointName,
+							},
+							...filters.items,
+						],
+						op: 'AND',
+					},
+					functions: [],
+					groupBy: [
+						{
+							dataType: DataTypes.String,
+							id: 'http.url--string--tag--false',
+							isColumn: false,
+							isJSON: false,
+							key: 'http.url',
+							type: 'tag',
+						},
+					],
+					having: [],
+					legend,
+					limit: null,
+					orderBy: [],
+					queryName: 'A',
+					reduceTo: 'avg',
+					spaceAggregation: 'sum',
+					stepInterval: 60,
+					timeAggregation: 'p99',
+				},
+			],
+			yAxisUnit: 'ns',
+		}),
+	);
+};
+
+/**
+ * Helper function to get the start and end status codes from a status code range string
+ * @param value Status code range string (e.g. '200-299') or boolean
+ * @returns Tuple of [startStatusCode, endStatusCode] as strings
+ */
+const getStartAndEndStatusCode = (
+	value: string | boolean,
+): [string, string] => {
+	if (!value) {
+		return ['', ''];
+	}
+
+	switch (value) {
+		case '100-199':
+			return ['100', '199'];
+		case '200-299':
+			return ['200', '299'];
+		case '300-399':
+			return ['300', '399'];
+		case '400-499':
+			return ['400', '499'];
+		case '500-599':
+			return ['500', '599'];
+		default:
+			return ['', ''];
+	}
+};
+
+/**
+ * Creates filter items for bar chart based on group by fields and request data
+ * Used specifically for filtering status code ranges in bar charts
+ * @param groupBy Array of group by fields to create filters for
+ * @param requestData Data from graph click containing values to filter on
+ * @returns Array of TagFilterItems with >= and < operators for status code ranges
+ */
+export const createGroupByFiltersForBarChart = (
+	groupBy: BaseAutocompleteData[],
+	requestData: GraphClickMetaData,
+): TagFilterItem[] =>
+	groupBy
+		.map((gb) => {
+			const value = requestData[gb.key];
+			const [startStatusCode, endStatusCode] = getStartAndEndStatusCode(value);
+			return value
+				? [
+						{
+							id: v4(),
+							key: gb,
+							op: '>=',
+							value: startStatusCode,
+						},
+						{
+							id: v4(),
+							key: gb,
+							op: '<=',
+							value: endStatusCode,
+						},
+				  ]
+				: [];
+		})
+		.flat();
+
+export const getCustomFiltersForBarChart = (
+	metric:
+		| {
+				[key: string]: string;
+		  }
+		| undefined,
+): TagFilterItem[] => {
+	if (!metric?.response_status_code) {
+		return [];
+	}
+	const [startStatusCode, endStatusCode] = getStartAndEndStatusCode(
+		metric.response_status_code,
+	);
+	return [
+		{
+			id: v4(),
+			key: {
+				dataType: DataTypes.String,
+				id: 'response_status_code--string--tag--false',
+				isColumn: false,
+				isJSON: false,
+				key: 'response_status_code',
+				type: 'tag',
+			},
+			op: '>=',
+			value: startStatusCode,
+		},
+		{
+			id: v4(),
+			key: {
+				dataType: DataTypes.String,
+				id: 'response_status_code--string--tag--false',
+				isColumn: false,
+				isJSON: false,
+				key: 'response_status_code',
+				type: 'tag',
+			},
+			op: '<=',
+			value: endStatusCode,
+		},
+	];
+};

frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx

@@ -49,6 +49,7 @@ function FullView({
 	isDependedDataLoaded = false,
 	onToggleModelHandler,
 	onClickHandler,
+	customOnDragSelect,
 	setCurrentGraphRef,
 }: FullViewProps): JSX.Element {
 	const { safeNavigate } = useSafeNavigate();
@@ -252,7 +253,7 @@ function FullView({
 						onToggleModelHandler={onToggleModelHandler}
 						setGraphVisibility={setGraphsVisibilityStates}
 						graphVisibility={graphsVisibilityStates}
-						onDragSelect={onDragSelect}
+						onDragSelect={customOnDragSelect ?? onDragSelect}
 						tableProcessedDataRef={tableProcessedDataRef}
 						searchTerm={searchTerm}
 						onClickHandler={onClickHandler}

frontend/src/container/GridCardLayout/GridCard/FullView/types.ts

@@ -50,6 +50,7 @@ export interface FullViewProps {
 	widget: Widgets;
 	fullViewOptions?: boolean;
 	onClickHandler?: OnClickPluginOpts['onClick'];
+	customOnDragSelect?: (start: number, end: number) => void;
 	name: string;
 	tableProcessedDataRef: MutableRefObject<RowData[]>;
 	version?: string;

frontend/src/container/GridCardLayout/GridCard/WidgetGraphComponent.tsx

@@ -50,6 +50,7 @@ function WidgetGraphComponent({
 	setRequestData,
 	onClickHandler,
 	onDragSelect,
+	customOnDragSelect,
 	customTooltipElement,
 	openTracesButton,
 	onOpenTraceBtnClick,
@@ -327,6 +328,7 @@ function WidgetGraphComponent({
 					onToggleModelHandler={onToggleModelHandler}
 					tableProcessedDataRef={tableProcessedDataRef}
 					onClickHandler={onClickHandler ?? graphClickHandler}
+					customOnDragSelect={customOnDragSelect}
 					setCurrentGraphRef={setCurrentGraphRef}
 				/>
 			</Modal>

frontend/src/container/GridCardLayout/GridCard/index.tsx

@@ -36,6 +36,7 @@ function GridCardGraph({
 	version,
 	onClickHandler,
 	onDragSelect,
+	customOnDragSelect,
 	customTooltipElement,
 	dataAvailable,
 	getGraphData,
@@ -272,6 +273,7 @@ function GridCardGraph({
 					setRequestData={setRequestData}
 					onClickHandler={onClickHandler}
 					onDragSelect={onDragSelect}
+					customOnDragSelect={customOnDragSelect}
 					customTooltipElement={customTooltipElement}
 					openTracesButton={openTracesButton}
 					onOpenTraceBtnClick={onOpenTraceBtnClick}

frontend/src/container/GridCardLayout/GridCard/types.ts

@@ -33,6 +33,7 @@ export interface WidgetGraphComponentProps {
 	setRequestData?: Dispatch<SetStateAction<GetQueryResultsProps>>;
 	onClickHandler?: OnClickPluginOpts['onClick'];
 	onDragSelect: (start: number, end: number) => void;
+	customOnDragSelect?: (start: number, end: number) => void;
 	customTooltipElement?: HTMLDivElement;
 	openTracesButton?: boolean;
 	onOpenTraceBtnClick?: (record: RowData) => void;
@@ -49,6 +50,7 @@ export interface GridCardGraphProps {
 	variables?: Dashboard['data']['variables'];
 	version?: string;
 	onDragSelect: (start: number, end: number) => void;
+	customOnDragSelect?: (start: number, end: number) => void;
 	customTooltipElement?: HTMLDivElement;
 	dataAvailable?: (isDataAvailable: boolean) => void;
 	getGraphData?: (graphData?: MetricRangePayloadProps['data']) => void;

frontend/src/container/GridCardLayout/GridCard/utils.ts

@@ -178,6 +178,7 @@ interface HandleGraphClickParams {
 	navigateToExplorer: (props: NavigateToExplorerProps) => void;
 	notifications: NotificationInstance;
 	graphClick: (props: GraphClickProps) => void;
+	customFilters?: TagFilterItem[];
 }
 
 export const handleGraphClick = async ({
@@ -192,6 +193,7 @@ export const handleGraphClick = async ({
 	navigateToExplorer,
 	notifications,
 	graphClick,
+	customFilters,
 }: HandleGraphClickParams): Promise<void> => {
 	const { stepInterval } = widget?.query?.builder?.queryData?.[0] ?? {};
 
@@ -221,7 +223,7 @@ export const handleGraphClick = async ({
 					  }: ${key}`,
 			onClick: (): void =>
 				navigateToExplorer({
-					filters: result[key].filters,
+					filters: [...result[key].filters, ...(customFilters || [])],
 					dataSource: result[key].dataSource as DataSource,
 					startTime: xValue,
 					endTime: xValue + (stepInterval ?? 60),

frontend/src/container/GridCardLayout/GridCardLayout.tsx

@@ -44,7 +44,10 @@ import { EditMenuAction, ViewMenuAction } from './config';
 import DashboardEmptyState from './DashboardEmptyState/DashboardEmptyState';
 import GridCard from './GridCard';
 import { Card, CardContainer, ReactGridLayout } from './styles';
-import { removeUndefinedValuesFromLayout } from './utils';
+import {
+	hasColumnWidthsChanged,
+	removeUndefinedValuesFromLayout,
+} from './utils';
 import { MenuItemKeys } from './WidgetHeader/contants';
 import { WidgetRowHeader } from './WidgetRow';
 
@@ -68,6 +71,7 @@ function GraphLayout(props: GraphLayoutProps): JSX.Element {
 		setDashboardQueryRangeCalled,
 		setSelectedRowWidgetId,
 		isDashboardFetching,
+		columnWidths,
 	} = useDashboard();
 	const { data } = selectedDashboard || {};
 	const { pathname } = useLocation();
@@ -162,6 +166,7 @@ function GraphLayout(props: GraphLayoutProps): JSX.Element {
 			logEventCalledRef.current = true;
 		}
 	}, [data]);
+
 	const onSaveHandler = (): void => {
 		if (!selectedDashboard) return;
 
@@ -171,6 +176,15 @@ function GraphLayout(props: GraphLayoutProps): JSX.Element {
 				...selectedDashboard.data,
 				panelMap: { ...currentPanelMap },
 				layout: dashboardLayout.filter((e) => e.i !== PANEL_TYPES.EMPTY_WIDGET),
+				widgets: selectedDashboard?.data?.widgets?.map((widget) => {
+					if (columnWidths?.[widget.id]) {
+						return {
+							...widget,
+							columnWidths: columnWidths[widget.id],
+						};
+					}
+					return widget;
+				}),
 			},
 			uuid: selectedDashboard.uuid,
 		};
@@ -227,20 +241,31 @@ function GraphLayout(props: GraphLayoutProps): JSX.Element {
 
 	useEffect(() => {
 		if (
+			isDashboardLocked ||
+			!saveLayoutPermission ||
+			updateDashboardMutation.isLoading ||
+			isDashboardFetching
+		) {
+			return;
+		}
+
+		const shouldSaveLayout =
 			dashboardLayout &&
 			Array.isArray(dashboardLayout) &&
 			dashboardLayout.length > 0 &&
-			!isEqual(layouts, dashboardLayout) &&
-			!isDashboardLocked &&
-			saveLayoutPermission &&
-			!updateDashboardMutation.isLoading &&
-			!isDashboardFetching
-		) {
+			!isEqual(layouts, dashboardLayout);
+
+		const shouldSaveColumnWidths =
+			dashboardLayout &&
+			Array.isArray(dashboardLayout) &&
+			dashboardLayout.length > 0 &&
+			hasColumnWidthsChanged(columnWidths, selectedDashboard);
+
+		if (shouldSaveLayout || shouldSaveColumnWidths) {
 			onSaveHandler();
 		}
-
 		// eslint-disable-next-line react-hooks/exhaustive-deps
-	}, [dashboardLayout]);
+	}, [dashboardLayout, columnWidths]);
 
 	const onSettingsModalSubmit = (): void => {
 		const newTitle = form.getFieldValue('title');

frontend/src/container/GridCardLayout/useNavigateToExplorerPages.ts

@@ -12,7 +12,7 @@ import { v4 } from 'uuid';
 
 import { extractQueryNamesFromExpression } from './utils';
 
-type GraphClickMetaData = {
+export type GraphClickMetaData = {
 	[key: string]: string | boolean;
 	queryName: string;
 	inFocusOrNot: boolean;

frontend/src/container/GridCardLayout/utils.ts

@@ -1,5 +1,7 @@
 import { FORMULA_REGEXP } from 'constants/regExp';
+import { isEmpty, isEqual } from 'lodash-es';
 import { Layout } from 'react-grid-layout';
+import { Dashboard, Widgets } from 'types/api/dashboard/getAll';
 
 export const removeUndefinedValuesFromLayout = (layout: Layout[]): Layout[] =>
 	layout.map((obj) =>
@@ -25,3 +27,27 @@ export function extractQueryNamesFromExpression(expression: string): string[] {
 	// Extract matches and deduplicate
 	return [...new Set(expression.match(queryNameRegex) || [])];
 }
+
+export const hasColumnWidthsChanged = (
+	columnWidths: Record<string, Record<string, number>>,
+	selectedDashboard?: Dashboard,
+): boolean => {
+	// If no column widths stored, no changes
+	if (isEmpty(columnWidths) || !selectedDashboard) return false;
+
+	// Check each widget's column widths
+	return Object.keys(columnWidths).some((widgetId) => {
+		const dashboardWidget = selectedDashboard?.data?.widgets?.find(
+			(widget) => widget.id === widgetId,
+		) as Widgets;
+
+		const newWidths = columnWidths[widgetId];
+		const existingWidths = dashboardWidget?.columnWidths;
+
+		// If both are empty/undefined, no change
+		if (isEmpty(newWidths) || isEmpty(existingWidths)) return false;
+
+		// Compare stored column widths with dashboard widget's column widths
+		return !isEqual(newWidths, existingWidths);
+	});
+};

frontend/src/container/GridTableComponent/index.tsx

@@ -43,6 +43,7 @@ function GridTableComponent({
 	sticky,
 	openTracesButton,
 	onOpenTraceBtnClick,
+	widgetId,
 	...props
 }: GridTableComponentProps): JSX.Element {
 	const { t } = useTranslation(['valueGraph']);
@@ -229,6 +230,7 @@ function GridTableComponent({
 				columns={openTracesButton ? columnDataWithOpenTracesButton : newColumnData}
 				dataSource={dataSource}
 				sticky={sticky}
+				widgetId={widgetId}
 				onRow={
 					openTracesButton
 						? (record): React.HTMLAttributes<HTMLElement> => ({

frontend/src/container/GridTableComponent/types.ts

@@ -17,6 +17,7 @@ export type GridTableComponentProps = {
 	searchTerm?: string;
 	openTracesButton?: boolean;
 	onOpenTraceBtnClick?: (record: RowData) => void;
+	widgetId?: string;
 } & Pick<LogsExplorerTableProps, 'data'> &
 	Omit<TableProps<RowData>, 'columns' | 'dataSource'>;
 

frontend/src/container/LogsPanelTable/LogsPanelComponent.tsx

@@ -1,9 +1,9 @@
 import './LogsPanelComponent.styles.scss';
 
-import { Table } from 'antd';
 import LogDetail from 'components/LogDetail';
 import { VIEW_TYPES } from 'components/LogDetail/constants';
 import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
+import { ResizeTable } from 'components/ResizeTable';
 import { SOMETHING_WENT_WRONG } from 'constants/api';
 import { PANEL_TYPES } from 'constants/queryBuilder';
 import Controls from 'container/Controls';
@@ -79,9 +79,14 @@ function LogsPanelComponent({
 
 	const { formatTimezoneAdjustedTimestamp } = useTimezone();
 
-	const columns = getLogPanelColumnsList(
-		widget.selectedLogFields,
-		formatTimezoneAdjustedTimestamp,
+	const columns = useMemo(
+		() =>
+			getLogPanelColumnsList(
+				widget.selectedLogFields,
+				formatTimezoneAdjustedTimestamp,
+			),
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+		[widget.selectedLogFields],
 	);
 
 	const dataLength =
@@ -216,16 +221,18 @@ function LogsPanelComponent({
 			<div className="logs-table">
 				<div className="resize-table">
 					<OverlayScrollbar>
-						<Table
+						<ResizeTable
 							pagination={false}
 							tableLayout="fixed"
-							scroll={{ x: `calc(50vw - 10px)` }}
+							scroll={{ x: `max-content` }}
 							sticky
 							loading={queryResponse.isFetching}
 							style={tableStyles}
 							dataSource={flattenLogData}
 							columns={columns}
 							onRow={handleRow}
+							widgetId={widget.id}
+							shouldPersistColumnWidths
 						/>
 					</OverlayScrollbar>
 				</div>

frontend/src/container/NewWidget/index.tsx

@@ -74,6 +74,7 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 		setToScrollWidgetId,
 		selectedRowWidgetId,
 		setSelectedRowWidgetId,
+		columnWidths,
 	} = useDashboard();
 
 	const { t } = useTranslation(['dashboard']);
@@ -238,8 +239,10 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 				selectedLogFields,
 				selectedTracesFields,
 				isLogScale,
+				columnWidths: columnWidths?.[selectedWidget?.id],
 			};
 		});
+		// eslint-disable-next-line react-hooks/exhaustive-deps
 	}, [
 		columnUnits,
 		currentQuery,
@@ -260,6 +263,7 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 		combineHistogram,
 		stackedBarChart,
 		isLogScale,
+		columnWidths,
 	]);
 
 	const closeModal = (): void => {

frontend/src/container/PanelWrapper/TablePanelWrapper.tsx

@@ -26,6 +26,7 @@ function TablePanelWrapper({
 			searchTerm={searchTerm}
 			openTracesButton={openTracesButton}
 			onOpenTraceBtnClick={onOpenTraceBtnClick}
+			widgetId={widget.id}
 			// eslint-disable-next-line react/jsx-props-no-spreading
 			{...GRID_TABLE_CONFIG}
 		/>

frontend/src/container/PanelWrapper/__tests__/__snapshots__/TablePanelWrapper.test.tsx.snap

@@ -9,6 +9,8 @@ exports[`Table panel wrappper tests table should render fine with the query resp
   width: 0.625rem;
   height: 100%;
   cursor: col-resize;
+  margin-left: 4px;
+  margin-right: 4px;
 }
 
 .c0 {
@@ -54,7 +56,7 @@ exports[`Table panel wrappper tests table should render fine with the query resp
       class="query-table"
     >
       <div
-        class="ant-table-wrapper css-dev-only-do-not-override-2i2tap"
+        class="ant-table-wrapper resize-main-table css-dev-only-do-not-override-2i2tap"
       >
         <div
           class="ant-spin-nested-loading css-dev-only-do-not-override-2i2tap"
@@ -82,7 +84,7 @@ exports[`Table panel wrappper tests table should render fine with the query resp
                       <tr>
                         <th
                           aria-label="service_name"
-                          class="ant-table-cell ant-table-column-has-sorters react-resizable"
+                          class="resizable-header react-resizable"
                           scope="col"
                           tabindex="0"
                         >
@@ -143,12 +145,12 @@ exports[`Table panel wrappper tests table should render fine with the query resp
                             </span>
                           </div>
                           <span
-                            class="c1 react-resizable-handle"
+                            class="c1 resize-handle"
                           />
                         </th>
                         <th
                           aria-label="latency-per-service"
-                          class="ant-table-cell ant-table-column-has-sorters react-resizable"
+                          class="resizable-header react-resizable"
                           scope="col"
                           tabindex="0"
                         >
@@ -209,7 +211,7 @@ exports[`Table panel wrappper tests table should render fine with the query resp
                             </span>
                           </div>
                           <span
-                            class="c1 react-resizable-handle"
+                            class="c1 resize-handle"
                           />
                         </th>
                       </tr>
@@ -221,7 +223,7 @@ exports[`Table panel wrappper tests table should render fine with the query resp
                   style="overflow-x: auto; overflow-y: hidden;"
                 >
                   <table
-                    style="width: auto; min-width: 100%; table-layout: fixed;"
+                    style="min-width: 100%; table-layout: fixed;"
                   >
                     <colgroup>
                       <col

frontend/src/container/QueryTable/QueryTable.intefaces.ts

@@ -20,4 +20,5 @@ export type QueryTableProps = Omit<
 	dataSource?: RowData[];
 	sticky?: TableProps<RowData>['sticky'];
 	searchTerm?: string;
+	widgetId?: string;
 };

frontend/src/container/QueryTable/QueryTable.tsx

@@ -24,6 +24,7 @@ export function QueryTable({
 	dataSource,
 	sticky,
 	searchTerm,
+	widgetId,
 	...props
 }: QueryTableProps): JSX.Element {
 	const { isDownloadEnabled = false, fileName = '' } = downloadOption || {};
@@ -95,8 +96,10 @@ export function QueryTable({
 				columns={tableColumns}
 				tableLayout="fixed"
 				dataSource={filterTable === null ? newDataSource : filterTable}
-				scroll={{ x: true }}
+				scroll={{ x: 'max-content' }}
 				pagination={paginationConfig}
+				widgetId={widgetId}
+				shouldPersistColumnWidths
 				sticky={sticky}
 				// eslint-disable-next-line react/jsx-props-no-spreading
 				{...props}

frontend/src/container/SideNav/SideNav.tsx

@@ -279,6 +279,17 @@ function SideNav(): JSX.Element {
 		let updatedUserManagementItems: UserManagementMenuItems[] = [
 			manageLicenseMenuItem,
 		];
+
+		const isApiMonitoringEnabled = featureFlags?.find(
+			(flag) => flag.name === FeatureKeys.THIRD_PARTY_API,
+		)?.active;
+
+		if (!isApiMonitoringEnabled) {
+			updatedMenuItems = updatedMenuItems.filter(
+				(item) => item.key !== ROUTES.API_MONITORING,
+			);
+		}
+
 		if (isCloudUserVal || isEECloudUserVal) {
 			const isOnboardingEnabled =
 				featureFlags?.find((feature) => feature.name === FeatureKeys.ONBOARDING)

frontend/src/container/SideNav/menuItems.tsx

@@ -128,6 +128,7 @@ const menuItems: SidebarItem[] = [
 		key: ROUTES.API_MONITORING,
 		label: 'API Monitoring',
 		icon: <Binoculars size={16} />,
+		isNew: true,
 	},
 	{
 		key: ROUTES.LIST_ALL_ALERT,

frontend/src/container/TracesTableComponent/TracesTableComponent.tsx

@@ -1,7 +1,7 @@
 import './TracesTableComponent.styles.scss';
 
-import { Table } from 'antd';
 import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
+import { ResizeTable } from 'components/ResizeTable';
 import { SOMETHING_WENT_WRONG } from 'constants/api';
 import Controls from 'container/Controls';
 import { PER_PAGE_OPTIONS } from 'container/TracesExplorer/ListView/configs';
@@ -54,9 +54,14 @@ function TracesTableComponent({
 
 	const { formatTimezoneAdjustedTimestamp } = useTimezone();
 
-	const columns = getListColumns(
-		widget.selectedTracesFields || [],
-		formatTimezoneAdjustedTimestamp,
+	const columns = useMemo(
+		() =>
+			getListColumns(
+				widget.selectedTracesFields || [],
+				formatTimezoneAdjustedTimestamp,
+			),
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+		[widget.selectedTracesFields],
 	);
 
 	const dataLength =
@@ -116,16 +121,18 @@ function TracesTableComponent({
 		<div className="traces-table">
 			<div className="resize-table">
 				<OverlayScrollbar>
-					<Table
+					<ResizeTable
 						pagination={false}
 						tableLayout="fixed"
-						scroll={{ x: true }}
+						scroll={{ x: 'max-content' }}
 						loading={queryResponse.isFetching}
 						style={tableStyles}
 						dataSource={transformedQueryTableData}
 						columns={columns}
 						onRow={handleRow}
 						sticky
+						widgetId={widget.id}
+						shouldPersistColumnWidths
 					/>
 				</OverlayScrollbar>
 			</div>

frontend/src/pages/TracesModulePage/TracesModulePage.tsx

@@ -2,18 +2,24 @@ import './TracesModulePage.styles.scss';
 
 import RouteTab from 'components/RouteTab';
 import { TabRoutes } from 'components/RouteTab/types';
+import { FeatureKeys } from 'constants/features';
 import history from 'lib/history';
+import { useAppContext } from 'providers/App/App';
 import { useLocation } from 'react-router-dom';
 
 import { tracesExplorer, tracesFunnel, tracesSaveView } from './constants';
 
 function TracesModulePage(): JSX.Element {
 	const { pathname } = useLocation();
+	const { featureFlags } = useAppContext();
+
+	const isTraceFunnelsEnabled =
+		featureFlags?.find((flag) => flag.name === FeatureKeys.TRACE_FUNNELS)
+			?.active || false;
 
 	const routes: TabRoutes[] = [
 		tracesExplorer,
-		// TODO(shaheer): remove this check after everything is ready
-		process.env.NODE_ENV === 'development' ? tracesFunnel : null,
+		isTraceFunnelsEnabled ? tracesFunnel : null,
 		tracesSaveView,
 	].filter(Boolean) as TabRoutes[];
 

frontend/src/providers/Dashboard/Dashboard.tsx

@@ -40,7 +40,11 @@ import { Dashboard, IDashboardVariable } from 'types/api/dashboard/getAll';
 import { GlobalReducer } from 'types/reducer/globalTime';
 import { v4 as generateUUID } from 'uuid';
 
-import { DashboardSortOrder, IDashboardContext } from './types';
+import {
+	DashboardSortOrder,
+	IDashboardContext,
+	WidgetColumnWidths,
+} from './types';
 import { sortLayout } from './util';
 
 const DashboardContext = createContext<IDashboardContext>({
@@ -74,6 +78,8 @@ const DashboardContext = createContext<IDashboardContext>({
 	selectedRowWidgetId: '',
 	setSelectedRowWidgetId: () => {},
 	isDashboardFetching: false,
+	columnWidths: {},
+	setColumnWidths: () => {},
 });
 
 interface Props {
@@ -408,6 +414,8 @@ export function DashboardProvider({
 		}
 	};
 
+	const [columnWidths, setColumnWidths] = useState<WidgetColumnWidths>({});
+
 	const value: IDashboardContext = useMemo(
 		() => ({
 			toScrollWidgetId,
@@ -435,6 +443,8 @@ export function DashboardProvider({
 			selectedRowWidgetId,
 			setSelectedRowWidgetId,
 			isDashboardFetching,
+			columnWidths,
+			setColumnWidths,
 		}),
 		// eslint-disable-next-line react-hooks/exhaustive-deps
 		[
@@ -457,6 +467,8 @@ export function DashboardProvider({
 			selectedRowWidgetId,
 			setSelectedRowWidgetId,
 			isDashboardFetching,
+			columnWidths,
+			setColumnWidths,
 		],
 	);
 

frontend/src/providers/Dashboard/types.ts

@@ -10,6 +10,10 @@ export interface DashboardSortOrder {
 	search: string;
 }
 
+export type WidgetColumnWidths = {
+	[widgetId: string]: Record<string, number>;
+};
+
 export interface IDashboardContext {
 	isDashboardSliderOpen: boolean;
 	isDashboardLocked: boolean;
@@ -48,4 +52,6 @@ export interface IDashboardContext {
 	selectedRowWidgetId: string | null;
 	setSelectedRowWidgetId: React.Dispatch<React.SetStateAction<string | null>>;
 	isDashboardFetching: boolean;
+	columnWidths: WidgetColumnWidths;
+	setColumnWidths: React.Dispatch<React.SetStateAction<WidgetColumnWidths>>;
 }

frontend/src/types/api/dashboard/getAll.ts

@@ -109,6 +109,7 @@ export interface IBaseWidget {
 	selectedLogFields: IField[] | null;
 	selectedTracesFields: BaseAutocompleteData[] | null;
 	isLogScale?: boolean;
+	columnWidths?: Record<string, number>;
 }
 export interface Widgets extends IBaseWidget {
 	query: Query;

pkg/alertmanager/alertmanager.go

@@ -6,7 +6,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
 var (
@@ -34,16 +33,16 @@ type Alertmanager interface {
 	ListAllChannels(context.Context) ([]*alertmanagertypes.Channel, error)
 
 	// GetChannelByID gets a channel for the organization.
-	GetChannelByID(context.Context, string, valuer.UUID) (*alertmanagertypes.Channel, error)
+	GetChannelByID(context.Context, string, int) (*alertmanagertypes.Channel, error)
 
 	// UpdateChannel updates a channel for the organization.
-	UpdateChannelByReceiverAndID(context.Context, string, alertmanagertypes.Receiver, valuer.UUID) error
+	UpdateChannelByReceiverAndID(context.Context, string, alertmanagertypes.Receiver, int) error
 
 	// CreateChannel creates a channel for the organization.
 	CreateChannel(context.Context, string, alertmanagertypes.Receiver) error
 
 	// DeleteChannelByID deletes a channel for the organization.
-	DeleteChannelByID(context.Context, string, valuer.UUID) error
+	DeleteChannelByID(context.Context, string, int) error
 
 	// SetConfig sets the config for the organization.
 	SetConfig(context.Context, *alertmanagertypes.Config) error

pkg/alertmanager/alertmanagerstore/sqlalertmanagerstore/config.go

@@ -8,7 +8,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/sqlstore"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/tidwall/gjson"
 	"github.com/uptrace/bun"
 )
@@ -100,7 +99,7 @@ func (store *config) CreateChannel(ctx context.Context, channel *alertmanagertyp
 	}, opts...)
 }
 
-func (store *config) GetChannelByID(ctx context.Context, orgID string, id valuer.UUID) (*alertmanagertypes.Channel, error) {
+func (store *config) GetChannelByID(ctx context.Context, orgID string, id int) (*alertmanagertypes.Channel, error) {
 	channel := new(alertmanagertypes.Channel)
 
 	err := store.
@@ -109,11 +108,11 @@ func (store *config) GetChannelByID(ctx context.Context, orgID string, id valuer
 		NewSelect().
 		Model(channel).
 		Where("org_id = ?", orgID).
-		Where("id = ?", id.StringValue()).
+		Where("id = ?", id).
 		Scan(ctx)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			return nil, errors.Newf(errors.TypeNotFound, alertmanagertypes.ErrCodeAlertmanagerChannelNotFound, "cannot find channel with id %s", id.StringValue())
+			return nil, errors.Newf(errors.TypeNotFound, alertmanagertypes.ErrCodeAlertmanagerChannelNotFound, "cannot find channel with id %d", id)
 		}
 		return nil, err
 	}
@@ -137,7 +136,7 @@ func (store *config) UpdateChannel(ctx context.Context, orgID string, channel *a
 	}, opts...)
 }
 
-func (store *config) DeleteChannelByID(ctx context.Context, orgID string, id valuer.UUID, opts ...alertmanagertypes.StoreOption) error {
+func (store *config) DeleteChannelByID(ctx context.Context, orgID string, id int, opts ...alertmanagertypes.StoreOption) error {
 	return store.wrap(ctx, func(ctx context.Context) error {
 		channel := new(alertmanagertypes.Channel)
 
@@ -147,7 +146,7 @@ func (store *config) DeleteChannelByID(ctx context.Context, orgID string, id val
 			NewDelete().
 			Model(channel).
 			Where("org_id = ?", orgID).
-			Where("id = ?", id.StringValue()).
+			Where("id = ?", id).
 			Exec(ctx); err != nil {
 			return err
 		}

pkg/alertmanager/api.go

@@ -4,13 +4,13 @@ import (
 	"context"
 	"io"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/http/render"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/gorilla/mux"
 )
 
@@ -140,9 +140,9 @@ func (api *API) GetChannelByID(rw http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	id, err := valuer.NewUUID(idString)
+	id, err := strconv.Atoi(idString)
 	if err != nil {
-		render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "id is not a valid uuid-v7"))
+		render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "id is not a valid integer"))
 		return
 	}
 
@@ -177,9 +177,9 @@ func (api *API) UpdateChannelByID(rw http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	id, err := valuer.NewUUID(idString)
+	id, err := strconv.Atoi(idString)
 	if err != nil {
-		render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "id is not a valid uuid-v7"))
+		render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "id is not a valid integer"))
 		return
 	}
 
@@ -227,9 +227,9 @@ func (api *API) DeleteChannelByID(rw http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	id, err := valuer.NewUUID(idString)
+	id, err := strconv.Atoi(idString)
 	if err != nil {
-		render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "id is not a valid uuid-v7"))
+		render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "id is not a valid integer"))
 		return
 	}
 

pkg/alertmanager/legacyalertmanager/provider.go

@@ -16,7 +16,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/sqlstore"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/tidwall/gjson"
 )
 
@@ -270,11 +269,11 @@ func (provider *provider) ListAllChannels(ctx context.Context) ([]*alertmanagert
 	return channels, nil
 }
 
-func (provider *provider) GetChannelByID(ctx context.Context, orgID string, channelID valuer.UUID) (*alertmanagertypes.Channel, error) {
+func (provider *provider) GetChannelByID(ctx context.Context, orgID string, channelID int) (*alertmanagertypes.Channel, error) {
 	return provider.configStore.GetChannelByID(ctx, orgID, channelID)
 }
 
-func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgID string, receiver alertmanagertypes.Receiver, id valuer.UUID) error {
+func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgID string, receiver alertmanagertypes.Receiver, id int) error {
 	channel, err := provider.configStore.GetChannelByID(ctx, orgID, id)
 	if err != nil {
 		return err
@@ -379,7 +378,7 @@ func (provider *provider) CreateChannel(ctx context.Context, orgID string, recei
 	}))
 }
 
-func (provider *provider) DeleteChannelByID(ctx context.Context, orgID string, channelID valuer.UUID) error {
+func (provider *provider) DeleteChannelByID(ctx context.Context, orgID string, channelID int) error {
 	channel, err := provider.configStore.GetChannelByID(ctx, orgID, channelID)
 	if err != nil {
 		return err

pkg/alertmanager/signozalertmanager/provider.go

@@ -10,7 +10,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/sqlstore"
 	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
 type provider struct {
@@ -100,11 +99,11 @@ func (provider *provider) ListAllChannels(ctx context.Context) ([]*alertmanagert
 	return nil, errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported, "not supported by provider signoz")
 }
 
-func (provider *provider) GetChannelByID(ctx context.Context, orgID string, channelID valuer.UUID) (*alertmanagertypes.Channel, error) {
+func (provider *provider) GetChannelByID(ctx context.Context, orgID string, channelID int) (*alertmanagertypes.Channel, error) {
 	return provider.configStore.GetChannelByID(ctx, orgID, channelID)
 }
 
-func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgID string, receiver alertmanagertypes.Receiver, id valuer.UUID) error {
+func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgID string, receiver alertmanagertypes.Receiver, id int) error {
 	channel, err := provider.configStore.GetChannelByID(ctx, orgID, id)
 	if err != nil {
 		return err
@@ -128,7 +127,7 @@ func (provider *provider) UpdateChannelByReceiverAndID(ctx context.Context, orgI
 	}))
 }
 
-func (provider *provider) DeleteChannelByID(ctx context.Context, orgID string, channelID valuer.UUID) error {
+func (provider *provider) DeleteChannelByID(ctx context.Context, orgID string, channelID int) error {
 	channel, err := provider.configStore.GetChannelByID(ctx, orgID, channelID)
 	if err != nil {
 		return err

pkg/modules/preference/api.go

@@ -1,149 +0,0 @@
-package preference
-
-import (
-	"encoding/json"
-	"net/http"
-
-	errorsV2 "github.com/SigNoz/signoz/pkg/errors"
-	"github.com/SigNoz/signoz/pkg/http/render"
-	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/types/preferencetypes"
-	"github.com/gorilla/mux"
-)
-
-type API interface {
-	GetOrgPreference(http.ResponseWriter, *http.Request)
-	UpdateOrgPreference(http.ResponseWriter, *http.Request)
-	GetAllOrgPreferences(http.ResponseWriter, *http.Request)
-
-	GetUserPreference(http.ResponseWriter, *http.Request)
-	UpdateUserPreference(http.ResponseWriter, *http.Request)
-	GetAllUserPreferences(http.ResponseWriter, *http.Request)
-}
-
-type preferenceAPI struct {
-	usecase Usecase
-}
-
-func NewAPI(usecase Usecase) API {
-	return &preferenceAPI{usecase: usecase}
-}
-
-func (p *preferenceAPI) GetOrgPreference(rw http.ResponseWriter, r *http.Request) {
-	preferenceId := mux.Vars(r)["preferenceId"]
-	claims, ok := authtypes.ClaimsFromContext(r.Context())
-	if !ok {
-		render.Error(rw, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
-		return
-	}
-	preference, err := p.usecase.GetOrgPreference(
-		r.Context(), preferenceId, claims.OrgID,
-	)
-	if err != nil {
-		render.Error(rw, err)
-		return
-	}
-
-	render.Success(rw, http.StatusOK, preference)
-}
-
-func (p *preferenceAPI) UpdateOrgPreference(rw http.ResponseWriter, r *http.Request) {
-	preferenceId := mux.Vars(r)["preferenceId"]
-	req := preferencetypes.UpdatablePreference{}
-	claims, ok := authtypes.ClaimsFromContext(r.Context())
-	if !ok {
-		render.Error(rw, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
-		return
-	}
-
-	err := json.NewDecoder(r.Body).Decode(&req)
-
-	if err != nil {
-		render.Error(rw, err)
-		return
-	}
-	err = p.usecase.UpdateOrgPreference(r.Context(), preferenceId, req.PreferenceValue, claims.OrgID)
-	if err != nil {
-		render.Error(rw, err)
-		return
-	}
-
-	render.Success(rw, http.StatusNoContent, nil)
-}
-
-func (p *preferenceAPI) GetAllOrgPreferences(rw http.ResponseWriter, r *http.Request) {
-	claims, ok := authtypes.ClaimsFromContext(r.Context())
-	if !ok {
-		render.Error(rw, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
-		return
-	}
-	preferences, err := p.usecase.GetAllOrgPreferences(
-		r.Context(), claims.OrgID,
-	)
-	if err != nil {
-		render.Error(rw, err)
-		return
-	}
-
-	render.Success(rw, http.StatusOK, preferences)
-}
-
-func (p *preferenceAPI) GetUserPreference(rw http.ResponseWriter, r *http.Request) {
-	preferenceId := mux.Vars(r)["preferenceId"]
-	claims, ok := authtypes.ClaimsFromContext(r.Context())
-	if !ok {
-		render.Error(rw, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
-		return
-	}
-
-	preference, err := p.usecase.GetUserPreference(
-		r.Context(), preferenceId, claims.OrgID, claims.UserID,
-	)
-	if err != nil {
-		render.Error(rw, err)
-		return
-	}
-
-	render.Success(rw, http.StatusOK, preference)
-}
-
-func (p *preferenceAPI) UpdateUserPreference(rw http.ResponseWriter, r *http.Request) {
-	preferenceId := mux.Vars(r)["preferenceId"]
-	claims, ok := authtypes.ClaimsFromContext(r.Context())
-	if !ok {
-		render.Error(rw, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
-		return
-	}
-	req := preferencetypes.UpdatablePreference{}
-
-	err := json.NewDecoder(r.Body).Decode(&req)
-
-	if err != nil {
-		render.Error(rw, err)
-		return
-	}
-	err = p.usecase.UpdateUserPreference(r.Context(), preferenceId, req.PreferenceValue, claims.UserID)
-	if err != nil {
-		render.Error(rw, err)
-		return
-	}
-
-	render.Success(rw, http.StatusNoContent, nil)
-}
-
-func (p *preferenceAPI) GetAllUserPreferences(rw http.ResponseWriter, r *http.Request) {
-	claims, ok := authtypes.ClaimsFromContext(r.Context())
-	if !ok {
-		render.Error(rw, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
-		return
-	}
-	preferences, err := p.usecase.GetAllUserPreferences(
-		r.Context(), claims.OrgID, claims.UserID,
-	)
-	if err != nil {
-		render.Error(rw, err)
-		return
-	}
-
-	render.Success(rw, http.StatusOK, preferences)
-}

pkg/modules/preference/core/preference.go

@@ -1,278 +0,0 @@
-package core
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"fmt"
-
-	"github.com/SigNoz/signoz/pkg/errors"
-	"github.com/SigNoz/signoz/pkg/modules/preference"
-	"github.com/SigNoz/signoz/pkg/types/preferencetypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
-)
-
-type usecase struct {
-	store      preferencetypes.PreferenceStore
-	defaultMap map[string]preferencetypes.Preference
-}
-
-func NewPreference(store preferencetypes.PreferenceStore, defaultMap map[string]preferencetypes.Preference) preference.Usecase {
-	return &usecase{store: store, defaultMap: defaultMap}
-}
-
-func (usecase *usecase) GetOrgPreference(ctx context.Context, preferenceID string, orgID string) (*preferencetypes.GettablePreference, error) {
-	preference, seen := usecase.defaultMap[preferenceID]
-	if !seen {
-		return nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, fmt.Sprintf("no such preferenceID exists: %s", preferenceID))
-	}
-
-	isPreferenceEnabled := preference.IsEnabledForScope(preferencetypes.OrgAllowedScope)
-	if !isPreferenceEnabled {
-		return nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, fmt.Sprintf("preference is not enabled at org scope: %s", preferenceID))
-	}
-
-	orgPreference, err := usecase.store.GetOrgPreference(ctx, orgID, preferenceID)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return &preferencetypes.GettablePreference{
-				PreferenceID:    preferenceID,
-				PreferenceValue: preference.DefaultValue,
-			}, nil
-		}
-		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, fmt.Sprintf("error in fetching the org preference: %s", preferenceID))
-	}
-
-	return &preferencetypes.GettablePreference{
-		PreferenceID:    preferenceID,
-		PreferenceValue: preference.SanitizeValue(orgPreference.PreferenceValue),
-	}, nil
-}
-
-func (usecase *usecase) UpdateOrgPreference(ctx context.Context, preferenceID string, preferenceValue interface{}, orgId string) error {
-	preference, seen := usecase.defaultMap[preferenceID]
-	if !seen {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, fmt.Sprintf("no such preferenceID exists: %s", preferenceID))
-	}
-
-	isPreferenceEnabled := preference.IsEnabledForScope(preferencetypes.OrgAllowedScope)
-	if !isPreferenceEnabled {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, fmt.Sprintf("preference is not enabled at org scope: %s", preferenceID))
-	}
-
-	err := preference.IsValidValue(preferenceValue)
-	if err != nil {
-		return err
-	}
-
-	storablePreferenceValue, encodeErr := json.Marshal(preferenceValue)
-	if encodeErr != nil {
-		return errors.Wrapf(encodeErr, errors.TypeInvalidInput, errors.CodeInvalidInput, "error in encoding the preference value")
-	}
-
-	orgPreference, dberr := usecase.store.GetOrgPreference(ctx, orgId, preferenceID)
-	if dberr != nil && dberr != sql.ErrNoRows {
-		return errors.Wrapf(dberr, errors.TypeInternal, errors.CodeInternal, "error in getting the preference value")
-	}
-
-	if dberr != nil {
-		orgPreference.ID = valuer.GenerateUUID()
-		orgPreference.PreferenceID = preferenceID
-		orgPreference.PreferenceValue = string(storablePreferenceValue)
-		orgPreference.OrgID = orgId
-	} else {
-		orgPreference.PreferenceValue = string(storablePreferenceValue)
-	}
-
-	dberr = usecase.store.UpsertOrgPreference(ctx, orgPreference)
-	if dberr != nil {
-		return errors.Wrapf(dberr, errors.TypeInternal, errors.CodeInternal, "error in setting the preference value")
-	}
-
-	return nil
-}
-
-func (usecase *usecase) GetAllOrgPreferences(ctx context.Context, orgID string) ([]*preferencetypes.PreferenceWithValue, error) {
-	allOrgPreferences := []*preferencetypes.PreferenceWithValue{}
-	orgPreferences, err := usecase.store.GetAllOrgPreferences(ctx, orgID)
-	if err != nil {
-		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "error in setting all org preference values")
-	}
-
-	preferenceValueMap := map[string]interface{}{}
-	for _, preferenceValue := range orgPreferences {
-		preferenceValueMap[preferenceValue.PreferenceID] = preferenceValue.PreferenceValue
-	}
-
-	for _, preference := range usecase.defaultMap {
-		isEnabledForOrgScope := preference.IsEnabledForScope(preferencetypes.OrgAllowedScope)
-		if isEnabledForOrgScope {
-			preferenceWithValue := &preferencetypes.PreferenceWithValue{}
-			preferenceWithValue.Key = preference.Key
-			preferenceWithValue.Name = preference.Name
-			preferenceWithValue.Description = preference.Description
-			preferenceWithValue.AllowedScopes = preference.AllowedScopes
-			preferenceWithValue.AllowedValues = preference.AllowedValues
-			preferenceWithValue.DefaultValue = preference.DefaultValue
-			preferenceWithValue.Range = preference.Range
-			preferenceWithValue.ValueType = preference.ValueType
-			preferenceWithValue.IsDiscreteValues = preference.IsDiscreteValues
-			value, seen := preferenceValueMap[preference.Key]
-
-			if seen {
-				preferenceWithValue.Value = value
-			} else {
-				preferenceWithValue.Value = preference.DefaultValue
-			}
-
-			preferenceWithValue.Value = preference.SanitizeValue(preferenceWithValue.Value)
-			allOrgPreferences = append(allOrgPreferences, preferenceWithValue)
-		}
-	}
-	return allOrgPreferences, nil
-}
-
-func (usecase *usecase) GetUserPreference(ctx context.Context, preferenceID string, orgId string, userId string) (*preferencetypes.GettablePreference, error) {
-	preference, seen := usecase.defaultMap[preferenceID]
-	if !seen {
-		return nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, fmt.Sprintf("no such preferenceID exists: %s", preferenceID))
-	}
-
-	preferenceValue := preferencetypes.GettablePreference{
-		PreferenceID:    preferenceID,
-		PreferenceValue: preference.DefaultValue,
-	}
-
-	isPreferenceEnabledAtUserScope := preference.IsEnabledForScope(preferencetypes.UserAllowedScope)
-	if !isPreferenceEnabledAtUserScope {
-		return nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, fmt.Sprintf("preference is not enabled at user scope: %s", preferenceID))
-	}
-
-	isPreferenceEnabledAtOrgScope := preference.IsEnabledForScope(preferencetypes.OrgAllowedScope)
-	if isPreferenceEnabledAtOrgScope {
-		orgPreference, err := usecase.store.GetOrgPreference(ctx, orgId, preferenceID)
-		if err != nil && err != sql.ErrNoRows {
-			return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, fmt.Sprintf("error in fetching the org preference: %s", preferenceID))
-		}
-		if err == nil {
-			preferenceValue.PreferenceValue = orgPreference.PreferenceValue
-		}
-	}
-
-	userPreference, err := usecase.store.GetUserPreference(ctx, userId, preferenceID)
-	if err != nil && err != sql.ErrNoRows {
-		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, fmt.Sprintf("error in fetching the user preference: %s", preferenceID))
-	}
-
-	if err == nil {
-		preferenceValue.PreferenceValue = userPreference.PreferenceValue
-	}
-
-	return &preferencetypes.GettablePreference{
-		PreferenceID:    preferenceValue.PreferenceID,
-		PreferenceValue: preference.SanitizeValue(preferenceValue.PreferenceValue),
-	}, nil
-}
-
-func (usecase *usecase) UpdateUserPreference(ctx context.Context, preferenceID string, preferenceValue interface{}, userId string) error {
-	preference, seen := usecase.defaultMap[preferenceID]
-	if !seen {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, fmt.Sprintf("no such preferenceID exists: %s", preferenceID))
-	}
-
-	isPreferenceEnabledAtUserScope := preference.IsEnabledForScope(preferencetypes.UserAllowedScope)
-	if !isPreferenceEnabledAtUserScope {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, fmt.Sprintf("preference is not enabled at user scope: %s", preferenceID))
-	}
-
-	err := preference.IsValidValue(preferenceValue)
-	if err != nil {
-		return err
-	}
-
-	storablePreferenceValue, encodeErr := json.Marshal(preferenceValue)
-	if encodeErr != nil {
-		return errors.Wrapf(encodeErr, errors.TypeInvalidInput, errors.CodeInvalidInput, "error in encoding the preference value")
-	}
-
-	userPreference, dberr := usecase.store.GetUserPreference(ctx, userId, preferenceID)
-	if dberr != nil && dberr != sql.ErrNoRows {
-		return errors.Wrapf(dberr, errors.TypeInternal, errors.CodeInternal, "error in getting the preference value")
-	}
-
-	if dberr != nil {
-		userPreference.ID = valuer.GenerateUUID()
-		userPreference.PreferenceID = preferenceID
-		userPreference.PreferenceValue = string(storablePreferenceValue)
-		userPreference.UserID = userId
-	} else {
-		userPreference.PreferenceValue = string(storablePreferenceValue)
-	}
-
-	dberr = usecase.store.UpsertUserPreference(ctx, userPreference)
-	if dberr != nil {
-		return errors.Wrapf(dberr, errors.TypeInternal, errors.CodeInternal, "error in setting the preference value")
-	}
-
-	return nil
-}
-
-func (usecase *usecase) GetAllUserPreferences(ctx context.Context, orgID string, userID string) ([]*preferencetypes.PreferenceWithValue, error) {
-	allUserPreferences := []*preferencetypes.PreferenceWithValue{}
-
-	orgPreferences, err := usecase.store.GetAllOrgPreferences(ctx, orgID)
-	if err != nil {
-		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "error in setting all org preference values")
-	}
-
-	preferenceOrgValueMap := map[string]interface{}{}
-	for _, preferenceValue := range orgPreferences {
-		preferenceOrgValueMap[preferenceValue.PreferenceID] = preferenceValue.PreferenceValue
-	}
-
-	userPreferences, err := usecase.store.GetAllUserPreferences(ctx, userID)
-	if err != nil {
-		return nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "error in setting all user preference values")
-	}
-
-	preferenceUserValueMap := map[string]interface{}{}
-	for _, preferenceValue := range userPreferences {
-		preferenceUserValueMap[preferenceValue.PreferenceID] = preferenceValue.PreferenceValue
-	}
-
-	for _, preference := range usecase.defaultMap {
-		isEnabledForUserScope := preference.IsEnabledForScope(preferencetypes.UserAllowedScope)
-
-		if isEnabledForUserScope {
-			preferenceWithValue := &preferencetypes.PreferenceWithValue{}
-			preferenceWithValue.Key = preference.Key
-			preferenceWithValue.Name = preference.Name
-			preferenceWithValue.Description = preference.Description
-			preferenceWithValue.AllowedScopes = preference.AllowedScopes
-			preferenceWithValue.AllowedValues = preference.AllowedValues
-			preferenceWithValue.DefaultValue = preference.DefaultValue
-			preferenceWithValue.Range = preference.Range
-			preferenceWithValue.ValueType = preference.ValueType
-			preferenceWithValue.IsDiscreteValues = preference.IsDiscreteValues
-			preferenceWithValue.Value = preference.DefaultValue
-
-			isEnabledForOrgScope := preference.IsEnabledForScope(preferencetypes.OrgAllowedScope)
-			if isEnabledForOrgScope {
-				value, seen := preferenceOrgValueMap[preference.Key]
-				if seen {
-					preferenceWithValue.Value = value
-				}
-			}
-
-			value, seen := preferenceUserValueMap[preference.Key]
-
-			if seen {
-				preferenceWithValue.Value = value
-			}
-
-			preferenceWithValue.Value = preference.SanitizeValue(preferenceWithValue.Value)
-			allUserPreferences = append(allUserPreferences, preferenceWithValue)
-		}
-	}
-	return allUserPreferences, nil
-}

pkg/modules/preference/core/store.go

@@ -1,116 +0,0 @@
-package core
-
-import (
-	"context"
-
-	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/types/preferencetypes"
-)
-
-type store struct {
-	store sqlstore.SQLStore
-}
-
-func NewStore(db sqlstore.SQLStore) preferencetypes.PreferenceStore {
-	return &store{store: db}
-}
-
-func (store *store) GetOrgPreference(ctx context.Context, orgID string, preferenceID string) (*preferencetypes.StorableOrgPreference, error) {
-	orgPreference := new(preferencetypes.StorableOrgPreference)
-	err := store.
-		store.
-		BunDB().
-		NewSelect().
-		Model(orgPreference).
-		Where("preference_id = ?", preferenceID).
-		Where("org_id = ?", orgID).
-		Scan(ctx)
-
-	if err != nil {
-		return orgPreference, err
-	}
-
-	return orgPreference, nil
-}
-
-func (store *store) GetAllOrgPreferences(ctx context.Context, orgID string) ([]*preferencetypes.StorableOrgPreference, error) {
-	orgPreferences := make([]*preferencetypes.StorableOrgPreference, 0)
-	err := store.
-		store.
-		BunDB().
-		NewSelect().
-		Model(&orgPreferences).
-		Where("org_id = ?", orgID).
-		Scan(ctx)
-
-	if err != nil {
-		return orgPreferences, err
-	}
-
-	return orgPreferences, nil
-}
-
-func (store *store) UpsertOrgPreference(ctx context.Context, orgPreference *preferencetypes.StorableOrgPreference) error {
-	_, err := store.
-		store.
-		BunDB().
-		NewInsert().
-		Model(orgPreference).
-		On("CONFLICT (id) DO UPDATE").
-		Exec(ctx)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (store *store) GetUserPreference(ctx context.Context, userID string, preferenceID string) (*preferencetypes.StorableUserPreference, error) {
-	userPreference := new(preferencetypes.StorableUserPreference)
-	err := store.
-		store.
-		BunDB().
-		NewSelect().
-		Model(userPreference).
-		Where("preference_id = ?", preferenceID).
-		Where("user_id = ?", userID).
-		Scan(ctx)
-
-	if err != nil {
-		return userPreference, err
-	}
-
-	return userPreference, nil
-}
-
-func (store *store) GetAllUserPreferences(ctx context.Context, userID string) ([]*preferencetypes.StorableUserPreference, error) {
-	userPreferences := make([]*preferencetypes.StorableUserPreference, 0)
-	err := store.
-		store.
-		BunDB().
-		NewSelect().
-		Model(&userPreferences).
-		Where("user_id = ?", userID).
-		Scan(ctx)
-
-	if err != nil {
-		return userPreferences, err
-	}
-
-	return userPreferences, nil
-}
-
-func (store *store) UpsertUserPreference(ctx context.Context, userPreference *preferencetypes.StorableUserPreference) error {
-	_, err := store.
-		store.
-		BunDB().
-		NewInsert().
-		Model(userPreference).
-		On("CONFLICT (id) DO UPDATE").
-		Exec(ctx)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

pkg/modules/preference/usecase.go

@@ -1,17 +0,0 @@
-package preference
-
-import (
-	"context"
-
-	"github.com/SigNoz/signoz/pkg/types/preferencetypes"
-)
-
-type Usecase interface {
-	GetOrgPreference(ctx context.Context, preferenceId string, orgId string) (*preferencetypes.GettablePreference, error)
-	UpdateOrgPreference(ctx context.Context, preferenceId string, preferenceValue interface{}, orgId string) error
-	GetAllOrgPreferences(ctx context.Context, orgId string) ([]*preferencetypes.PreferenceWithValue, error)
-
-	GetUserPreference(ctx context.Context, preferenceId string, orgId string, userId string) (*preferencetypes.GettablePreference, error)
-	UpdateUserPreference(ctx context.Context, preferenceId string, preferenceValue interface{}, userId string) error
-	GetAllUserPreferences(ctx context.Context, orgId string, userId string) ([]*preferencetypes.PreferenceWithValue, error)
-}

pkg/query-service/agentConf/agent_features.go

@@ -1,9 +1,6 @@
 package agentConf
 
-import (
-	"github.com/SigNoz/signoz/pkg/query-service/model"
-	"github.com/SigNoz/signoz/pkg/types"
-)
+import "github.com/SigNoz/signoz/pkg/query-service/model"
 
 // Interface for features implemented via agent config.
 // Eg: ingestion side signal pre-processing features like log processing pipelines etc
@@ -14,9 +11,8 @@ type AgentFeature interface {
 	// Recommend config for an agent based on its `currentConfYaml` and
 	// `configVersion` for the feature's settings
 	RecommendAgentConfig(
-		orgId string,
 		currentConfYaml []byte,
-		configVersion *types.AgentConfigVersion,
+		configVersion *ConfigVersion,
 	) (
 		recommendedConfYaml []byte,
 

pkg/query-service/agentConf/db.go

@@ -6,9 +6,8 @@ import (
 	"fmt"
 
 	"github.com/SigNoz/signoz/pkg/query-service/model"
-	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/types"
-	"github.com/SigNoz/signoz/pkg/valuer"
+	"github.com/google/uuid"
+	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	"golang.org/x/exp/slices"
@@ -16,33 +15,42 @@ import (
 
 // Repo handles DDL and DML ops on ingestion rules
 type Repo struct {
-	store sqlstore.SQLStore
+	db *sqlx.DB
 }
 
 func (r *Repo) GetConfigHistory(
-	ctx context.Context, orgId string, typ types.ElementTypeDef, limit int,
-) ([]types.AgentConfigVersion, *model.ApiError) {
-	var c []types.AgentConfigVersion
-	err := r.store.BunDB().NewSelect().
-		Model(&c).
-		ColumnExpr("id, version, element_type, active, is_valid, disabled, deploy_status, deploy_result, created_at").
-		ColumnExpr("COALESCE(created_by, '') as created_by").
-		ColumnExpr(`COALESCE((SELECT NAME FROM users WHERE users.id = acv.created_by), 'unknown') as created_by_name`).
-		ColumnExpr("COALESCE(last_hash, '') as last_hash, COALESCE(last_config, '{}') as last_config").
-		Where("acv.element_type = ?", typ).
-		Where("acv.org_id = ?", orgId).
-		OrderExpr("acv.created_at DESC, acv.version DESC").
-		Limit(limit).
-		Scan(ctx)
+	ctx context.Context, typ ElementTypeDef, limit int,
+) ([]ConfigVersion, *model.ApiError) {
+	var c []ConfigVersion
+	err := r.db.SelectContext(ctx, &c, fmt.Sprintf(`SELECT 
+		version, 
+		id, 
+		element_type, 
+		COALESCE(created_by, -1) as created_by, 
+		created_at,
+		COALESCE((SELECT NAME FROM users 
+ 		WHERE id = v.created_by), "unknown") created_by_name, 
+		active, 
+		is_valid, 
+		disabled, 
+		deploy_status, 
+		deploy_result,
+		coalesce(last_hash, '') as last_hash,
+		coalesce(last_config, '{}') as last_config
+		FROM agent_config_versions AS v
+		WHERE element_type = $1
+		ORDER BY created_at desc, version desc
+		limit %v`, limit),
+		typ)
 
 	if err != nil {
 		return nil, model.InternalError(err)
 	}
 
-	incompleteStatuses := []types.DeployStatus{types.DeployInitiated, types.Deploying}
+	incompleteStatuses := []DeployStatus{DeployInitiated, Deploying}
 	for idx := 1; idx < len(c); idx++ {
 		if slices.Contains(incompleteStatuses, c[idx].DeployStatus) {
-			c[idx].DeployStatus = types.DeployStatusUnknown
+			c[idx].DeployStatus = DeployStatusUnknown
 		}
 	}
 
@@ -50,24 +58,32 @@ func (r *Repo) GetConfigHistory(
 }
 
 func (r *Repo) GetConfigVersion(
-	ctx context.Context, orgId string, typ types.ElementTypeDef, v int,
-) (*types.AgentConfigVersion, *model.ApiError) {
-	var c types.AgentConfigVersion
-	err := r.store.BunDB().NewSelect().
-		Model(&c).
-		ColumnExpr("id, version, element_type, active, is_valid, disabled, deploy_status, deploy_result, created_at").
-		ColumnExpr("COALESCE(created_by, '') as created_by").
-		ColumnExpr(`COALESCE((SELECT NAME FROM users WHERE users.id = acv.created_by), 'unknown') as created_by_name`).
-		ColumnExpr("COALESCE(last_hash, '') as last_hash, COALESCE(last_config, '{}') as last_config").
-		Where("acv.element_type = ?", typ).
-		Where("acv.version = ?", v).
-		Where("acv.org_id = ?", orgId).
-		Scan(ctx)
+	ctx context.Context, typ ElementTypeDef, v int,
+) (*ConfigVersion, *model.ApiError) {
+	var c ConfigVersion
+	err := r.db.GetContext(ctx, &c, `SELECT 
+		id, 
+		version, 
+		element_type,
+		COALESCE(created_by, -1) as created_by, 
+		created_at,
+		COALESCE((SELECT NAME FROM users 
+		WHERE id = v.created_by), "unknown") created_by_name,
+		active, 
+		is_valid, 
+		disabled, 
+		deploy_status, 
+		deploy_result,
+		coalesce(last_hash, '') as last_hash,
+		coalesce(last_config, '{}') as last_config
+		FROM agent_config_versions v 
+		WHERE element_type = $1 
+		AND version = $2`, typ, v)
 
+	if err == sql.ErrNoRows {
+		return nil, model.NotFoundError(err)
+	}
 	if err != nil {
-		if errors.Is(err, sql.ErrNoRows) {
-			return nil, model.NotFoundError(err)
-		}
 		return nil, model.InternalError(err)
 	}
 
@@ -75,23 +91,33 @@ func (r *Repo) GetConfigVersion(
 }
 
 func (r *Repo) GetLatestVersion(
-	ctx context.Context, orgId string, typ types.ElementTypeDef,
-) (*types.AgentConfigVersion, *model.ApiError) {
-	var c types.AgentConfigVersion
-	err := r.store.BunDB().NewSelect().
-		Model(&c).
-		ColumnExpr("id, version, element_type, active, is_valid, disabled, deploy_status, deploy_result, created_at").
-		ColumnExpr("COALESCE(created_by, '') as created_by").
-		ColumnExpr(`COALESCE((SELECT NAME FROM users WHERE users.id = acv.created_by), 'unknown') as created_by_name`).
-		Where("acv.element_type = ?", typ).
-		Where("acv.org_id = ?", orgId).
-		Where("version = (SELECT MAX(version) FROM agent_config_versions WHERE acv.element_type = ?)", typ).
-		Scan(ctx)
+	ctx context.Context, typ ElementTypeDef,
+) (*ConfigVersion, *model.ApiError) {
+	var c ConfigVersion
+	err := r.db.GetContext(ctx, &c, `SELECT 
+		id, 
+		version, 
+		element_type, 
+		COALESCE(created_by, -1) as created_by, 
+		created_at,
+		COALESCE((SELECT NAME FROM users 
+ 		WHERE id = v.created_by), "unknown") created_by_name, 
+		active, 
+		is_valid, 
+		disabled, 
+		deploy_status, 
+		deploy_result 
+		FROM agent_config_versions AS v
+		WHERE element_type = $1 
+		AND version = ( 
+			SELECT MAX(version) 
+			FROM agent_config_versions 
+			WHERE element_type=$2)`, typ, typ)
 
+	if err == sql.ErrNoRows {
+		return nil, model.NotFoundError(err)
+	}
 	if err != nil {
-		if errors.Is(err, sql.ErrNoRows) {
-			return nil, model.NotFoundError(err)
-		}
 		return nil, model.InternalError(err)
 	}
 
@@ -99,7 +125,7 @@ func (r *Repo) GetLatestVersion(
 }
 
 func (r *Repo) insertConfig(
-	ctx context.Context, orgId string, userId string, c *types.AgentConfigVersion, elements []string,
+	ctx context.Context, userId string, c *ConfigVersion, elements []string,
 ) (fnerr *model.ApiError) {
 
 	if string(c.ElementType) == "" {
@@ -109,7 +135,7 @@ func (r *Repo) insertConfig(
 	}
 
 	// allowing empty elements for logs - use case is deleting all pipelines
-	if len(elements) == 0 && c.ElementType != types.ElementTypeLogPipelines {
+	if len(elements) == 0 && c.ElementType != ElementTypeLogPipelines {
 		zap.L().Error("insert config called with no elements ", zap.String("ElementType", string(c.ElementType)))
 		return model.BadRequest(fmt.Errorf("config must have atleast one element"))
 	}
@@ -124,14 +150,14 @@ func (r *Repo) insertConfig(
 		))
 	}
 
-	configVersion, err := r.GetLatestVersion(ctx, orgId, c.ElementType)
+	configVersion, err := r.GetLatestVersion(ctx, c.ElementType)
 	if err != nil && err.Type() != model.ErrorNotFound {
 		zap.L().Error("failed to fetch latest config version", zap.Error(err))
 		return model.InternalError(fmt.Errorf("failed to fetch latest config version"))
 	}
 
 	if configVersion != nil {
-		c.Version = types.UpdateVersion(configVersion.Version)
+		c.Version = updateVersion(configVersion.Version)
 	} else {
 		// first version
 		c.Version = 1
@@ -140,43 +166,57 @@ func (r *Repo) insertConfig(
 	defer func() {
 		if fnerr != nil {
 			// remove all the damage (invalid rows from db)
-			r.store.BunDB().NewDelete().Model((*types.AgentConfigVersion)(nil)).Where("id = ?", c.ID).Where("org_id = ?", orgId).Exec(ctx)
-			r.store.BunDB().NewDelete().Model((*types.AgentConfigElement)(nil)).Where("version_id = ?", c.ID).Where("org_id = ?", orgId).Exec(ctx)
+			_, _ = r.db.Exec("DELETE FROM agent_config_versions WHERE id = $1", c.ID)
+			_, _ = r.db.Exec("DELETE FROM agent_config_elements WHERE version_id=$1", c.ID)
 		}
 	}()
 
 	// insert config
-	_, dbErr := r.store.BunDB().NewInsert().
-		Model(&types.AgentConfigVersion{
-			OrgID:        orgId,
-			Identifiable: types.Identifiable{ID: c.ID},
-			Version:      c.Version,
-			UserAuditable: types.UserAuditable{
-				CreatedBy: userId,
-			},
-			ElementType:  c.ElementType,
-			Active:       false, // default value
-			IsValid:      false, // default value
-			Disabled:     false, // default value
-			DeployStatus: c.DeployStatus,
-			DeployResult: c.DeployResult,
-		}).
-		Exec(ctx)
+	configQuery := `INSERT INTO agent_config_versions(	
+		id, 
+		version, 
+		created_by,
+		element_type, 
+		active, 
+		is_valid, 
+		disabled,
+		deploy_status, 
+		deploy_result) 
+	VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`
+
+	_, dbErr := r.db.ExecContext(ctx,
+		configQuery,
+		c.ID,
+		c.Version,
+		userId,
+		c.ElementType,
+		false,
+		false,
+		false,
+		c.DeployStatus,
+		c.DeployResult)
 
 	if dbErr != nil {
 		zap.L().Error("error in inserting config version: ", zap.Error(dbErr))
 		return model.InternalError(errors.Wrap(dbErr, "failed to insert ingestion rule"))
 	}
 
+	elementsQuery := `INSERT INTO agent_config_elements(	
+		id, 
+		version_id, 
+		element_type, 
+		element_id) 
+	VALUES ($1, $2, $3, $4)`
+
 	for _, e := range elements {
-		agentConfigElement := &types.AgentConfigElement{
-			OrgID:        orgId,
-			Identifiable: types.Identifiable{ID: valuer.GenerateUUID()},
-			VersionID:    c.ID.StringValue(),
-			ElementType:  string(c.ElementType),
-			ElementID:    e,
-		}
-		_, dbErr = r.store.BunDB().NewInsert().Model(agentConfigElement).Exec(ctx)
+		_, dbErr = r.db.ExecContext(
+			ctx,
+			elementsQuery,
+			uuid.NewString(),
+			c.ID,
+			c.ElementType,
+			e,
+		)
 		if dbErr != nil {
 			return model.InternalError(dbErr)
 		}
@@ -186,43 +226,40 @@ func (r *Repo) insertConfig(
 }
 
 func (r *Repo) updateDeployStatus(ctx context.Context,
-	orgId string,
-	elementType types.ElementTypeDef,
+	elementType ElementTypeDef,
 	version int,
 	status string,
 	result string,
 	lastHash string,
 	lastconf string) *model.ApiError {
 
-	_, err := r.store.BunDB().NewUpdate().
-		Model((*types.AgentConfigVersion)(nil)).
-		Set("deploy_status = ?", status).
-		Set("deploy_result = ?", result).
-		Set("last_hash = COALESCE(?, last_hash)", lastHash).
-		Set("last_config = ?", lastconf).
-		Where("version = ?", version).
-		Where("element_type = ?", elementType).
-		Where("org_id = ?", orgId).
-		Exec(ctx)
+	updateQuery := `UPDATE agent_config_versions
+	set deploy_status = $1, 
+	deploy_result = $2,
+	last_hash = COALESCE($3, last_hash),
+	last_config = $4
+	WHERE version=$5
+	AND element_type = $6`
+
+	_, err := r.db.ExecContext(ctx, updateQuery, status, result, lastHash, lastconf, version, string(elementType))
 	if err != nil {
 		zap.L().Error("failed to update deploy status", zap.Error(err))
-		return model.BadRequest(fmt.Errorf("failed to update deploy status"))
+		return model.BadRequest(fmt.Errorf("failed to  update deploy status"))
 	}
 
 	return nil
 }
 
 func (r *Repo) updateDeployStatusByHash(
-	ctx context.Context, orgId string, confighash string, status string, result string,
+	ctx context.Context, confighash string, status string, result string,
 ) *model.ApiError {
 
-	_, err := r.store.BunDB().NewUpdate().
-		Model((*types.AgentConfigVersion)(nil)).
-		Set("deploy_status = ?", status).
-		Set("deploy_result = ?", result).
-		Where("last_hash = ?", confighash).
-		Where("org_id = ?", orgId).
-		Exec(ctx)
+	updateQuery := `UPDATE agent_config_versions
+	set deploy_status = $1, 
+	deploy_result = $2
+	WHERE last_hash=$4`
+
+	_, err := r.db.ExecContext(ctx, updateQuery, status, result, confighash)
 	if err != nil {
 		zap.L().Error("failed to update deploy status", zap.Error(err))
 		return model.InternalError(errors.Wrap(err, "failed to update deploy status"))

pkg/query-service/agentConf/manager.go

@@ -12,9 +12,8 @@ import (
 	filterprocessor "github.com/SigNoz/signoz/pkg/query-service/app/opamp/otelconfig/filterprocessor"
 	tsp "github.com/SigNoz/signoz/pkg/query-service/app/opamp/otelconfig/tailsampler"
 	"github.com/SigNoz/signoz/pkg/query-service/model"
-	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/types"
 	"github.com/google/uuid"
+	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 	yaml "gopkg.in/yaml.v3"
@@ -40,7 +39,7 @@ type Manager struct {
 }
 
 type ManagerOptions struct {
-	Store sqlstore.SQLStore
+	DB *sqlx.DB
 
 	// When acting as opamp.AgentConfigProvider, agent conf recommendations are
 	// applied to the base conf in the order the features have been specified here.
@@ -61,7 +60,7 @@ func Initiate(options *ManagerOptions) (*Manager, error) {
 	}
 
 	m = &Manager{
-		Repo:              Repo{options.Store},
+		Repo:              Repo{options.DB},
 		agentFeatures:     options.AgentFeatures,
 		configSubscribers: map[string]func(){},
 	}
@@ -91,7 +90,7 @@ func (m *Manager) notifyConfigUpdateSubscribers() {
 }
 
 // Implements opamp.AgentConfigProvider
-func (m *Manager) RecommendAgentConfig(orgId string, currentConfYaml []byte) (
+func (m *Manager) RecommendAgentConfig(currentConfYaml []byte) (
 	recommendedConfYaml []byte,
 	// Opaque id of the recommended config, used for reporting deployment status updates
 	configId string,
@@ -101,13 +100,13 @@ func (m *Manager) RecommendAgentConfig(orgId string, currentConfYaml []byte) (
 	settingVersionsUsed := []string{}
 
 	for _, feature := range m.agentFeatures {
-		featureType := types.ElementTypeDef(feature.AgentFeatureType())
-		latestConfig, apiErr := GetLatestVersion(context.Background(), orgId, featureType)
+		featureType := ElementTypeDef(feature.AgentFeatureType())
+		latestConfig, apiErr := GetLatestVersion(context.Background(), featureType)
 		if apiErr != nil && apiErr.Type() != model.ErrorNotFound {
 			return nil, "", errors.Wrap(apiErr.ToError(), "failed to get latest agent config version")
 		}
 
-		updatedConf, serializedSettingsUsed, apiErr := feature.RecommendAgentConfig(orgId, recommendation, latestConfig)
+		updatedConf, serializedSettingsUsed, apiErr := feature.RecommendAgentConfig(recommendation, latestConfig)
 		if apiErr != nil {
 			return nil, "", errors.Wrap(apiErr.ToError(), fmt.Sprintf(
 				"failed to generate agent config recommendation for %s", featureType,
@@ -130,10 +129,9 @@ func (m *Manager) RecommendAgentConfig(orgId string, currentConfYaml []byte) (
 
 		_ = m.updateDeployStatus(
 			context.Background(),
-			orgId,
 			featureType,
 			configVersion,
-			string(types.DeployInitiated),
+			string(DeployInitiated),
 			"Deployment has started",
 			configId,
 			serializedSettingsUsed,
@@ -156,53 +154,52 @@ func (m *Manager) RecommendAgentConfig(orgId string, currentConfYaml []byte) (
 
 // Implements opamp.AgentConfigProvider
 func (m *Manager) ReportConfigDeploymentStatus(
-	orgId string,
 	agentId string,
 	configId string,
 	err error,
 ) {
 	featureConfigIds := strings.Split(configId, ",")
 	for _, featureConfId := range featureConfigIds {
-		newStatus := string(types.Deployed)
+		newStatus := string(Deployed)
 		message := "Deployment was successful"
 		if err != nil {
-			newStatus = string(types.DeployFailed)
+			newStatus = string(DeployFailed)
 			message = fmt.Sprintf("%s: %s", agentId, err.Error())
 		}
 		_ = m.updateDeployStatusByHash(
-			context.Background(), orgId, featureConfId, newStatus, message,
+			context.Background(), featureConfId, newStatus, message,
 		)
 	}
 }
 
 func GetLatestVersion(
-	ctx context.Context, orgId string, elementType types.ElementTypeDef,
-) (*types.AgentConfigVersion, *model.ApiError) {
-	return m.GetLatestVersion(ctx, orgId, elementType)
+	ctx context.Context, elementType ElementTypeDef,
+) (*ConfigVersion, *model.ApiError) {
+	return m.GetLatestVersion(ctx, elementType)
 }
 
 func GetConfigVersion(
-	ctx context.Context, orgId string, elementType types.ElementTypeDef, version int,
-) (*types.AgentConfigVersion, *model.ApiError) {
-	return m.GetConfigVersion(ctx, orgId, elementType, version)
+	ctx context.Context, elementType ElementTypeDef, version int,
+) (*ConfigVersion, *model.ApiError) {
+	return m.GetConfigVersion(ctx, elementType, version)
 }
 
 func GetConfigHistory(
-	ctx context.Context, orgId string, typ types.ElementTypeDef, limit int,
-) ([]types.AgentConfigVersion, *model.ApiError) {
-	return m.GetConfigHistory(ctx, orgId, typ, limit)
+	ctx context.Context, typ ElementTypeDef, limit int,
+) ([]ConfigVersion, *model.ApiError) {
+	return m.GetConfigHistory(ctx, typ, limit)
 }
 
 // StartNewVersion launches a new config version for given set of elements
 func StartNewVersion(
-	ctx context.Context, orgId string, userId string, eleType types.ElementTypeDef, elementIds []string,
-) (*types.AgentConfigVersion, *model.ApiError) {
+	ctx context.Context, userId string, eleType ElementTypeDef, elementIds []string,
+) (*ConfigVersion, *model.ApiError) {
 
 	// create a new version
-	cfg := types.NewAgentConfigVersion(orgId, eleType)
+	cfg := NewConfigVersion(eleType)
 
 	// insert new config and elements into database
-	err := m.insertConfig(ctx, orgId, userId, cfg, elementIds)
+	err := m.insertConfig(ctx, userId, cfg, elementIds)
 	if err != nil {
 		return nil, err
 	}
@@ -216,22 +213,22 @@ func NotifyConfigUpdate(ctx context.Context) {
 	m.notifyConfigUpdateSubscribers()
 }
 
-func Redeploy(ctx context.Context, orgId string, typ types.ElementTypeDef, version int) *model.ApiError {
+func Redeploy(ctx context.Context, typ ElementTypeDef, version int) *model.ApiError {
 
-	configVersion, err := GetConfigVersion(ctx, orgId, typ, version)
+	configVersion, err := GetConfigVersion(ctx, typ, version)
 	if err != nil {
 		zap.L().Error("failed to fetch config version during redeploy", zap.Error(err))
 		return model.WrapApiError(err, "failed to fetch details of the config version")
 	}
 
-	if configVersion == nil || (configVersion != nil && configVersion.LastConfig == "") {
+	if configVersion == nil || (configVersion != nil && configVersion.LastConf == "") {
 		zap.L().Debug("config version has no conf yaml", zap.Any("configVersion", configVersion))
 		return model.BadRequest(fmt.Errorf("the config version can not be redeployed"))
 	}
 	switch typ {
-	case types.ElementTypeSamplingRules:
+	case ElementTypeSamplingRules:
 		var config *tsp.Config
-		if err := yaml.Unmarshal([]byte(configVersion.LastConfig), &config); err != nil {
+		if err := yaml.Unmarshal([]byte(configVersion.LastConf), &config); err != nil {
 			zap.L().Debug("failed to read last conf correctly", zap.Error(err))
 			return model.BadRequest(fmt.Errorf("failed to read the stored config correctly"))
 		}
@@ -248,10 +245,10 @@ func Redeploy(ctx context.Context, orgId string, typ types.ElementTypeDef, versi
 			return model.InternalError(fmt.Errorf("failed to deploy the config"))
 		}
 
-		m.updateDeployStatus(ctx, orgId, types.ElementTypeSamplingRules, version, string(types.DeployInitiated), "Deployment started", configHash, configVersion.LastConfig)
-	case types.ElementTypeDropRules:
+		_ = m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, configVersion.LastConf)
+	case ElementTypeDropRules:
 		var filterConfig *filterprocessor.Config
-		if err := yaml.Unmarshal([]byte(configVersion.LastConfig), &filterConfig); err != nil {
+		if err := yaml.Unmarshal([]byte(configVersion.LastConf), &filterConfig); err != nil {
 			zap.L().Error("failed to read last conf correctly", zap.Error(err))
 			return model.InternalError(fmt.Errorf("failed to read the stored config correctly"))
 		}
@@ -266,14 +263,14 @@ func Redeploy(ctx context.Context, orgId string, typ types.ElementTypeDef, versi
 			return err
 		}
 
-		m.updateDeployStatus(ctx, orgId, types.ElementTypeSamplingRules, version, string(types.DeployInitiated), "Deployment started", configHash, configVersion.LastConfig)
+		_ = m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, configVersion.LastConf)
 	}
 
 	return nil
 }
 
 // UpsertFilterProcessor updates the agent config with new filter processor params
-func UpsertFilterProcessor(ctx context.Context, orgId string, version int, config *filterprocessor.Config) error {
+func UpsertFilterProcessor(ctx context.Context, version int, config *filterprocessor.Config) error {
 	if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) {
 		return fmt.Errorf("agent updater is busy")
 	}
@@ -297,7 +294,7 @@ func UpsertFilterProcessor(ctx context.Context, orgId string, version int, confi
 		zap.L().Warn("unexpected error while transforming processor config to yaml", zap.Error(yamlErr))
 	}
 
-	m.updateDeployStatus(ctx, orgId, types.ElementTypeDropRules, version, string(types.DeployInitiated), "Deployment started", configHash, string(processorConfYaml))
+	_ = m.updateDeployStatus(ctx, ElementTypeDropRules, version, string(DeployInitiated), "Deployment started", configHash, string(processorConfYaml))
 	return nil
 }
 
@@ -306,9 +303,9 @@ func UpsertFilterProcessor(ctx context.Context, orgId string, version int, confi
 // successful deployment if no error is received.
 // this method is currently expected to be called only once in the lifecycle
 // but can be improved in future to accept continuous request status updates from opamp
-func (m *Manager) OnConfigUpdate(orgId string, agentId string, hash string, err error) {
+func (m *Manager) OnConfigUpdate(agentId string, hash string, err error) {
 
-	status := string(types.Deployed)
+	status := string(Deployed)
 
 	message := "Deployment was successful"
 
@@ -317,15 +314,15 @@ func (m *Manager) OnConfigUpdate(orgId string, agentId string, hash string, err
 	}()
 
 	if err != nil {
-		status = string(types.DeployFailed)
+		status = string(DeployFailed)
 		message = fmt.Sprintf("%s: %s", agentId, err.Error())
 	}
 
-	_ = m.updateDeployStatusByHash(context.Background(), orgId, hash, status, message)
+	_ = m.updateDeployStatusByHash(context.Background(), hash, status, message)
 }
 
 // UpsertSamplingProcessor updates the agent config with new filter processor params
-func UpsertSamplingProcessor(ctx context.Context, orgId string, version int, config *tsp.Config) error {
+func UpsertSamplingProcessor(ctx context.Context, version int, config *tsp.Config) error {
 	if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) {
 		return fmt.Errorf("agent updater is busy")
 	}
@@ -348,6 +345,6 @@ func UpsertSamplingProcessor(ctx context.Context, orgId string, version int, con
 		zap.L().Warn("unexpected error while transforming processor config to yaml", zap.Error(yamlErr))
 	}
 
-	m.updateDeployStatus(ctx, orgId, types.ElementTypeSamplingRules, version, string(types.DeployInitiated), "Deployment started", configHash, string(processorConfYaml))
+	_ = m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, string(processorConfYaml))
 	return nil
 }

pkg/query-service/agentConf/version.go

@@ -1,10 +1,72 @@
 package agentConf
 
-import "github.com/SigNoz/signoz/pkg/types"
+import (
+	"time"
+
+	"github.com/google/uuid"
+)
+
+type ElementTypeDef string
+
+const (
+	ElementTypeSamplingRules ElementTypeDef = "sampling_rules"
+	ElementTypeDropRules     ElementTypeDef = "drop_rules"
+	ElementTypeLogPipelines  ElementTypeDef = "log_pipelines"
+	ElementTypeLbExporter    ElementTypeDef = "lb_exporter"
+)
+
+type DeployStatus string
+
+const (
+	PendingDeploy       DeployStatus = "DIRTY"
+	Deploying           DeployStatus = "DEPLOYING"
+	Deployed            DeployStatus = "DEPLOYED"
+	DeployInitiated     DeployStatus = "IN_PROGRESS"
+	DeployFailed        DeployStatus = "FAILED"
+	DeployStatusUnknown DeployStatus = "UNKNOWN"
+)
+
+type ConfigVersion struct {
+	ID          string         `json:"id" db:"id"`
+	Version     int            `json:"version" db:"version"`
+	ElementType ElementTypeDef `json:"elementType" db:"element_type"`
+	Active      bool           `json:"active" db:"active"`
+	IsValid     bool           `json:"is_valid" db:"is_valid"`
+	Disabled    bool           `json:"disabled" db:"disabled"`
+
+	DeployStatus DeployStatus `json:"deployStatus" db:"deploy_status"`
+	DeployResult string       `json:"deployResult" db:"deploy_result"`
+
+	LastHash string `json:"lastHash" db:"last_hash"`
+	LastConf string `json:"lastConf" db:"last_config"`
+
+	CreatedBy     string    `json:"createdBy" db:"created_by"`
+	CreatedByName string    `json:"createdByName" db:"created_by_name"`
+	CreatedAt     time.Time `json:"createdAt" db:"created_at"`
+}
+
+func NewConfigVersion(typeDef ElementTypeDef) *ConfigVersion {
+	return &ConfigVersion{
+		ID:           uuid.NewString(),
+		ElementType:  typeDef,
+		Active:       false,
+		IsValid:      false,
+		Disabled:     false,
+		DeployStatus: PendingDeploy,
+		LastHash:     "",
+		LastConf:     "{}",
+		// todo: get user id from context?
+		// CreatedBy
+	}
+}
+
+func updateVersion(v int) int {
+	return v + 1
+}
 
 type ConfigElements struct {
 	VersionID   string
 	Version     int
-	ElementType types.ElementTypeDef
+	ElementType ElementTypeDef
 	ElementId   string
 }

pkg/query-service/app/clickhouseReader/reader.go

@@ -17,10 +17,6 @@ import (
 	"time"
 
 	"github.com/SigNoz/signoz/pkg/query-service/model/metrics_explorer"
-	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/types"
-	"github.com/SigNoz/signoz/pkg/valuer"
-	"github.com/uptrace/bun"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
@@ -40,6 +36,7 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"github.com/SigNoz/signoz/pkg/cache"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
+	"github.com/jmoiron/sqlx"
 
 	promModel "github.com/prometheus/common/model"
 	"go.uber.org/zap"
@@ -123,7 +120,7 @@ var (
 // SpanWriter for reading spans from ClickHouse
 type ClickHouseReader struct {
 	db                      clickhouse.Conn
-	sqlDB                   sqlstore.SQLStore
+	localDB                 *sqlx.DB
 	TraceDB                 string
 	operationsTable         string
 	durationTable           string
@@ -177,7 +174,7 @@ type ClickHouseReader struct {
 
 // NewTraceReader returns a TraceReader for the database
 func NewReader(
-	sqlDB sqlstore.SQLStore,
+	localDB *sqlx.DB,
 	db driver.Conn,
 	configFile string,
 	featureFlag interfaces.FeatureLookup,
@@ -188,13 +185,13 @@ func NewReader(
 	cache cache.Cache,
 ) *ClickHouseReader {
 	options := NewOptions(primaryNamespace, archiveNamespace)
-	return NewReaderFromClickhouseConnection(db, options, sqlDB, configFile, featureFlag, cluster, useLogsNewSchema, useTraceNewSchema, fluxIntervalForTraceDetail, cache)
+	return NewReaderFromClickhouseConnection(db, options, localDB, configFile, featureFlag, cluster, useLogsNewSchema, useTraceNewSchema, fluxIntervalForTraceDetail, cache)
 }
 
 func NewReaderFromClickhouseConnection(
 	db driver.Conn,
 	options *Options,
-	sqlDB sqlstore.SQLStore,
+	localDB *sqlx.DB,
 	configFile string,
 	featureFlag interfaces.FeatureLookup,
 	cluster string,
@@ -219,7 +216,7 @@ func NewReaderFromClickhouseConnection(
 
 	return &ClickHouseReader{
 		db:                      db,
-		sqlDB:                   sqlDB,
+		localDB:                 localDB,
 		TraceDB:                 options.primary.TraceDB,
 		operationsTable:         options.primary.OperationsTable,
 		indexTable:              options.primary.IndexTable,
@@ -1842,7 +1839,7 @@ func getLocalTableName(tableName string) string {
 
 }
 
-func (r *ClickHouseReader) SetTTLLogsV2(ctx context.Context, orgID string, params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
+func (r *ClickHouseReader) SetTTLLogsV2(ctx context.Context, params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
 	// Keep only latest 100 transactions/requests
 	r.deleteTtlTransactions(ctx, 100)
 	// uuid is used as transaction id
@@ -1858,7 +1855,7 @@ func (r *ClickHouseReader) SetTTLLogsV2(ctx context.Context, orgID string, param
 
 	// check if there is existing things to be done
 	for _, tableName := range tableNameArray {
-		statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+		statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 		if err != nil {
 			return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
 		}
@@ -1900,27 +1897,7 @@ func (r *ClickHouseReader) SetTTLLogsV2(ctx context.Context, orgID string, param
 			// we will change ttl for only the new parts and not the old ones
 			query += " SETTINGS materialize_ttl_after_modify=0"
 
-			ttl := types.TTLSetting{
-				Identifiable: types.Identifiable{
-					ID: valuer.GenerateUUID(),
-				},
-				TimeAuditable: types.TimeAuditable{
-					CreatedAt: time.Now(),
-					UpdatedAt: time.Now(),
-				},
-				TransactionID:  uuid,
-				TableName:      tableName,
-				TTL:            int(params.DelDuration),
-				Status:         constants.StatusPending,
-				ColdStorageTTL: coldStorageDuration,
-				OrgID:          orgID,
-			}
-			_, dbErr := r.
-				sqlDB.
-				BunDB().
-				NewInsert().
-				Model(&ttl).
-				Exec(ctx)
+			_, dbErr := r.localDB.Exec("INSERT INTO ttl_status (transaction_id, created_at, updated_at, table_name, ttl, status, cold_storage_ttl) VALUES (?, ?, ?, ?, ?, ?, ?)", uuid, time.Now(), time.Now(), tableName, params.DelDuration, constants.StatusPending, coldStorageDuration)
 			if dbErr != nil {
 				zap.L().Error("error in inserting to ttl_status table", zap.Error(dbErr))
 				return
@@ -1929,17 +1906,9 @@ func (r *ClickHouseReader) SetTTLLogsV2(ctx context.Context, orgID string, param
 			err := r.setColdStorage(context.Background(), tableName, params.ColdStorageVolume)
 			if err != nil {
 				zap.L().Error("error in setting cold storage", zap.Error(err))
-				statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+				statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 				if err == nil {
-					_, dbErr := r.
-						sqlDB.
-						BunDB().
-						NewUpdate().
-						Model(new(types.TTLSetting)).
-						Set("updated_at = ?", time.Now()).
-						Set("status = ?", constants.StatusFailed).
-						Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-						Exec(ctx)
+					_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 					if dbErr != nil {
 						zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 						return
@@ -1948,33 +1917,17 @@ func (r *ClickHouseReader) SetTTLLogsV2(ctx context.Context, orgID string, param
 				return
 			}
 			zap.L().Info("Executing TTL request: ", zap.String("request", query))
-			statusItem, _ := r.checkTTLStatusItem(ctx, orgID, tableName)
+			statusItem, _ := r.checkTTLStatusItem(ctx, tableName)
 			if err := r.db.Exec(ctx, query); err != nil {
 				zap.L().Error("error while setting ttl", zap.Error(err))
-				_, dbErr := r.
-					sqlDB.
-					BunDB().
-					NewUpdate().
-					Model(new(types.TTLSetting)).
-					Set("updated_at = ?", time.Now()).
-					Set("status = ?", constants.StatusFailed).
-					Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-					Exec(ctx)
+				_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 				if dbErr != nil {
 					zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 					return
 				}
 				return
 			}
-			_, dbErr = r.
-				sqlDB.
-				BunDB().
-				NewUpdate().
-				Model(new(types.TTLSetting)).
-				Set("updated_at = ?", time.Now()).
-				Set("status = ?", constants.StatusSuccess).
-				Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-				Exec(ctx)
+			_, dbErr = r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusSuccess, statusItem.Id)
 			if dbErr != nil {
 				zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 				return
@@ -1985,7 +1938,7 @@ func (r *ClickHouseReader) SetTTLLogsV2(ctx context.Context, orgID string, param
 	return &model.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
 }
 
-func (r *ClickHouseReader) SetTTLTracesV2(ctx context.Context, orgID string, params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
+func (r *ClickHouseReader) SetTTLTracesV2(ctx context.Context, params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
 	// uuid is used as transaction id
 	uuidWithHyphen := uuid.New()
 	uuid := strings.Replace(uuidWithHyphen.String(), "-", "", -1)
@@ -2005,7 +1958,7 @@ func (r *ClickHouseReader) SetTTLTracesV2(ctx context.Context, orgID string, par
 
 	// check if there is existing things to be done
 	for _, tableName := range tableNames {
-		statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+		statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 		if err != nil {
 			return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
 		}
@@ -2032,32 +1985,11 @@ func (r *ClickHouseReader) SetTTLTracesV2(ctx context.Context, orgID string, par
 				timestamp = "end"
 			}
 
-			ttl := types.TTLSetting{
-				Identifiable: types.Identifiable{
-					ID: valuer.GenerateUUID(),
-				},
-				TimeAuditable: types.TimeAuditable{
-					CreatedAt: time.Now(),
-					UpdatedAt: time.Now(),
-				},
-				TransactionID:  uuid,
-				TableName:      tableName,
-				TTL:            int(params.DelDuration),
-				Status:         constants.StatusPending,
-				ColdStorageTTL: coldStorageDuration,
-				OrgID:          orgID,
-			}
-			_, dbErr := r.
-				sqlDB.
-				BunDB().
-				NewInsert().
-				Model(&ttl).
-				Exec(ctx)
+			_, dbErr := r.localDB.Exec("INSERT INTO ttl_status (transaction_id, created_at, updated_at, table_name, ttl, status, cold_storage_ttl) VALUES (?, ?, ?, ?, ?, ?, ?)", uuid, time.Now(), time.Now(), tableName, params.DelDuration, constants.StatusPending, coldStorageDuration)
 			if dbErr != nil {
-				zap.L().Error("error in inserting to ttl_status table", zap.Error(dbErr))
+				zap.L().Error("Error in inserting to ttl_status table", zap.Error(dbErr))
 				return
 			}
-
 			req := fmt.Sprintf(ttlV2, tableName, r.cluster, timestamp, params.DelDuration)
 			if strings.HasSuffix(distributedTableName, r.traceResourceTableV3) {
 				req = fmt.Sprintf(ttlV2Resource, tableName, r.cluster, params.DelDuration)
@@ -2073,17 +2005,9 @@ func (r *ClickHouseReader) SetTTLTracesV2(ctx context.Context, orgID string, par
 			err := r.setColdStorage(context.Background(), tableName, params.ColdStorageVolume)
 			if err != nil {
 				zap.L().Error("Error in setting cold storage", zap.Error(err))
-				statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+				statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 				if err == nil {
-					_, dbErr := r.
-						sqlDB.
-						BunDB().
-						NewUpdate().
-						Model(new(types.TTLSetting)).
-						Set("updated_at = ?", time.Now()).
-						Set("status = ?", constants.StatusFailed).
-						Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-						Exec(ctx)
+					_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 					if dbErr != nil {
 						zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 						return
@@ -2093,33 +2017,17 @@ func (r *ClickHouseReader) SetTTLTracesV2(ctx context.Context, orgID string, par
 			}
 			req += " SETTINGS materialize_ttl_after_modify=0;"
 			zap.L().Error(" ExecutingTTL request: ", zap.String("request", req))
-			statusItem, _ := r.checkTTLStatusItem(ctx, orgID, tableName)
+			statusItem, _ := r.checkTTLStatusItem(ctx, tableName)
 			if err := r.db.Exec(ctx, req); err != nil {
 				zap.L().Error("Error in executing set TTL query", zap.Error(err))
-				_, dbErr := r.
-					sqlDB.
-					BunDB().
-					NewUpdate().
-					Model(new(types.TTLSetting)).
-					Set("updated_at = ?", time.Now()).
-					Set("status = ?", constants.StatusFailed).
-					Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-					Exec(ctx)
+				_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 				if dbErr != nil {
 					zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 					return
 				}
 				return
 			}
-			_, dbErr = r.
-				sqlDB.
-				BunDB().
-				NewUpdate().
-				Model(new(types.TTLSetting)).
-				Set("updated_at = ?", time.Now()).
-				Set("status = ?", constants.StatusSuccess).
-				Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-				Exec(ctx)
+			_, dbErr = r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusSuccess, statusItem.Id)
 			if dbErr != nil {
 				zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 				return
@@ -2132,7 +2040,7 @@ func (r *ClickHouseReader) SetTTLTracesV2(ctx context.Context, orgID string, par
 // SetTTL sets the TTL for traces or metrics or logs tables.
 // This is an async API which creates goroutines to set TTL.
 // Status of TTL update is tracked with ttl_status table in sqlite db.
-func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
+func (r *ClickHouseReader) SetTTL(ctx context.Context,
 	params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
 	// Keep only latest 100 transactions/requests
 	r.deleteTtlTransactions(ctx, 100)
@@ -2148,7 +2056,7 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 	switch params.Type {
 	case constants.TraceTTL:
 		if r.useTraceNewSchema {
-			return r.SetTTLTracesV2(ctx, orgID, params)
+			return r.SetTTLTracesV2(ctx, params)
 		}
 
 		tableNames := []string{
@@ -2161,7 +2069,7 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 		}
 		for _, tableName := range tableNames {
 			tableName := getLocalTableName(tableName)
-			statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+			statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 			if err != nil {
 				return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
 			}
@@ -2173,29 +2081,9 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 			tableName := getLocalTableName(tableName)
 			// TODO: DB queries should be implemented with transactional statements but currently clickhouse doesn't support them. Issue: https://github.com/ClickHouse/ClickHouse/issues/22086
 			go func(tableName string) {
-				ttl := types.TTLSetting{
-					Identifiable: types.Identifiable{
-						ID: valuer.GenerateUUID(),
-					},
-					TimeAuditable: types.TimeAuditable{
-						CreatedAt: time.Now(),
-						UpdatedAt: time.Now(),
-					},
-					TransactionID:  uuid,
-					TableName:      tableName,
-					TTL:            int(params.DelDuration),
-					Status:         constants.StatusPending,
-					ColdStorageTTL: coldStorageDuration,
-					OrgID:          orgID,
-				}
-				_, dbErr := r.
-					sqlDB.
-					BunDB().
-					NewInsert().
-					Model(&ttl).
-					Exec(ctx)
+				_, dbErr := r.localDB.Exec("INSERT INTO ttl_status (transaction_id, created_at, updated_at, table_name, ttl, status, cold_storage_ttl) VALUES (?, ?, ?, ?, ?, ?, ?)", uuid, time.Now(), time.Now(), tableName, params.DelDuration, constants.StatusPending, coldStorageDuration)
 				if dbErr != nil {
-					zap.L().Error("error in inserting to ttl_status table", zap.Error(dbErr))
+					zap.L().Error("Error in inserting to ttl_status table", zap.Error(dbErr))
 					return
 				}
 				req := fmt.Sprintf(
@@ -2208,17 +2096,9 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 				err := r.setColdStorage(context.Background(), tableName, params.ColdStorageVolume)
 				if err != nil {
 					zap.L().Error("Error in setting cold storage", zap.Error(err))
-					statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+					statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 					if err == nil {
-						_, dbErr := r.
-							sqlDB.
-							BunDB().
-							NewUpdate().
-							Model(new(types.TTLSetting)).
-							Set("updated_at = ?", time.Now()).
-							Set("status = ?", constants.StatusFailed).
-							Where("id = ?", statusItem.ID.StringValue()).
-							Exec(ctx)
+						_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 						if dbErr != nil {
 							zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 							return
@@ -2228,33 +2108,17 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 				}
 				req += " SETTINGS materialize_ttl_after_modify=0;"
 				zap.L().Error("Executing TTL request: ", zap.String("request", req))
-				statusItem, _ := r.checkTTLStatusItem(ctx, orgID, tableName)
+				statusItem, _ := r.checkTTLStatusItem(ctx, tableName)
 				if err := r.db.Exec(context.Background(), req); err != nil {
 					zap.L().Error("Error in executing set TTL query", zap.Error(err))
-					_, dbErr := r.
-						sqlDB.
-						BunDB().
-						NewUpdate().
-						Model(new(types.TTLSetting)).
-						Set("updated_at = ?", time.Now()).
-						Set("status = ?", constants.StatusFailed).
-						Where("id = ?", statusItem.ID.StringValue()).
-						Exec(ctx)
+					_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 					if dbErr != nil {
 						zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 						return
 					}
 					return
 				}
-				_, dbErr = r.
-					sqlDB.
-					BunDB().
-					NewUpdate().
-					Model(new(types.TTLSetting)).
-					Set("updated_at = ?", time.Now()).
-					Set("status = ?", constants.StatusSuccess).
-					Where("id = ?", statusItem.ID.StringValue()).
-					Exec(ctx)
+				_, dbErr = r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusSuccess, statusItem.Id)
 				if dbErr != nil {
 					zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 					return
@@ -2274,7 +2138,7 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 			signozMetricDBName + "." + signozTSLocalTableNameV41Week,
 		}
 		for _, tableName := range tableNames {
-			statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+			statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 			if err != nil {
 				return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
 			}
@@ -2283,29 +2147,9 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 			}
 		}
 		metricTTL := func(tableName string) {
-			ttl := types.TTLSetting{
-				Identifiable: types.Identifiable{
-					ID: valuer.GenerateUUID(),
-				},
-				TimeAuditable: types.TimeAuditable{
-					CreatedAt: time.Now(),
-					UpdatedAt: time.Now(),
-				},
-				TransactionID:  uuid,
-				TableName:      tableName,
-				TTL:            int(params.DelDuration),
-				Status:         constants.StatusPending,
-				ColdStorageTTL: coldStorageDuration,
-				OrgID:          orgID,
-			}
-			_, dbErr := r.
-				sqlDB.
-				BunDB().
-				NewInsert().
-				Model(&ttl).
-				Exec(ctx)
+			_, dbErr := r.localDB.Exec("INSERT INTO ttl_status (transaction_id, created_at, updated_at, table_name, ttl, status, cold_storage_ttl) VALUES (?, ?, ?, ?, ?, ?, ?)", uuid, time.Now(), time.Now(), tableName, params.DelDuration, constants.StatusPending, coldStorageDuration)
 			if dbErr != nil {
-				zap.L().Error("error in inserting to ttl_status table", zap.Error(dbErr))
+				zap.L().Error("Error in inserting to ttl_status table", zap.Error(dbErr))
 				return
 			}
 			timeColumn := "timestamp_ms"
@@ -2324,17 +2168,9 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 			err := r.setColdStorage(context.Background(), tableName, params.ColdStorageVolume)
 			if err != nil {
 				zap.L().Error("Error in setting cold storage", zap.Error(err))
-				statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+				statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 				if err == nil {
-					_, dbErr := r.
-						sqlDB.
-						BunDB().
-						NewUpdate().
-						Model(new(types.TTLSetting)).
-						Set("updated_at = ?", time.Now()).
-						Set("status = ?", constants.StatusFailed).
-						Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-						Exec(ctx)
+					_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 					if dbErr != nil {
 						zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 						return
@@ -2344,33 +2180,17 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 			}
 			req += " SETTINGS materialize_ttl_after_modify=0"
 			zap.L().Info("Executing TTL request: ", zap.String("request", req))
-			statusItem, _ := r.checkTTLStatusItem(ctx, orgID, tableName)
+			statusItem, _ := r.checkTTLStatusItem(ctx, tableName)
 			if err := r.db.Exec(ctx, req); err != nil {
 				zap.L().Error("error while setting ttl.", zap.Error(err))
-				_, dbErr := r.
-					sqlDB.
-					BunDB().
-					NewUpdate().
-					Model(new(types.TTLSetting)).
-					Set("updated_at = ?", time.Now()).
-					Set("status = ?", constants.StatusFailed).
-					Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-					Exec(ctx)
+				_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 				if dbErr != nil {
 					zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 					return
 				}
 				return
 			}
-			_, dbErr = r.
-				sqlDB.
-				BunDB().
-				NewUpdate().
-				Model(new(types.TTLSetting)).
-				Set("updated_at = ?", time.Now()).
-				Set("status = ?", constants.StatusSuccess).
-				Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-				Exec(ctx)
+			_, dbErr = r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusSuccess, statusItem.Id)
 			if dbErr != nil {
 				zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 				return
@@ -2381,11 +2201,11 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 		}
 	case constants.LogsTTL:
 		if r.useLogsNewSchema {
-			return r.SetTTLLogsV2(ctx, orgID, params)
+			return r.SetTTLLogsV2(ctx, params)
 		}
 
 		tableName := r.logsDB + "." + r.logsLocalTable
-		statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+		statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 		if err != nil {
 			return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
 		}
@@ -2393,27 +2213,7 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 			return nil, &model.ApiError{Typ: model.ErrorConflict, Err: fmt.Errorf("TTL is already running")}
 		}
 		go func(tableName string) {
-			ttl := types.TTLSetting{
-				Identifiable: types.Identifiable{
-					ID: valuer.GenerateUUID(),
-				},
-				TimeAuditable: types.TimeAuditable{
-					CreatedAt: time.Now(),
-					UpdatedAt: time.Now(),
-				},
-				TransactionID:  uuid,
-				TableName:      tableName,
-				TTL:            int(params.DelDuration),
-				Status:         constants.StatusPending,
-				ColdStorageTTL: coldStorageDuration,
-				OrgID:          orgID,
-			}
-			_, dbErr := r.
-				sqlDB.
-				BunDB().
-				NewInsert().
-				Model(&ttl).
-				Exec(ctx)
+			_, dbErr := r.localDB.Exec("INSERT INTO ttl_status (transaction_id, created_at, updated_at, table_name, ttl, status, cold_storage_ttl) VALUES (?, ?, ?, ?, ?, ?, ?)", uuid, time.Now(), time.Now(), tableName, params.DelDuration, constants.StatusPending, coldStorageDuration)
 			if dbErr != nil {
 				zap.L().Error("error in inserting to ttl_status table", zap.Error(dbErr))
 				return
@@ -2429,17 +2229,9 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 			err := r.setColdStorage(context.Background(), tableName, params.ColdStorageVolume)
 			if err != nil {
 				zap.L().Error("error in setting cold storage", zap.Error(err))
-				statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
+				statusItem, err := r.checkTTLStatusItem(ctx, tableName)
 				if err == nil {
-					_, dbErr := r.
-						sqlDB.
-						BunDB().
-						NewUpdate().
-						Model(new(types.TTLSetting)).
-						Set("updated_at = ?", time.Now()).
-						Set("status = ?", constants.StatusFailed).
-						Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-						Exec(ctx)
+					_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 					if dbErr != nil {
 						zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 						return
@@ -2449,33 +2241,17 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 			}
 			req += " SETTINGS materialize_ttl_after_modify=0"
 			zap.L().Info("Executing TTL request: ", zap.String("request", req))
-			statusItem, _ := r.checkTTLStatusItem(ctx, orgID, tableName)
+			statusItem, _ := r.checkTTLStatusItem(ctx, tableName)
 			if err := r.db.Exec(ctx, req); err != nil {
 				zap.L().Error("error while setting ttl", zap.Error(err))
-				_, dbErr := r.
-					sqlDB.
-					BunDB().
-					NewUpdate().
-					Model(new(types.TTLSetting)).
-					Set("updated_at = ?", time.Now()).
-					Set("status = ?", constants.StatusFailed).
-					Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-					Exec(ctx)
+				_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
 				if dbErr != nil {
 					zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 					return
 				}
 				return
 			}
-			_, dbErr = r.
-				sqlDB.
-				BunDB().
-				NewUpdate().
-				Model(new(types.TTLSetting)).
-				Set("updated_at = ?", time.Now()).
-				Set("status = ?", constants.StatusSuccess).
-				Where("id = ?", statusItem.Identifiable.ID.StringValue()).
-				Exec(ctx)
+			_, dbErr = r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusSuccess, statusItem.Id)
 			if dbErr != nil {
 				zap.L().Error("Error in processing ttl_status update sql query", zap.Error(dbErr))
 				return
@@ -2490,62 +2266,47 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string,
 	return &model.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
 }
 
-func (r *ClickHouseReader) deleteTtlTransactions(ctx context.Context, numberOfTransactionsStore int) {
-	limitTransactions := []string{}
-	err := r.
-		sqlDB.
-		BunDB().
-		NewSelect().
-		ColumnExpr("distinct(transaction_id)").
-		Model(new(types.TTLSetting)).
-		OrderExpr("created_at DESC").
-		Limit(numberOfTransactionsStore).
-		Scan(ctx, &limitTransactions)
-
-	if err != nil {
-		zap.L().Error("Error in processing ttl_status delete sql query", zap.Error(err))
-	}
-
-	_, err = r.
-		sqlDB.
-		BunDB().
-		NewDelete().
-		Model(new(types.TTLSetting)).
-		Where("transaction_id NOT IN (?)", bun.In(limitTransactions)).
-		Exec(ctx)
+func (r *ClickHouseReader) deleteTtlTransactions(_ context.Context, numberOfTransactionsStore int) {
+	_, err := r.localDB.Exec("DELETE FROM ttl_status WHERE transaction_id NOT IN (SELECT distinct transaction_id FROM ttl_status ORDER BY created_at DESC LIMIT ?)", numberOfTransactionsStore)
 	if err != nil {
 		zap.L().Error("Error in processing ttl_status delete sql query", zap.Error(err))
 	}
 }
 
 // checkTTLStatusItem checks if ttl_status table has an entry for the given table name
-func (r *ClickHouseReader) checkTTLStatusItem(ctx context.Context, orgID string, tableName string) (*types.TTLSetting, *model.ApiError) {
-	zap.L().Info("checkTTLStatusItem query", zap.String("tableName", tableName))
-	ttl := new(types.TTLSetting)
-	err := r.
-		sqlDB.
-		BunDB().
-		NewSelect().
-		Model(ttl).
-		Where("table_name = ?", tableName).
-		Where("org_id = ?", orgID).
-		OrderExpr("created_at DESC").
-		Limit(1).
-		Scan(ctx)
-	if err != nil && err != sql.ErrNoRows {
-		zap.L().Error("Error in processing sql query", zap.Error(err))
-		return ttl, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
+func (r *ClickHouseReader) checkTTLStatusItem(_ context.Context, tableName string) (model.TTLStatusItem, *model.ApiError) {
+	statusItem := []model.TTLStatusItem{}
+
+	query := `SELECT id, status, ttl, cold_storage_ttl FROM ttl_status WHERE table_name = ? ORDER BY created_at DESC`
+
+	zap.L().Info("checkTTLStatusItem query", zap.String("query", query), zap.String("tableName", tableName))
+
+	stmt, err := r.localDB.Preparex(query)
+
+	if err != nil {
+		zap.L().Error("Error preparing query for checkTTLStatusItem", zap.Error(err))
+		return model.TTLStatusItem{}, &model.ApiError{Typ: model.ErrorInternal, Err: err}
 	}
-	return ttl, nil
+
+	err = stmt.Select(&statusItem, tableName)
+
+	if len(statusItem) == 0 {
+		return model.TTLStatusItem{}, nil
+	}
+	if err != nil {
+		zap.L().Error("Error in processing sql query", zap.Error(err))
+		return model.TTLStatusItem{}, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
+	}
+	return statusItem[0], nil
 }
 
 // setTTLQueryStatus fetches ttl_status table status from DB
-func (r *ClickHouseReader) setTTLQueryStatus(ctx context.Context, orgID string, tableNameArray []string) (string, *model.ApiError) {
+func (r *ClickHouseReader) setTTLQueryStatus(ctx context.Context, tableNameArray []string) (string, *model.ApiError) {
 	failFlag := false
 	status := constants.StatusSuccess
 	for _, tableName := range tableNameArray {
-		statusItem, err := r.checkTTLStatusItem(ctx, orgID, tableName)
-		emptyStatusStruct := new(types.TTLSetting)
+		statusItem, err := r.checkTTLStatusItem(ctx, tableName)
+		emptyStatusStruct := model.TTLStatusItem{}
 		if statusItem == emptyStatusStruct {
 			return "", nil
 		}
@@ -2606,7 +2367,7 @@ func getLocalTableNameArray(tableNames []string) []string {
 }
 
 // GetTTL returns current ttl, expected ttl and past setTTL status for metrics/traces.
-func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *model.GetTTLParams) (*model.GetTTLResponseItem, *model.ApiError) {
+func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLParams) (*model.GetTTLResponseItem, *model.ApiError) {
 
 	parseTTL := func(queryResp string) (int, int) {
 
@@ -2696,7 +2457,7 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
 		tableNameArray := []string{signozTraceDBName + "." + signozTraceTableName, signozTraceDBName + "." + signozDurationMVTable, signozTraceDBName + "." + signozSpansTable, signozTraceDBName + "." + signozErrorIndexTable, signozTraceDBName + "." + signozUsageExplorerTable, signozTraceDBName + "." + defaultDependencyGraphTable}
 
 		tableNameArray = getLocalTableNameArray(tableNameArray)
-		status, err := r.setTTLQueryStatus(ctx, orgID, tableNameArray)
+		status, err := r.setTTLQueryStatus(ctx, tableNameArray)
 		if err != nil {
 			return nil, err
 		}
@@ -2704,22 +2465,22 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
 		if err != nil {
 			return nil, err
 		}
-		ttlQuery, err := r.checkTTLStatusItem(ctx, orgID, tableNameArray[0])
+		ttlQuery, err := r.checkTTLStatusItem(ctx, tableNameArray[0])
 		if err != nil {
 			return nil, err
 		}
 		ttlQuery.TTL = ttlQuery.TTL / 3600 // convert to hours
-		if ttlQuery.ColdStorageTTL != -1 {
-			ttlQuery.ColdStorageTTL = ttlQuery.ColdStorageTTL / 3600 // convert to hours
+		if ttlQuery.ColdStorageTtl != -1 {
+			ttlQuery.ColdStorageTtl = ttlQuery.ColdStorageTtl / 3600 // convert to hours
 		}
 
 		delTTL, moveTTL := parseTTL(dbResp.EngineFull)
-		return &model.GetTTLResponseItem{TracesTime: delTTL, TracesMoveTime: moveTTL, ExpectedTracesTime: ttlQuery.TTL, ExpectedTracesMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
+		return &model.GetTTLResponseItem{TracesTime: delTTL, TracesMoveTime: moveTTL, ExpectedTracesTime: ttlQuery.TTL, ExpectedTracesMoveTime: ttlQuery.ColdStorageTtl, Status: status}, nil
 
 	case constants.MetricsTTL:
 		tableNameArray := []string{signozMetricDBName + "." + signozSampleTableName}
 		tableNameArray = getLocalTableNameArray(tableNameArray)
-		status, err := r.setTTLQueryStatus(ctx, orgID, tableNameArray)
+		status, err := r.setTTLQueryStatus(ctx, tableNameArray)
 		if err != nil {
 			return nil, err
 		}
@@ -2727,22 +2488,22 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
 		if err != nil {
 			return nil, err
 		}
-		ttlQuery, err := r.checkTTLStatusItem(ctx, orgID, tableNameArray[0])
+		ttlQuery, err := r.checkTTLStatusItem(ctx, tableNameArray[0])
 		if err != nil {
 			return nil, err
 		}
 		ttlQuery.TTL = ttlQuery.TTL / 3600 // convert to hours
-		if ttlQuery.ColdStorageTTL != -1 {
-			ttlQuery.ColdStorageTTL = ttlQuery.ColdStorageTTL / 3600 // convert to hours
+		if ttlQuery.ColdStorageTtl != -1 {
+			ttlQuery.ColdStorageTtl = ttlQuery.ColdStorageTtl / 3600 // convert to hours
 		}
 
 		delTTL, moveTTL := parseTTL(dbResp.EngineFull)
-		return &model.GetTTLResponseItem{MetricsTime: delTTL, MetricsMoveTime: moveTTL, ExpectedMetricsTime: ttlQuery.TTL, ExpectedMetricsMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
+		return &model.GetTTLResponseItem{MetricsTime: delTTL, MetricsMoveTime: moveTTL, ExpectedMetricsTime: ttlQuery.TTL, ExpectedMetricsMoveTime: ttlQuery.ColdStorageTtl, Status: status}, nil
 
 	case constants.LogsTTL:
 		tableNameArray := []string{r.logsDB + "." + r.logsTable}
 		tableNameArray = getLocalTableNameArray(tableNameArray)
-		status, err := r.setTTLQueryStatus(ctx, orgID, tableNameArray)
+		status, err := r.setTTLQueryStatus(ctx, tableNameArray)
 		if err != nil {
 			return nil, err
 		}
@@ -2750,17 +2511,17 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
 		if err != nil {
 			return nil, err
 		}
-		ttlQuery, err := r.checkTTLStatusItem(ctx, tableNameArray[0], orgID)
+		ttlQuery, err := r.checkTTLStatusItem(ctx, tableNameArray[0])
 		if err != nil {
 			return nil, err
 		}
 		ttlQuery.TTL = ttlQuery.TTL / 3600 // convert to hours
-		if ttlQuery.ColdStorageTTL != -1 {
-			ttlQuery.ColdStorageTTL = ttlQuery.ColdStorageTTL / 3600 // convert to hours
+		if ttlQuery.ColdStorageTtl != -1 {
+			ttlQuery.ColdStorageTtl = ttlQuery.ColdStorageTtl / 3600 // convert to hours
 		}
 
 		delTTL, moveTTL := parseTTL(dbResp.EngineFull)
-		return &model.GetTTLResponseItem{LogsTime: delTTL, LogsMoveTime: moveTTL, ExpectedLogsTime: ttlQuery.TTL, ExpectedLogsMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
+		return &model.GetTTLResponseItem{LogsTime: delTTL, LogsMoveTime: moveTTL, ExpectedLogsTime: ttlQuery.TTL, ExpectedLogsMoveTime: ttlQuery.ColdStorageTtl, Status: status}, nil
 
 	default:
 		return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error while getting ttl. ttl type should be metrics|traces, got %v",

pkg/query-service/app/http_handler.go

@@ -21,7 +21,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/alertmanager"
 	errorsV2 "github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/http/render"
-	"github.com/SigNoz/signoz/pkg/modules/preference"
 	"github.com/SigNoz/signoz/pkg/query-service/app/metricsexplorer"
 	"github.com/SigNoz/signoz/pkg/signoz"
 	"github.com/SigNoz/signoz/pkg/valuer"
@@ -45,6 +44,7 @@ import (
 	logsv4 "github.com/SigNoz/signoz/pkg/query-service/app/logs/v4"
 	"github.com/SigNoz/signoz/pkg/query-service/app/metrics"
 	metricsv3 "github.com/SigNoz/signoz/pkg/query-service/app/metrics/v3"
+	"github.com/SigNoz/signoz/pkg/query-service/app/preferences"
 	"github.com/SigNoz/signoz/pkg/query-service/app/querier"
 	querierV2 "github.com/SigNoz/signoz/pkg/query-service/app/querier/v2"
 	"github.com/SigNoz/signoz/pkg/query-service/app/queryBuilder"
@@ -142,8 +142,6 @@ type APIHandler struct {
 	AlertmanagerAPI *alertmanager.API
 
 	Signoz *signoz.SigNoz
-
-	Preference preference.API
 }
 
 type APIHandlerOpts struct {
@@ -189,8 +187,6 @@ type APIHandlerOpts struct {
 	AlertmanagerAPI *alertmanager.API
 
 	Signoz *signoz.SigNoz
-
-	Preference preference.API
 }
 
 // NewAPIHandler returns an APIHandler
@@ -261,7 +257,6 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) {
 		SummaryService:                summaryService,
 		AlertmanagerAPI:               opts.AlertmanagerAPI,
 		Signoz:                        opts.Signoz,
-		Preference:                    opts.Preference,
 	}
 
 	logsQueryBuilder := logsv3.PrepareLogsQuery
@@ -1863,15 +1858,8 @@ func (aH *APIHandler) setTTL(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	ctx := r.Context()
-	claims, ok := authtypes.ClaimsFromContext(ctx)
-	if !ok {
-		RespondError(w, &model.ApiError{Err: errors.New("failed to get org id from context"), Typ: model.ErrorInternal}, nil)
-		return
-	}
-
 	// Context is not used here as TTL is long duration DB operation
-	result, apiErr := aH.reader.SetTTL(context.Background(), claims.OrgID, ttlParams)
+	result, apiErr := aH.reader.SetTTL(context.Background(), ttlParams)
 	if apiErr != nil {
 		if apiErr.Typ == model.ErrorConflict {
 			aH.HandleError(w, apiErr.Err, http.StatusConflict)
@@ -1891,14 +1879,7 @@ func (aH *APIHandler) getTTL(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	ctx := r.Context()
-	claims, ok := authtypes.ClaimsFromContext(ctx)
-	if !ok {
-		RespondError(w, &model.ApiError{Err: errors.New("failed to get org id from context"), Typ: model.ErrorInternal}, nil)
-		return
-	}
-
-	result, apiErr := aH.reader.GetTTL(r.Context(), claims.OrgID, ttlParams)
+	result, apiErr := aH.reader.GetTTL(r.Context(), ttlParams)
 	if apiErr != nil && aH.HandleError(w, apiErr.Err, http.StatusInternalServerError) {
 		return
 	}
@@ -3427,37 +3408,132 @@ func (aH *APIHandler) getProducerConsumerEval(
 func (aH *APIHandler) getUserPreference(
 	w http.ResponseWriter, r *http.Request,
 ) {
-	aH.Preference.GetUserPreference(w, r)
+	preferenceId := mux.Vars(r)["preferenceId"]
+	claims, ok := authtypes.ClaimsFromContext(r.Context())
+	if !ok {
+		render.Error(w, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
+		return
+	}
+
+	preference, apiErr := preferences.GetUserPreference(
+		r.Context(), preferenceId, claims.OrgID, claims.UserID,
+	)
+	if apiErr != nil {
+		RespondError(w, apiErr, nil)
+		return
+	}
+
+	aH.Respond(w, preference)
 }
 
 func (aH *APIHandler) updateUserPreference(
 	w http.ResponseWriter, r *http.Request,
 ) {
-	aH.Preference.UpdateUserPreference(w, r)
+	preferenceId := mux.Vars(r)["preferenceId"]
+	claims, ok := authtypes.ClaimsFromContext(r.Context())
+	if !ok {
+		render.Error(w, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
+		return
+	}
+	req := preferences.UpdatePreference{}
+
+	err := json.NewDecoder(r.Body).Decode(&req)
+
+	if err != nil {
+		RespondError(w, model.BadRequest(err), nil)
+		return
+	}
+	preference, apiErr := preferences.UpdateUserPreference(r.Context(), preferenceId, req.PreferenceValue, claims.UserID)
+	if apiErr != nil {
+		RespondError(w, apiErr, nil)
+		return
+	}
+
+	aH.Respond(w, preference)
 }
 
 func (aH *APIHandler) getAllUserPreferences(
 	w http.ResponseWriter, r *http.Request,
 ) {
-	aH.Preference.GetAllUserPreferences(w, r)
+	claims, ok := authtypes.ClaimsFromContext(r.Context())
+	if !ok {
+		render.Error(w, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
+		return
+	}
+	preference, apiErr := preferences.GetAllUserPreferences(
+		r.Context(), claims.OrgID, claims.UserID,
+	)
+	if apiErr != nil {
+		RespondError(w, apiErr, nil)
+		return
+	}
+
+	aH.Respond(w, preference)
 }
 
 func (aH *APIHandler) getOrgPreference(
 	w http.ResponseWriter, r *http.Request,
 ) {
-	aH.Preference.GetOrgPreference(w, r)
+	preferenceId := mux.Vars(r)["preferenceId"]
+	claims, ok := authtypes.ClaimsFromContext(r.Context())
+	if !ok {
+		render.Error(w, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
+		return
+	}
+	preference, apiErr := preferences.GetOrgPreference(
+		r.Context(), preferenceId, claims.OrgID,
+	)
+	if apiErr != nil {
+		RespondError(w, apiErr, nil)
+		return
+	}
+
+	aH.Respond(w, preference)
 }
 
 func (aH *APIHandler) updateOrgPreference(
 	w http.ResponseWriter, r *http.Request,
 ) {
-	aH.Preference.UpdateOrgPreference(w, r)
+	preferenceId := mux.Vars(r)["preferenceId"]
+	req := preferences.UpdatePreference{}
+	claims, ok := authtypes.ClaimsFromContext(r.Context())
+	if !ok {
+		render.Error(w, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
+		return
+	}
+
+	err := json.NewDecoder(r.Body).Decode(&req)
+
+	if err != nil {
+		RespondError(w, model.BadRequest(err), nil)
+		return
+	}
+	preference, apiErr := preferences.UpdateOrgPreference(r.Context(), preferenceId, req.PreferenceValue, claims.OrgID)
+	if apiErr != nil {
+		RespondError(w, apiErr, nil)
+		return
+	}
+
+	aH.Respond(w, preference)
 }
 
 func (aH *APIHandler) getAllOrgPreferences(
 	w http.ResponseWriter, r *http.Request,
 ) {
-	aH.Preference.GetAllOrgPreferences(w, r)
+	claims, ok := authtypes.ClaimsFromContext(r.Context())
+	if !ok {
+		render.Error(w, errorsV2.Newf(errorsV2.TypeUnauthenticated, errorsV2.CodeUnauthenticated, "unauthenticated"))
+		return
+	}
+	preference, apiErr := preferences.GetAllOrgPreferences(
+		r.Context(), claims.OrgID,
+	)
+	if apiErr != nil {
+		RespondError(w, apiErr, nil)
+		return
+	}
+
+	aH.Respond(w, preference)
 }
 
 // RegisterIntegrationRoutes Registers all Integrations
@@ -3736,14 +3812,9 @@ func (aH *APIHandler) InstallIntegration(
 		RespondError(w, model.BadRequest(err), nil)
 		return
 	}
-	claims, ok := authtypes.ClaimsFromContext(r.Context())
-	if !ok {
-		RespondError(w, model.UnauthorizedError(errors.New("unauthorized")), nil)
-		return
-	}
 
 	integration, apiErr := aH.IntegrationsController.Install(
-		r.Context(), claims.OrgID, &req,
+		r.Context(), &req,
 	)
 	if apiErr != nil {
 		RespondError(w, apiErr, nil)
@@ -3763,13 +3834,8 @@ func (aH *APIHandler) UninstallIntegration(
 		RespondError(w, model.BadRequest(err), nil)
 		return
 	}
-	claims, ok := authtypes.ClaimsFromContext(r.Context())
-	if !ok {
-		RespondError(w, model.UnauthorizedError(errors.New("unauthorized")), nil)
-		return
-	}
 
-	apiErr := aH.IntegrationsController.Uninstall(r.Context(), claims.OrgID, &req)
+	apiErr := aH.IntegrationsController.Uninstall(r.Context(), &req)
 	if apiErr != nil {
 		RespondError(w, apiErr, nil)
 		return
@@ -4428,7 +4494,7 @@ func (aH *APIHandler) listLogsPipelines(ctx context.Context, orgID string) (
 ) {
 	// get lateset agent config
 	latestVersion := -1
-	lastestConfig, err := agentConf.GetLatestVersion(ctx, orgID, logPipelines)
+	lastestConfig, err := agentConf.GetLatestVersion(ctx, logPipelines)
 	if err != nil && err.Type() != model.ErrorNotFound {
 		return nil, model.WrapApiError(err, "failed to get latest agent config version")
 	}
@@ -4437,14 +4503,14 @@ func (aH *APIHandler) listLogsPipelines(ctx context.Context, orgID string) (
 		latestVersion = lastestConfig.Version
 	}
 
-	payload, err := aH.LogsParsingPipelineController.GetPipelinesByVersion(ctx, orgID, latestVersion)
+	payload, err := aH.LogsParsingPipelineController.GetPipelinesByVersion(ctx, latestVersion)
 	if err != nil {
 		return nil, model.WrapApiError(err, "failed to get pipelines")
 	}
 
 	// todo(Nitya): make a new API for history pagination
 	limit := 10
-	history, err := agentConf.GetConfigHistory(ctx, orgID, logPipelines, limit)
+	history, err := agentConf.GetConfigHistory(ctx, logPipelines, limit)
 	if err != nil {
 		return nil, model.WrapApiError(err, "failed to get config history")
 	}
@@ -4456,14 +4522,14 @@ func (aH *APIHandler) listLogsPipelines(ctx context.Context, orgID string) (
 func (aH *APIHandler) listLogsPipelinesByVersion(ctx context.Context, orgID string, version int) (
 	*logparsingpipeline.PipelinesResponse, *model.ApiError,
 ) {
-	payload, err := aH.LogsParsingPipelineController.GetPipelinesByVersion(ctx, orgID, version)
+	payload, err := aH.LogsParsingPipelineController.GetPipelinesByVersion(ctx, version)
 	if err != nil {
 		return nil, model.WrapApiError(err, "failed to get pipelines by version")
 	}
 
 	// todo(Nitya): make a new API for history pagination
 	limit := 10
-	history, err := agentConf.GetConfigHistory(ctx, orgID, logPipelines, limit)
+	history, err := agentConf.GetConfigHistory(ctx, logPipelines, limit)
 	if err != nil {
 		return nil, model.WrapApiError(err, "failed to retrieve agent config history")
 	}

pkg/query-service/app/integrations/controller.go

@@ -87,7 +87,7 @@ type InstallIntegrationRequest struct {
 }
 
 func (c *Controller) Install(
-	ctx context.Context, orgId string, req *InstallIntegrationRequest,
+	ctx context.Context, req *InstallIntegrationRequest,
 ) (*IntegrationsListItem, *model.ApiError) {
 	res, apiErr := c.mgr.InstallIntegration(
 		ctx, req.IntegrationId, req.Config,
@@ -104,7 +104,7 @@ type UninstallIntegrationRequest struct {
 }
 
 func (c *Controller) Uninstall(
-	ctx context.Context, orgId string, req *UninstallIntegrationRequest,
+	ctx context.Context, req *UninstallIntegrationRequest,
 ) *model.ApiError {
 	if len(req.IntegrationId) < 1 {
 		return model.BadRequest(fmt.Errorf(

pkg/query-service/app/logparsingpipeline/controller.go

@@ -12,7 +12,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/query-service/model"
 	"github.com/SigNoz/signoz/pkg/query-service/utils"
 	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/types"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
 	"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
 	"github.com/google/uuid"
@@ -40,10 +39,10 @@ func NewLogParsingPipelinesController(
 
 // PipelinesResponse is used to prepare http response for pipelines config related requests
 type PipelinesResponse struct {
-	*types.AgentConfigVersion
+	*agentConf.ConfigVersion
 
 	Pipelines []pipelinetypes.GettablePipeline `json:"pipelines"`
-	History   []types.AgentConfigVersion       `json:"history"`
+	History   []agentConf.ConfigVersion        `json:"history"`
 }
 
 // ApplyPipelines stores new or changed pipelines and initiates a new config update
@@ -87,12 +86,12 @@ func (ic *LogParsingPipelineController) ApplyPipelines(
 	}
 
 	// prepare config by calling gen func
-	cfg, err := agentConf.StartNewVersion(ctx, claims.OrgID, claims.UserID, types.ElementTypeLogPipelines, elements)
+	cfg, err := agentConf.StartNewVersion(ctx, claims.UserID, agentConf.ElementTypeLogPipelines, elements)
 	if err != nil || cfg == nil {
 		return nil, err
 	}
 
-	return ic.GetPipelinesByVersion(ctx, claims.OrgID, cfg.Version)
+	return ic.GetPipelinesByVersion(ctx, cfg.Version)
 }
 
 func (ic *LogParsingPipelineController) ValidatePipelines(
@@ -139,12 +138,21 @@ func (ic *LogParsingPipelineController) ValidatePipelines(
 // Returns effective list of pipelines including user created
 // pipelines and pipelines for installed integrations
 func (ic *LogParsingPipelineController) getEffectivePipelinesByVersion(
-	ctx context.Context, orgID string, version int,
+	ctx context.Context, version int,
 ) ([]pipelinetypes.GettablePipeline, *model.ApiError) {
 	result := []pipelinetypes.GettablePipeline{}
 
+	// todo(nitya): remove this once we fix agents in multitenancy
+	defaultOrgID, err := ic.GetDefaultOrgID(ctx)
+	if err != nil {
+		// we don't want to fail the request if we can't get the default org ID
+		// we will just return an empty list of pipelines
+		zap.L().Warn("failed to get default org ID", zap.Error(err))
+		return result, nil
+	}
+
 	if version >= 0 {
-		savedPipelines, errors := ic.getPipelinesByVersion(ctx, orgID, version)
+		savedPipelines, errors := ic.getPipelinesByVersion(ctx, defaultOrgID, version)
 		if errors != nil {
 			zap.L().Error("failed to get pipelines for version", zap.Int("version", version), zap.Errors("errors", errors))
 			return nil, model.InternalError(fmt.Errorf("failed to get pipelines for given version %v", errors))
@@ -196,18 +204,18 @@ func (ic *LogParsingPipelineController) getEffectivePipelinesByVersion(
 
 // GetPipelinesByVersion responds with version info and associated pipelines
 func (ic *LogParsingPipelineController) GetPipelinesByVersion(
-	ctx context.Context, orgId string, version int,
+	ctx context.Context, version int,
 ) (*PipelinesResponse, *model.ApiError) {
 
-	pipelines, errors := ic.getEffectivePipelinesByVersion(ctx, orgId, version)
+	pipelines, errors := ic.getEffectivePipelinesByVersion(ctx, version)
 	if errors != nil {
 		zap.L().Error("failed to get pipelines for version", zap.Int("version", version), zap.Error(errors))
 		return nil, model.InternalError(fmt.Errorf("failed to get pipelines for given version %v", errors))
 	}
 
-	var configVersion *types.AgentConfigVersion
+	var configVersion *agentConf.ConfigVersion
 	if version >= 0 {
-		cv, err := agentConf.GetConfigVersion(ctx, orgId, types.ElementTypeLogPipelines, version)
+		cv, err := agentConf.GetConfigVersion(ctx, agentConf.ElementTypeLogPipelines, version)
 		if err != nil {
 			zap.L().Error("failed to get config for version", zap.Int("version", version), zap.Error(err))
 			return nil, model.WrapApiError(err, "failed to get config for given version")
@@ -216,8 +224,8 @@ func (ic *LogParsingPipelineController) GetPipelinesByVersion(
 	}
 
 	return &PipelinesResponse{
-		AgentConfigVersion: configVersion,
-		Pipelines:          pipelines,
+		ConfigVersion: configVersion,
+		Pipelines:     pipelines,
 	}, nil
 }
 
@@ -256,9 +264,8 @@ func (pc *LogParsingPipelineController) AgentFeatureType() agentConf.AgentFeatur
 
 // Implements agentConf.AgentFeature interface.
 func (pc *LogParsingPipelineController) RecommendAgentConfig(
-	orgId string,
 	currentConfYaml []byte,
-	configVersion *types.AgentConfigVersion,
+	configVersion *agentConf.ConfigVersion,
 ) (
 	recommendedConfYaml []byte,
 	serializedSettingsUsed string,
@@ -270,7 +277,7 @@ func (pc *LogParsingPipelineController) RecommendAgentConfig(
 	}
 
 	pipelinesResp, apiErr := pc.GetPipelinesByVersion(
-		context.Background(), orgId, pipelinesVersion,
+		context.Background(), pipelinesVersion,
 	)
 	if apiErr != nil {
 		return nil, "", apiErr

pkg/query-service/app/logparsingpipeline/db.go

@@ -129,6 +129,20 @@ func (r *Repo) getPipelinesByVersion(
 	return gettablePipelines, errors
 }
 
+func (r *Repo) GetDefaultOrgID(ctx context.Context) (string, *model.ApiError) {
+	var orgs []types.Organization
+	err := r.sqlStore.BunDB().NewSelect().
+		Model(&orgs).
+		Scan(ctx)
+	if err != nil {
+		return "", model.InternalError(errors.Wrap(err, "failed to get default org ID"))
+	}
+	if len(orgs) == 0 {
+		return "", model.InternalError(errors.New("no orgs found"))
+	}
+	return orgs[0].ID, nil
+}
+
 // GetPipelines returns pipeline and errors (if any)
 func (r *Repo) GetPipeline(
 	ctx context.Context, orgID string, id string,

pkg/query-service/app/metrics/v4/cumulative/table_test.go

@@ -51,7 +51,7 @@ func TestPrepareTableQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = sum, temporality = cumulative",
@@ -93,7 +93,7 @@ func TestPrepareTableQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test time aggregation = avg, space aggregation = avg, temporality = unspecified, testing metrics and attribute name with dot",
@@ -148,7 +148,7 @@ func TestPrepareTableQuery(t *testing.T) {
 			},
 			start:                 1735295140000,
 			end:                   1735554340000,
-			expectedQueryContains: "SELECT state, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system.memory.usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1735257600000 AND unix_milli < 1735554340000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name IN ['system.memory.usage'] AND unix_milli >= 1735295140000 AND unix_milli < 1735554340000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, ts ORDER BY state desc, ts ASC",
+			expectedQueryContains: "SELECT state, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system.memory.usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1735257600000 AND unix_milli < 1735554340000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name IN ['system.memory.usage'] AND unix_milli >= 1735295140000 AND unix_milli < 1735554340000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, ts ORDER BY state desc, ts ASC",
 		},
 	}
 

pkg/query-service/app/metrics/v4/cumulative/timeseries.go

@@ -116,7 +116,7 @@ func prepareTimeAggregationSubQuery(start, end, step int64, mq *v3.BuilderQuery)
 		return "", err
 	}
 
-	samplesTableFilter := fmt.Sprintf("metric_name IN %s AND unix_milli >= %d AND unix_milli < %d", utils.ClickHouseFormattedMetricNames(mq.AggregateAttribute.Key), start, end)
+	samplesTableFilter := fmt.Sprintf("metric_name IN %s AND unix_milli >= %d AND unix_milli < %d AND bitAnd(flags, 1) = 0", utils.ClickHouseFormattedMetricNames(mq.AggregateAttribute.Key), start, end)
 
 	tableName := helpers.WhichSamplesTableToUse(start, end, mq)
 

pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go

@@ -66,7 +66,7 @@ func TestPrepareTimeAggregationSubQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'service_name') != 'payment_service' AND JSONExtractString(labels, 'endpoint') IN ['/paycallback','/payme','/paypal']) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts",
+			expectedQueryContains: "SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'service_name') != 'payment_service' AND JSONExtractString(labels, 'endpoint') IN ['/paycallback','/payme','/paypal']) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts",
 		},
 		{
 			name: "test time aggregation = rate, temporality = cumulative",
@@ -107,7 +107,7 @@ func TestPrepareTimeAggregationSubQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)",
+			expectedQueryContains: "SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)",
 		},
 	}
 
@@ -168,7 +168,7 @@ func TestPrepareTimeseriesQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = sum, temporality = cumulative",
@@ -210,7 +210,7 @@ func TestPrepareTimeseriesQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test time aggregation = avg, space aggregation = avg, temporality = unspecified, testing metrics and attribute name with dot",
@@ -265,7 +265,7 @@ func TestPrepareTimeseriesQuery(t *testing.T) {
 			},
 			start:                 1735295140000,
 			end:                   1735554340000,
-			expectedQueryContains: "SELECT state, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system.memory.usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1735257600000 AND unix_milli < 1735554340000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name IN ['system.memory.usage'] AND unix_milli >= 1735295140000 AND unix_milli < 1735554340000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, ts ORDER BY state desc, ts ASC",
+			expectedQueryContains: "SELECT state, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system.memory.usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1735257600000 AND unix_milli < 1735554340000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name IN ['system.memory.usage'] AND unix_milli >= 1735295140000 AND unix_milli < 1735554340000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, ts ORDER BY state desc, ts ASC",
 		},
 	}
 

pkg/query-service/app/metrics/v4/delta/table_test.go

@@ -53,7 +53,7 @@ func TestPrepareTableQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = sum, temporality = delta",
@@ -95,7 +95,7 @@ func TestPrepareTableQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = avg, temporality = delta, testing metrics and attribute name with dot",
@@ -143,7 +143,7 @@ func TestPrepareTableQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT ts, avg(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['signoz.latency.sum'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'host.name') = '4f6ec470feea') as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz.latency.sum'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, avg(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['signoz.latency.sum'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'host.name') = '4f6ec470feea') as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz.latency.sum'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 	}
 

pkg/query-service/app/metrics/v4/delta/time_series_test.go

@@ -66,7 +66,7 @@ func TestPrepareTimeAggregationSubQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'service_name') != 'payment_service' AND JSONExtractString(labels, 'endpoint') IN ['/paycallback','/payme','/paypal']) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts",
+			expectedQueryContains: "SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'service_name') != 'payment_service' AND JSONExtractString(labels, 'endpoint') IN ['/paycallback','/payme','/paypal']) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts",
 		},
 		{
 			name: "test time aggregation = rate, temporality = delta",
@@ -107,7 +107,7 @@ func TestPrepareTimeAggregationSubQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts",
+			expectedQueryContains: "SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts",
 		},
 	}
 
@@ -168,7 +168,7 @@ func TestPrepareTimeseriesQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = sum, temporality = delta",
@@ -210,7 +210,7 @@ func TestPrepareTimeseriesQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['http_requests'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000 AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['http_requests'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation percentile99, type = ExponentialHistogram",
@@ -244,7 +244,7 @@ func TestPrepareTimeseriesQuery(t *testing.T) {
 			},
 			start:                 1701794980000,
 			end:                   1701796780000,
-			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, quantilesDDMerge(0.01, 0.990000)(sketch)[1] as value FROM signoz_metrics.distributed_exp_hist INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['signoz_latency'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, quantilesDDMerge(0.01, 0.990000)(sketch)[1] as value FROM signoz_metrics.distributed_exp_hist INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['signoz_latency'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1701792000000 AND unix_milli < 1701796780000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency'] AND unix_milli >= 1701794980000 AND unix_milli < 1701796780000 AND bitAnd(flags, 1) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = max, temporality = delta, testing metrics and attribute name with dot",
@@ -299,7 +299,7 @@ func TestPrepareTimeseriesQuery(t *testing.T) {
 			},
 			start:                 1735036101000,
 			end:                   1735637901000,
-			expectedQueryContains: "SELECT `host.name`, ts, max(per_series_value) as value FROM (SELECT fingerprint, any(`host.name`) as `host.name`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'host.name') as `host.name`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz.latency.sum'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1734998400000 AND unix_milli < 1735637901000 AND JSONExtractString(labels, 'host_name') = '4f6ec470feea') as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz.latency.sum'] AND unix_milli >= 1735036101000 AND unix_milli < 1735637901000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY `host.name`, ts ORDER BY `host.name` ASC, ts ASC",
+			expectedQueryContains: "SELECT `host.name`, ts, max(per_series_value) as value FROM (SELECT fingerprint, any(`host.name`) as `host.name`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'host.name') as `host.name`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz.latency.sum'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1734998400000 AND unix_milli < 1735637901000 AND JSONExtractString(labels, 'host_name') = '4f6ec470feea') as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz.latency.sum'] AND unix_milli >= 1735036101000 AND unix_milli < 1735637901000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY `host.name`, ts ORDER BY `host.name` ASC, ts ASC",
 		},
 	}
 

pkg/query-service/app/metrics/v4/delta/timeseries.go

@@ -24,7 +24,7 @@ func prepareTimeAggregationSubQuery(start, end, step int64, mq *v3.BuilderQuery)
 		return "", err
 	}
 
-	samplesTableFilter := fmt.Sprintf("metric_name IN %s AND unix_milli >= %d AND unix_milli < %d", utils.ClickHouseFormattedMetricNames(mq.AggregateAttribute.Key), start, end)
+	samplesTableFilter := fmt.Sprintf("metric_name IN %s AND unix_milli >= %d AND unix_milli < %d AND bitAnd(flags, 1) = 0", utils.ClickHouseFormattedMetricNames(mq.AggregateAttribute.Key), start, end)
 
 	tableName := helpers.WhichSamplesTableToUse(start, end, mq)
 
@@ -83,7 +83,7 @@ func prepareQueryOptimized(start, end, step int64, mq *v3.BuilderQuery) (string,
 		return "", err
 	}
 
-	samplesTableFilter := fmt.Sprintf("metric_name IN %s AND unix_milli >= %d AND unix_milli < %d", utils.ClickHouseFormattedMetricNames(mq.AggregateAttribute.Key), start, end)
+	samplesTableFilter := fmt.Sprintf("metric_name IN %s AND unix_milli >= %d AND unix_milli < %d AND bitAnd(flags, 1) = 0", utils.ClickHouseFormattedMetricNames(mq.AggregateAttribute.Key), start, end)
 
 	tableName := helpers.WhichSamplesTableToUse(start, end, mq)
 

pkg/query-service/app/metrics/v4/query_builder_pre_agg_test.go

@@ -49,7 +49,7 @@ func TestPrepareMetricQueryCumulativeRatePreAgg(t *testing.T) {
 				TimeAggregation:  v3.TimeAggregationRate,
 				SpaceAggregation: v3.SpaceAggregationSum,
 			},
-			expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(max) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991920000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(max) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991920000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = sum, temporality = cumulative, multiple group by",
@@ -82,7 +82,7 @@ func TestPrepareMetricQueryCumulativeRatePreAgg(t *testing.T) {
 				TimeAggregation:  v3.TimeAggregationRate,
 				SpaceAggregation: v3.SpaceAggregationSum,
 			},
-			expectedQueryContains: "SELECT service_name, endpoint, ts, sum(per_series_value) as value FROM (SELECT service_name, endpoint, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, any(endpoint) as endpoint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(max) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'endpoint') as endpoint, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991920000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, endpoint, ts ORDER BY service_name ASC, endpoint ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, endpoint, ts, sum(per_series_value) as value FROM (SELECT service_name, endpoint, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, any(endpoint) as endpoint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(max) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'endpoint') as endpoint, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991920000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, endpoint, ts ORDER BY service_name ASC, endpoint ASC, ts ASC",
 		},
 	}
 
@@ -123,7 +123,7 @@ func TestPrepareMetricQueryDeltaRatePreAgg(t *testing.T) {
 				TimeAggregation:  v3.TimeAggregationRate,
 				SpaceAggregation: v3.SpaceAggregationSum,
 			},
-			expectedQueryContains: "SELECT  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = sum, temporality = delta, group by service_name",
@@ -149,7 +149,7 @@ func TestPrepareMetricQueryDeltaRatePreAgg(t *testing.T) {
 				TimeAggregation:  v3.TimeAggregationRate,
 				SpaceAggregation: v3.SpaceAggregationSum,
 			},
-			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 	}
 
@@ -204,7 +204,7 @@ func TestPrepreMetricQueryCumulativeQuantilePreAgg(t *testing.T) {
 				Disabled:         false,
 				SpaceAggregation: v3.SpaceAggregationPercentile99,
 			},
-			expectedQueryContains: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT service_name, le, ts, sum(per_series_value) as value FROM (SELECT service_name, le, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, any(le) as le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(max) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, le, ts ORDER BY service_name ASC, le ASC, ts ASC) GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT service_name, le, ts, sum(per_series_value) as value FROM (SELECT service_name, le, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, any(le) as le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(max) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, le, ts ORDER BY service_name ASC, le ASC, ts ASC) GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test temporality = cumulative, quantile = 0.99 without group by",
@@ -234,7 +234,7 @@ func TestPrepreMetricQueryCumulativeQuantilePreAgg(t *testing.T) {
 				Disabled:         false,
 				SpaceAggregation: v3.SpaceAggregationPercentile99,
 			},
-			expectedQueryContains: "SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT le, ts, sum(per_series_value) as value FROM (SELECT le, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(le) as le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(max) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY le, ts ORDER BY le ASC, ts ASC) GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT le, ts, sum(per_series_value) as value FROM (SELECT le, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(le) as le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(max) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY le, ts ORDER BY le ASC, ts ASC) GROUP BY ts ORDER BY ts ASC",
 		},
 	}
 
@@ -289,7 +289,7 @@ func TestPrepreMetricQueryDeltaQuantilePreAgg(t *testing.T) {
 				Disabled:         false,
 				SpaceAggregation: v3.SpaceAggregationPercentile99,
 			},
-			expectedQueryContains: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT service_name, le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY service_name, le, ts ORDER BY service_name ASC, le ASC, ts ASC) GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT service_name, le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY service_name, le, ts ORDER BY service_name ASC, le ASC, ts ASC) GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test temporality = delta, quantile = 0.99 no group by",
@@ -319,7 +319,7 @@ func TestPrepreMetricQueryDeltaQuantilePreAgg(t *testing.T) {
 				Disabled:         false,
 				SpaceAggregation: v3.SpaceAggregationPercentile99,
 			},
-			expectedQueryContains: "SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY le, ts ORDER BY le ASC, ts ASC) GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum)/60 as value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY le, ts ORDER BY le ASC, ts ASC) GROUP BY ts ORDER BY ts ASC",
 		},
 	}
 
@@ -360,7 +360,7 @@ func TestPrepareMetricQueryGaugePreAgg(t *testing.T) {
 				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
-			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system_cpu_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['system_cpu_usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system_cpu_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['system_cpu_usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 		{
 			name: "test gauge query with group by host_name",
@@ -386,7 +386,7 @@ func TestPrepareMetricQueryGaugePreAgg(t *testing.T) {
 				Expression:       "A",
 				Disabled:         false,
 			},
-			expectedQueryContains: "SELECT host_name, ts, sum(per_series_value) as value FROM (SELECT fingerprint, any(host_name) as host_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'host_name') as host_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system_cpu_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['system_cpu_usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY host_name, ts ORDER BY host_name ASC, ts ASC",
+			expectedQueryContains: "SELECT host_name, ts, sum(per_series_value) as value FROM (SELECT fingerprint, any(host_name) as host_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(sum) / sum(count) as per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'host_name') as host_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system_cpu_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['system_cpu_usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY host_name, ts ORDER BY host_name ASC, ts ASC",
 		},
 	}
 

pkg/query-service/app/metrics/v4/query_builder_test.go

@@ -194,7 +194,7 @@ func TestPrepareMetricQueryCumulativeRate(t *testing.T) {
 				TimeAggregation:  v3.TimeAggregationRate,
 				SpaceAggregation: v3.SpaceAggregationSum,
 			},
-			expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991920000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991920000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = sum, temporality = cumulative, multiple group by",
@@ -227,7 +227,7 @@ func TestPrepareMetricQueryCumulativeRate(t *testing.T) {
 				TimeAggregation:  v3.TimeAggregationRate,
 				SpaceAggregation: v3.SpaceAggregationSum,
 			},
-			expectedQueryContains: "SELECT service_name, endpoint, ts, sum(per_series_value) as value FROM (SELECT service_name, endpoint, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, any(endpoint) as endpoint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'endpoint') as endpoint, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991920000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, endpoint, ts ORDER BY service_name ASC, endpoint ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, endpoint, ts, sum(per_series_value) as value FROM (SELECT service_name, endpoint, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, any(endpoint) as endpoint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'endpoint') as endpoint, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991920000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, endpoint, ts ORDER BY service_name ASC, endpoint ASC, ts ASC",
 		},
 	}
 
@@ -268,7 +268,7 @@ func TestPrepareMetricQueryDeltaRate(t *testing.T) {
 				TimeAggregation:  v3.TimeAggregationRate,
 				SpaceAggregation: v3.SpaceAggregationSum,
 			},
-			expectedQueryContains: "SELECT  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 		{
 			name: "test time aggregation = rate, space aggregation = sum, temporality = delta, group by service_name",
@@ -294,7 +294,7 @@ func TestPrepareMetricQueryDeltaRate(t *testing.T) {
 				TimeAggregation:  v3.TimeAggregationRate,
 				SpaceAggregation: v3.SpaceAggregationSum,
 			},
-			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_calls_total'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_calls_total'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 	}
 
@@ -349,7 +349,7 @@ func TestPrepreMetricQueryCumulativeQuantile(t *testing.T) {
 				Disabled:         false,
 				SpaceAggregation: v3.SpaceAggregationPercentile99,
 			},
-			expectedQueryContains: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT service_name, le, ts, sum(per_series_value) as value FROM (SELECT service_name, le, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, any(le) as le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, le, ts ORDER BY service_name ASC, le ASC, ts ASC) GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT service_name, le, ts, sum(per_series_value) as value FROM (SELECT service_name, le, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, any(le) as le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY service_name, le, ts ORDER BY service_name ASC, le ASC, ts ASC) GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test temporality = cumulative, quantile = 0.99 without group by",
@@ -379,7 +379,7 @@ func TestPrepreMetricQueryCumulativeQuantile(t *testing.T) {
 				Disabled:         false,
 				SpaceAggregation: v3.SpaceAggregationPercentile99,
 			},
-			expectedQueryContains: "SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT le, ts, sum(per_series_value) as value FROM (SELECT le, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(le) as le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY le, ts ORDER BY le ASC, ts ASC) GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT le, ts, sum(per_series_value) as value FROM (SELECT le, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(le) as le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY le, ts ORDER BY le ASC, ts ASC) GROUP BY ts ORDER BY ts ASC",
 		},
 	}
 
@@ -434,7 +434,7 @@ func TestPrepreMetricQueryDeltaQuantile(t *testing.T) {
 				Disabled:         false,
 				SpaceAggregation: v3.SpaceAggregationPercentile99,
 			},
-			expectedQueryContains: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT service_name, le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY service_name, le, ts ORDER BY service_name ASC, le ASC, ts ASC) GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
+			expectedQueryContains: "SELECT service_name, ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT service_name, le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY service_name, le, ts ORDER BY service_name ASC, le ASC, ts ASC) GROUP BY service_name, ts ORDER BY service_name ASC, ts ASC",
 		},
 		{
 			name: "test temporality = delta, quantile = 0.99 no group by",
@@ -464,7 +464,7 @@ func TestPrepreMetricQueryDeltaQuantile(t *testing.T) {
 				Disabled:         false,
 				SpaceAggregation: v3.SpaceAggregationPercentile99,
 			},
-			expectedQueryContains: "SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY le, ts ORDER BY le ASC, ts ASC) GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) as value FROM (SELECT le, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, sum(value)/60 as value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'le') as le, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['signoz_latency_bucket'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND like(JSONExtractString(labels, 'service_name'), '%frontend%')) as filtered_time_series USING fingerprint WHERE metric_name IN ['signoz_latency_bucket'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY le, ts ORDER BY le ASC, ts ASC) GROUP BY ts ORDER BY ts ASC",
 		},
 	}
 
@@ -505,7 +505,7 @@ func TestPrepareMetricQueryGauge(t *testing.T) {
 				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
-			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system_cpu_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['system_cpu_usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
+			expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint,  toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system_cpu_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['system_cpu_usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY ts ORDER BY ts ASC",
 		},
 		{
 			name: "test value filter with string value",
@@ -562,7 +562,7 @@ func TestPrepareMetricQueryGauge(t *testing.T) {
 				Expression:       "A",
 				Disabled:         false,
 			},
-			expectedQueryContains: "SELECT host_name, ts, sum(per_series_value) as value FROM (SELECT fingerprint, any(host_name) as host_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'host_name') as host_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system_cpu_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['system_cpu_usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY host_name, ts ORDER BY host_name ASC, ts ASC",
+			expectedQueryContains: "SELECT host_name, ts, sum(per_series_value) as value FROM (SELECT fingerprint, any(host_name) as host_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'host_name') as host_name, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system_cpu_usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000) as filtered_time_series USING fingerprint WHERE metric_name IN ['system_cpu_usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY host_name, ts ORDER BY host_name ASC, ts ASC",
 		},
 		{
 			name: "test gauge query with multiple group by with metric and attribute name containing dot",
@@ -631,7 +631,7 @@ func TestPrepareMetricQueryGauge(t *testing.T) {
 				ReduceTo: v3.ReduceToOperatorAvg,
 				Having:   []v3.Having{},
 			},
-			expectedQueryContains: "SELECT `os.type`, state, `host.name`, ts, max(per_series_value) as value FROM (SELECT fingerprint, any(`os.type`) as `os.type`, any(state) as state, any(`host.name`) as `host.name`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'os.type') as `os.type`, JSONExtractString(labels, 'state') as state, JSONExtractString(labels, 'host.name') as `host.name`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system.memory.usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name IN ['system.memory.usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY `os.type`, state, `host.name`, ts ORDER BY `os.type` desc, state asc, `host.name` ASC, ts ASC",
+			expectedQueryContains: "SELECT `os.type`, state, `host.name`, ts, max(per_series_value) as value FROM (SELECT fingerprint, any(`os.type`) as `os.type`, any(state) as state, any(`host.name`) as `host.name`, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'os.type') as `os.type`, JSONExtractString(labels, 'state') as state, JSONExtractString(labels, 'host.name') as `host.name`, fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN ['system.memory.usage'] AND temporality = 'Unspecified' AND __normalized = true AND unix_milli >= 1650931200000 AND unix_milli < 1651078380000 AND JSONExtractString(labels, 'host.name') = 'signoz-host') as filtered_time_series USING fingerprint WHERE metric_name IN ['system.memory.usage'] AND unix_milli >= 1650991980000 AND unix_milli < 1651078380000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY `os.type`, state, `host.name`, ts ORDER BY `os.type` desc, state asc, `host.name` ASC, ts ASC",
 		},
 	}
 
@@ -711,7 +711,7 @@ func TestPrepareMetricQueryValueTypePanelWithGroupBY(t *testing.T) {
 					},
 				},
 			},
-			expectedQueryContains: "SELECT max(value) as aggregated_value, ts FROM (SELECT state, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, anyLast(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1735891200000 AND unix_milli < 1735894800000 AND JSONExtractString(labels, 'os_type') = 'linux') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1735891800000 AND unix_milli < 1735894800000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, ts ORDER BY state desc, ts ASC) GROUP BY ts ORDER BY ts",
+			expectedQueryContains: "SELECT max(value) as aggregated_value, ts FROM (SELECT state, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, anyLast(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Delta' AND __normalized = true AND unix_milli >= 1735891200000 AND unix_milli < 1735894800000 AND JSONExtractString(labels, 'os_type') = 'linux') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1735891800000 AND unix_milli < 1735894800000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, ts ORDER BY state desc, ts ASC) GROUP BY ts ORDER BY ts",
 		},
 		{
 			name: "test temporality = cumulative, panel = value, series agg = max group by state, host_name",
@@ -777,7 +777,7 @@ func TestPrepareMetricQueryValueTypePanelWithGroupBY(t *testing.T) {
 					},
 				},
 			},
-			expectedQueryContains: "SELECT max(value) as aggregated_value, ts FROM (SELECT state, host_name, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, any(host_name) as host_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, anyLast(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, JSONExtractString(labels, 'host_name') as host_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1735891200000 AND unix_milli < 1735894800000 AND JSONExtractString(labels, 'os_type') = 'linux') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1735891800000 AND unix_milli < 1735894800000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, host_name, ts ORDER BY state desc, host_name ASC, ts ASC) GROUP BY ts ORDER BY ts",
+			expectedQueryContains: "SELECT max(value) as aggregated_value, ts FROM (SELECT state, host_name, ts, avg(per_series_value) as value FROM (SELECT fingerprint, any(state) as state, any(host_name) as host_name, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL 60 SECOND) as ts, anyLast(value) as per_series_value FROM signoz_metrics.distributed_samples_v4 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'state') as state, JSONExtractString(labels, 'host_name') as host_name, fingerprint FROM signoz_metrics.time_series_v4 WHERE metric_name IN ['system_memory_usage'] AND temporality = 'Cumulative' AND __normalized = true AND unix_milli >= 1735891200000 AND unix_milli < 1735894800000 AND JSONExtractString(labels, 'os_type') = 'linux') as filtered_time_series USING fingerprint WHERE metric_name IN ['system_memory_usage'] AND unix_milli >= 1735891800000 AND unix_milli < 1735894800000 AND bitAnd(flags, 1) = 0 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY state, host_name, ts ORDER BY state desc, host_name ASC, ts ASC) GROUP BY ts ORDER BY ts",
 		},
 	}
 

pkg/query-service/app/opamp/config_provider_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/query-service/app/opamp/model"
 	"github.com/SigNoz/signoz/pkg/query-service/utils"
-	"github.com/SigNoz/signoz/pkg/sqlstore"
 	"github.com/knadh/koanf"
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/rawbytes"
@@ -22,9 +21,6 @@ func TestOpAMPServerToAgentCommunicationWithConfigProvider(t *testing.T) {
 
 	tb := newTestbed(t)
 
-	orgID, err := utils.GetTestOrgId(tb.sqlStore)
-	require.Nil(err)
-
 	require.Equal(
 		0, len(tb.testConfigProvider.ConfigUpdateSubscribers),
 		"there should be no agent config subscribers at the start",
@@ -40,7 +36,6 @@ func TestOpAMPServerToAgentCommunicationWithConfigProvider(t *testing.T) {
 	require.False(tb.testConfigProvider.HasRecommendations())
 	agent1Conn := &MockOpAmpConnection{}
 	agent1Id := "testAgent1"
-	// get orgId from the db
 	tb.opampServer.OnMessage(
 		agent1Conn,
 		&protobufs.AgentToServer{
@@ -48,16 +43,6 @@ func TestOpAMPServerToAgentCommunicationWithConfigProvider(t *testing.T) {
 			EffectiveConfig: &protobufs.EffectiveConfig{
 				ConfigMap: initialAgentConf(),
 			},
-			AgentDescription: &protobufs.AgentDescription{
-				IdentifyingAttributes: []*protobufs.KeyValue{
-					{
-						Key: "orgId",
-						Value: &protobufs.AnyValue{
-							Value: &protobufs.AnyValue_StringValue{StringValue: orgID},
-						},
-					},
-				},
-			},
 		},
 	)
 	lastAgent1Msg := agent1Conn.LatestMsgFromServer()
@@ -81,16 +66,6 @@ func TestOpAMPServerToAgentCommunicationWithConfigProvider(t *testing.T) {
 			EffectiveConfig: &protobufs.EffectiveConfig{
 				ConfigMap: initialAgentConf(),
 			},
-			AgentDescription: &protobufs.AgentDescription{
-				IdentifyingAttributes: []*protobufs.KeyValue{
-					{
-						Key: "orgId",
-						Value: &protobufs.AnyValue{
-							Value: &protobufs.AnyValue_StringValue{StringValue: orgID},
-						},
-					},
-				},
-			},
 		},
 	)
 	lastAgent2Msg := agent2Conn.LatestMsgFromServer()
@@ -187,26 +162,22 @@ type testbed struct {
 	testConfigProvider *MockAgentConfigProvider
 	opampServer        *Server
 	t                  *testing.T
-	sqlStore           sqlstore.SQLStore
 }
 
 func newTestbed(t *testing.T) *testbed {
 	testDB := utils.NewQueryServiceDBForTests(t)
-	model.InitDB(testDB)
+	_, err := model.InitDB(testDB.SQLxDB())
+	if err != nil {
+		t.Fatalf("could not init opamp model: %v", err)
+	}
+
 	testConfigProvider := NewMockAgentConfigProvider()
 	opampServer := InitializeServer(nil, testConfigProvider)
 
-	// create a test org
-	err := utils.CreateTestOrg(t, testDB)
-	if err != nil {
-		t.Fatalf("could not create test org: %v", err)
-	}
-
 	return &testbed{
 		testConfigProvider: testConfigProvider,
 		opampServer:        opampServer,
 		t:                  t,
-		sqlStore:           testDB,
 	}
 }
 

pkg/query-service/app/opamp/configure_ingestionRules.go

@@ -60,13 +60,13 @@ func UpsertControlProcessors(
 
 		agenthash, err := addIngestionControlToAgent(agent, signal, processors, false)
 		if err != nil {
-			zap.L().Error("failed to push ingestion rules config to agent", zap.String("agentID", agent.ID.StringValue()), zap.Error(err))
+			zap.L().Error("failed to push ingestion rules config to agent", zap.String("agentID", agent.ID), zap.Error(err))
 			continue
 		}
 
 		if agenthash != "" {
 			// subscribe callback
-			model.ListenToConfigUpdate(agent.ID.StringValue(), agenthash, callback)
+			model.ListenToConfigUpdate(agent.ID, agenthash, callback)
 		}
 
 		hash = agenthash
@@ -89,7 +89,7 @@ func addIngestionControlToAgent(agent *model.Agent, signal string, processors ma
 	// add ingestion control spec
 	err = makeIngestionControlSpec(agentConf, Signal(signal), processors)
 	if err != nil {
-		zap.L().Error("failed to prepare ingestion control processors for agent", zap.String("agentID", agent.ID.StringValue()), zap.Error(err))
+		zap.L().Error("failed to prepare ingestion control processors for agent", zap.String("agentID", agent.ID), zap.Error(err))
 		return confHash, err
 	}
 

pkg/query-service/app/opamp/mocks.go

@@ -67,7 +67,7 @@ func (ta *MockAgentConfigProvider) HasRecommendations() bool {
 }
 
 // AgentConfigProvider interface
-func (ta *MockAgentConfigProvider) RecommendAgentConfig(orgId string, baseConfYaml []byte) (
+func (ta *MockAgentConfigProvider) RecommendAgentConfig(baseConfYaml []byte) (
 	[]byte, string, error,
 ) {
 	if len(ta.ZPagesEndpoint) < 1 {
@@ -92,7 +92,6 @@ func (ta *MockAgentConfigProvider) RecommendAgentConfig(orgId string, baseConfYa
 
 // AgentConfigProvider interface
 func (ta *MockAgentConfigProvider) ReportConfigDeploymentStatus(
-	orgId string,
 	agentId string,
 	configId string,
 	err error,

pkg/query-service/app/opamp/model/agent.go

@@ -7,20 +7,33 @@ import (
 	"sync"
 	"time"
 
-	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/types"
-	"github.com/SigNoz/signoz/pkg/valuer"
 	"go.uber.org/zap"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/open-telemetry/opamp-go/protobufs"
-	opampTypes "github.com/open-telemetry/opamp-go/server/types"
+	"github.com/open-telemetry/opamp-go/server/types"
 )
 
+type AgentStatus int
+
+const (
+	AgentStatusUnknown AgentStatus = iota
+	AgentStatusConnected
+	AgentStatusDisconnected
+)
+
+// set in agent description when agent is capable of supporting
+// lb exporter configuration. values: 1 (true) or 0 (false)
+const lbExporterFlag = "capabilities.lbexporter"
+
 type Agent struct {
-	types.StorableAgent
-	remoteConfig *protobufs.AgentRemoteConfig
-	Status       *protobufs.AgentToServer
+	ID              string      `json:"agentId" yaml:"agentId" db:"agent_id"`
+	StartedAt       time.Time   `json:"startedAt" yaml:"startedAt" db:"started_at"`
+	TerminatedAt    time.Time   `json:"terminatedAt" yaml:"terminatedAt" db:"terminated_at"`
+	EffectiveConfig string      `json:"effectiveConfig" yaml:"effectiveConfig" db:"effective_config"`
+	CurrentStatus   AgentStatus `json:"currentStatus" yaml:"currentStatus" db:"current_status"`
+	remoteConfig    *protobufs.AgentRemoteConfig
+	Status          *protobufs.AgentToServer
 
 	// can this agent be load balancer
 	CanLB bool
@@ -28,18 +41,13 @@ type Agent struct {
 	// is this agent setup as load balancer
 	IsLb bool
 
-	conn      opampTypes.Connection
+	conn      types.Connection
 	connMutex sync.Mutex
 	mux       sync.RWMutex
-	store     sqlstore.SQLStore
 }
 
-// set in agent description when agent is capable of supporting
-// lb exporter configuration. values: 1 (true) or 0 (false)
-const lbExporterFlag = "capabilities.lbexporter"
-
-func New(store sqlstore.SQLStore, orgID string, ID string, conn opampTypes.Connection) *Agent {
-	return &Agent{StorableAgent: types.StorableAgent{OrgID: orgID, Identifiable: types.Identifiable{ID: valuer.GenerateUUID()}, StartedAt: time.Now(), CurrentStatus: types.AgentStatusConnected}, conn: conn, store: store}
+func New(ID string, conn types.Connection) *Agent {
+	return &Agent{ID: ID, StartedAt: time.Now(), CurrentStatus: AgentStatusConnected, conn: conn}
 }
 
 // Upsert inserts or updates the agent in the database.
@@ -47,13 +55,17 @@ func (agent *Agent) Upsert() error {
 	agent.mux.Lock()
 	defer agent.mux.Unlock()
 
-	_, err := agent.store.BunDB().NewInsert().
-		Model(&agent.StorableAgent).
-		On("CONFLICT (org_id, id) DO UPDATE").
-		Set("started_at = EXCLUDED.started_at").
-		Set("effective_config = EXCLUDED.effective_config").
-		Set("current_status = EXCLUDED.current_status").
-		Exec(context.Background())
+	_, err := db.NamedExec(`INSERT OR REPLACE INTO agents (
+		agent_id,
+		started_at,
+		effective_config,
+		current_status
+	) VALUES (
+		:agent_id,
+		:started_at,
+		:effective_config,
+		:current_status
+	)`, agent)
 	if err != nil {
 		return err
 	}
@@ -123,11 +135,11 @@ func (agent *Agent) updateAgentDescription(newStatus *protobufs.AgentToServer) (
 			// todo: need to address multiple agent scenario here
 			// for now, the first response will be sent back to the UI
 			if agent.Status.RemoteConfigStatus.Status == protobufs.RemoteConfigStatuses_RemoteConfigStatuses_APPLIED {
-				onConfigSuccess(agent.OrgID, agent.ID.StringValue(), string(agent.Status.RemoteConfigStatus.LastRemoteConfigHash))
+				onConfigSuccess(agent.ID, string(agent.Status.RemoteConfigStatus.LastRemoteConfigHash))
 			}
 
 			if agent.Status.RemoteConfigStatus.Status == protobufs.RemoteConfigStatuses_RemoteConfigStatuses_FAILED {
-				onConfigFailure(agent.OrgID, agent.ID.StringValue(), string(agent.Status.RemoteConfigStatus.LastRemoteConfigHash), agent.Status.RemoteConfigStatus.ErrorMessage)
+				onConfigFailure(agent.ID, string(agent.Status.RemoteConfigStatus.LastRemoteConfigHash), agent.Status.RemoteConfigStatus.ErrorMessage)
 			}
 		}
 	}
@@ -257,7 +269,7 @@ func (agent *Agent) processStatusUpdate(
 		agent.SendToAgent(response)
 
 		ListenToConfigUpdate(
-			agent.ID.StringValue(),
+			agent.ID,
 			string(response.RemoteConfig.ConfigHash),
 			configProvider.ReportConfigDeploymentStatus,
 		)
@@ -265,9 +277,9 @@ func (agent *Agent) processStatusUpdate(
 }
 
 func (agent *Agent) updateRemoteConfig(configProvider AgentConfigProvider) bool {
-	recommendedConfig, confId, err := configProvider.RecommendAgentConfig(agent.OrgID, []byte(agent.EffectiveConfig))
+	recommendedConfig, confId, err := configProvider.RecommendAgentConfig([]byte(agent.EffectiveConfig))
 	if err != nil {
-		zap.L().Error("could not generate config recommendation for agent", zap.String("agentID", agent.ID.StringValue()), zap.Error(err))
+		zap.L().Error("could not generate config recommendation for agent", zap.String("agentID", agent.ID), zap.Error(err))
 		return false
 	}
 

pkg/query-service/app/opamp/model/agents.go

@@ -1,19 +1,19 @@
 package model
 
 import (
-	"context"
 	"fmt"
 	"sync"
 	"time"
 
-	"github.com/SigNoz/signoz/pkg/sqlstore"
-	signozTypes "github.com/SigNoz/signoz/pkg/types"
+	"github.com/jmoiron/sqlx"
 	"github.com/open-telemetry/opamp-go/protobufs"
 	"github.com/open-telemetry/opamp-go/server/types"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 )
 
+var db *sqlx.DB
+
 var AllAgents = Agents{
 	agentsById:  map[string]*Agent{},
 	connections: map[types.Connection]map[string]bool{},
@@ -23,7 +23,6 @@ type Agents struct {
 	mux         sync.RWMutex
 	agentsById  map[string]*Agent
 	connections map[types.Connection]map[string]bool
-	store       sqlstore.SQLStore
 }
 
 func (a *Agents) Count() int {
@@ -31,14 +30,15 @@ func (a *Agents) Count() int {
 }
 
 // Initialize the database and create schema if needed
-func InitDB(sqlStore sqlstore.SQLStore) {
+func InitDB(qsDB *sqlx.DB) (*sqlx.DB, error) {
+	db = qsDB
 
 	AllAgents = Agents{
 		agentsById:  make(map[string]*Agent),
 		connections: make(map[types.Connection]map[string]bool),
 		mux:         sync.RWMutex{},
-		store:       sqlStore,
 	}
+	return db, nil
 }
 
 // RemoveConnection removes the connection all Agent instances associated with the
@@ -49,7 +49,7 @@ func (agents *Agents) RemoveConnection(conn types.Connection) {
 
 	for instanceId := range agents.connections[conn] {
 		agent := agents.agentsById[instanceId]
-		agent.CurrentStatus = signozTypes.AgentStatusDisconnected
+		agent.CurrentStatus = AgentStatusDisconnected
 		agent.TerminatedAt = time.Now()
 		_ = agent.Upsert()
 		delete(agents.agentsById, instanceId)
@@ -67,43 +67,27 @@ func (agents *Agents) FindAgent(agentID string) *Agent {
 // FindOrCreateAgent returns the Agent instance associated with the given agentID.
 // If the Agent instance does not exist, it is created and added to the list of
 // Agent instances.
-func (agents *Agents) FindOrCreateAgent(agentID string, conn types.Connection, orgId string) (*Agent, bool, error) {
+func (agents *Agents) FindOrCreateAgent(agentID string, conn types.Connection) (*Agent, bool, error) {
 	agents.mux.Lock()
 	defer agents.mux.Unlock()
+	var created bool
 	agent, ok := agents.agentsById[agentID]
-
-	if ok && agent != nil {
-		return agent, false, nil
-	}
-
-	// This is for single org mode
-	if orgId == "SIGNOZ##DEFAULT##ORG##ID" {
-		err := agents.store.BunDB().NewSelect().
-			Model((*signozTypes.Organization)(nil)).
-			ColumnExpr("id").
-			Limit(1).
-			Scan(context.Background(), &orgId)
+	var err error
+	if !ok || agent == nil {
+		agent = New(agentID, conn)
+		err = agent.Upsert()
 		if err != nil {
-			return nil, false, err
+			return nil, created, err
 		}
-	}
+		agents.agentsById[agentID] = agent
 
-	if !ok && orgId == "" {
-		return nil, false, errors.New("cannot create agent without orgId")
+		if agents.connections[conn] == nil {
+			agents.connections[conn] = map[string]bool{}
+		}
+		agents.connections[conn][agentID] = true
+		created = true
 	}
-
-	agent = New(agents.store, orgId, agentID, conn)
-	err := agent.Upsert()
-	if err != nil {
-		return nil, false, err
-	}
-	agents.agentsById[agentID] = agent
-
-	if agents.connections[conn] == nil {
-		agents.connections[conn] = map[string]bool{}
-	}
-	agents.connections[conn][agentID] = true
-	return agent, true, nil
+	return agent, created, nil
 }
 
 func (agents *Agents) GetAllAgents() []*Agent {
@@ -124,19 +108,18 @@ func (agents *Agents) RecommendLatestConfigToAll(
 ) error {
 	for _, agent := range agents.GetAllAgents() {
 		newConfig, confId, err := provider.RecommendAgentConfig(
-			agent.OrgID,
 			[]byte(agent.EffectiveConfig),
 		)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf(
-				"could not generate conf recommendation for %v", agent.ID.StringValue(),
+				"could not generate conf recommendation for %v", agent.ID,
 			))
 		}
 
 		// Recommendation is same as current config
 		if string(newConfig) == agent.EffectiveConfig {
 			zap.L().Info(
-				"Recommended config same as current effective config for agent", zap.String("agentID", agent.ID.StringValue()),
+				"Recommended config same as current effective config for agent", zap.String("agentID", agent.ID),
 			)
 			return nil
 		}
@@ -161,7 +144,7 @@ func (agents *Agents) RecommendLatestConfigToAll(
 			RemoteConfig: newRemoteConfig,
 		})
 
-		ListenToConfigUpdate(agent.ID.StringValue(), confId, provider.ReportConfigDeploymentStatus)
+		ListenToConfigUpdate(agent.ID, confId, provider.ReportConfigDeploymentStatus)
 	}
 	return nil
 }

pkg/query-service/app/opamp/model/config.go

@@ -4,7 +4,7 @@ package model
 type AgentConfigProvider interface {
 	// Generate recommended config for an agent based on its `currentConfYaml`
 	// and current state of user facing settings for agent based features.
-	RecommendAgentConfig(orgId string, currentConfYaml []byte) (
+	RecommendAgentConfig(currentConfYaml []byte) (
 		recommendedConfYaml []byte,
 		// Opaque id of the recommended config, used for reporting deployment status updates
 		configId string,
@@ -13,7 +13,6 @@ type AgentConfigProvider interface {
 
 	// Report deployment status for config recommendations generated by RecommendAgentConfig
 	ReportConfigDeploymentStatus(
-		orgId string,
 		agentId string,
 		configId string,
 		err error,

pkg/query-service/app/opamp/model/coordinator.go

@@ -15,7 +15,7 @@ func init() {
 	}
 }
 
-type OnChangeCallback func(orgId string, agentId string, hash string, err error)
+type OnChangeCallback func(agentId string, hash string, err error)
 
 // responsible for managing subscribers on config change
 type Coordinator struct {
@@ -25,16 +25,16 @@ type Coordinator struct {
 	subscribers map[string][]OnChangeCallback
 }
 
-func onConfigSuccess(orgId string, agentId string, hash string) {
-	notifySubscribers(orgId, agentId, hash, nil)
+func onConfigSuccess(agentId string, hash string) {
+	notifySubscribers(agentId, hash, nil)
 }
 
-func onConfigFailure(orgId string, agentId string, hash string, errorMessage string) {
-	notifySubscribers(orgId, agentId, hash, fmt.Errorf(errorMessage))
+func onConfigFailure(agentId string, hash string, errorMessage string) {
+	notifySubscribers(agentId, hash, fmt.Errorf(errorMessage))
 }
 
 // OnSuccess listens to config changes and notifies subscribers
-func notifySubscribers(orgId string, agentId string, hash string, err error) {
+func notifySubscribers(agentId string, hash string, err error) {
 	// this method currently does not handle multi-agent scenario.
 	// as soon as a message is delivered, we release all the subscribers
 	// for a given hash
@@ -44,7 +44,7 @@ func notifySubscribers(orgId string, agentId string, hash string, err error) {
 	}
 
 	for _, s := range subs {
-		s(orgId, agentId, hash, err)
+		s(agentId, hash, err)
 	}
 
 	// delete all subscribers for this hash, assume future

pkg/query-service/app/opamp/opamp_server.go

@@ -2,7 +2,6 @@ package opamp
 
 import (
 	"context"
-	"time"
 
 	model "github.com/SigNoz/signoz/pkg/query-service/app/opamp/model"
 	"github.com/open-telemetry/opamp-go/protobufs"
@@ -54,7 +53,6 @@ func (srv *Server) Start(listener string) error {
 		ListenEndpoint: listener,
 	}
 
-	// This will have to send request to all the agents of all tenants
 	unsubscribe := srv.agentConfigProvider.SubscribeToConfigUpdates(func() {
 		err := srv.agents.RecommendLatestConfigToAll(srv.agentConfigProvider)
 		if err != nil {
@@ -80,46 +78,20 @@ func (srv *Server) onDisconnect(conn types.Connection) {
 	srv.agents.RemoveConnection(conn)
 }
 
-// When the agent sends the message for the first time, then we need to know the orgID
-// For the subsequent requests, agents don't send the attributes unless something is changed
-// but we keep them in context mapped which is mapped to the instanceID, so we would know the
-// orgID from the context
 func (srv *Server) OnMessage(conn types.Connection, msg *protobufs.AgentToServer) *protobufs.ServerToAgent {
 	agentID := msg.InstanceUid
 
-	var orgId string
-	if msg.AgentDescription != nil && msg.AgentDescription.IdentifyingAttributes != nil {
-		for _, attr := range msg.AgentDescription.IdentifyingAttributes {
-			if attr.Key == "orgId" {
-				orgId = attr.Value.GetStringValue()
-				break
-			}
-		}
-	}
-
-	agent, created, err := srv.agents.FindOrCreateAgent(agentID, conn, orgId)
+	agent, created, err := srv.agents.FindOrCreateAgent(agentID, conn)
 	if err != nil {
 		zap.L().Error("Failed to find or create agent", zap.String("agentID", agentID), zap.Error(err))
-
-		// Return error response according to OpAMP protocol
-		return &protobufs.ServerToAgent{
-			InstanceUid: agentID,
-			ErrorResponse: &protobufs.ServerErrorResponse{
-				Type: protobufs.ServerErrorResponseType_ServerErrorResponseType_Unavailable,
-				Details: &protobufs.ServerErrorResponse_RetryInfo{
-					RetryInfo: &protobufs.RetryInfo{
-						RetryAfterNanoseconds: uint64(5 * time.Second), // minimum recommended retry interval
-					},
-				},
-			},
-		}
+		// TODO: handle error
 	}
 
 	if created {
 		agent.CanLB = model.ExtractLbFlag(msg.AgentDescription)
 		zap.L().Debug(
 			"New agent added", zap.Bool("canLb", agent.CanLB),
-			zap.String("ID", agent.ID.StringValue()),
+			zap.String("ID", agent.ID),
 			zap.Any("status", agent.CurrentStatus),
 		)
 	}

pkg/query-service/app/preferences/map.go

@@ -0,0 +1,84 @@
+package preferences
+
+var preferenceMap = map[string]Preference{
+	"ORG_ONBOARDING": {
+		Key:              "ORG_ONBOARDING",
+		Name:             "Organisation Onboarding",
+		Description:      "Organisation Onboarding",
+		ValueType:        "boolean",
+		DefaultValue:     false,
+		AllowedValues:    []interface{}{true, false},
+		IsDiscreteValues: true,
+		AllowedScopes:    []string{"org"},
+	},
+	"WELCOME_CHECKLIST_DO_LATER": {
+		Key:              "WELCOME_CHECKLIST_DO_LATER",
+		Name:             "Welcome Checklist Do Later",
+		Description:      "Welcome Checklist Do Later",
+		ValueType:        "boolean",
+		DefaultValue:     false,
+		AllowedValues:    []interface{}{true, false},
+		IsDiscreteValues: true,
+		AllowedScopes:    []string{"user"},
+	},
+	"WELCOME_CHECKLIST_SEND_LOGS_SKIPPED": {
+		Key:              "WELCOME_CHECKLIST_SEND_LOGS_SKIPPED",
+		Name:             "Welcome Checklist Send Logs Skipped",
+		Description:      "Welcome Checklist Send Logs Skipped",
+		ValueType:        "boolean",
+		DefaultValue:     false,
+		AllowedValues:    []interface{}{true, false},
+		IsDiscreteValues: true,
+		AllowedScopes:    []string{"user"},
+	},
+	"WELCOME_CHECKLIST_SEND_TRACES_SKIPPED": {
+		Key:              "WELCOME_CHECKLIST_SEND_TRACES_SKIPPED",
+		Name:             "Welcome Checklist Send Traces Skipped",
+		Description:      "Welcome Checklist Send Traces Skipped",
+		ValueType:        "boolean",
+		DefaultValue:     false,
+		AllowedValues:    []interface{}{true, false},
+		IsDiscreteValues: true,
+		AllowedScopes:    []string{"user"},
+	},
+	"WELCOME_CHECKLIST_SEND_INFRA_METRICS_SKIPPED": {
+		Key:              "WELCOME_CHECKLIST_SEND_INFRA_METRICS_SKIPPED",
+		Name:             "Welcome Checklist Send Infra Metrics Skipped",
+		Description:      "Welcome Checklist Send Infra Metrics Skipped",
+		ValueType:        "boolean",
+		DefaultValue:     false,
+		AllowedValues:    []interface{}{true, false},
+		IsDiscreteValues: true,
+		AllowedScopes:    []string{"user"},
+	},
+	"WELCOME_CHECKLIST_SETUP_DASHBOARDS_SKIPPED": {
+		Key:              "WELCOME_CHECKLIST_SETUP_DASHBOARDS_SKIPPED",
+		Name:             "Welcome Checklist Setup Dashboards Skipped",
+		Description:      "Welcome Checklist Setup Dashboards Skipped",
+		ValueType:        "boolean",
+		DefaultValue:     false,
+		AllowedValues:    []interface{}{true, false},
+		IsDiscreteValues: true,
+		AllowedScopes:    []string{"user"},
+	},
+	"WELCOME_CHECKLIST_SETUP_ALERTS_SKIPPED": {
+		Key:              "WELCOME_CHECKLIST_SETUP_ALERTS_SKIPPED",
+		Name:             "Welcome Checklist Setup Alerts Skipped",
+		Description:      "Welcome Checklist Setup Alerts Skipped",
+		ValueType:        "boolean",
+		DefaultValue:     false,
+		AllowedValues:    []interface{}{true, false},
+		IsDiscreteValues: true,
+		AllowedScopes:    []string{"user"},
+	},
+	"WELCOME_CHECKLIST_SETUP_SAVED_VIEW_SKIPPED": {
+		Key:              "WELCOME_CHECKLIST_SETUP_SAVED_VIEW_SKIPPED",
+		Name:             "Welcome Checklist Setup Saved View Skipped",
+		Description:      "Welcome Checklist Setup Saved View Skipped",
+		ValueType:        "boolean",
+		DefaultValue:     false,
+		AllowedValues:    []interface{}{true, false},
+		IsDiscreteValues: true,
+		AllowedScopes:    []string{"user"},
+	},
+}

pkg/query-service/app/preferences/model.go

@@ -0,0 +1,500 @@
+package preferences
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/query-service/model"
+	"github.com/jmoiron/sqlx"
+)
+
+type Range struct {
+	Min int64 `json:"min"`
+	Max int64 `json:"max"`
+}
+
+type Preference struct {
+	Key              string        `json:"key"`
+	Name             string        `json:"name"`
+	Description      string        `json:"description"`
+	ValueType        string        `json:"valueType"`
+	DefaultValue     interface{}   `json:"defaultValue"`
+	AllowedValues    []interface{} `json:"allowedValues"`
+	IsDiscreteValues bool          `json:"isDiscreteValues"`
+	Range            Range         `json:"range"`
+	AllowedScopes    []string      `json:"allowedScopes"`
+}
+
+func (p *Preference) ErrorValueTypeMismatch() *model.ApiError {
+	return &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("the preference value is not of expected type: %s", p.ValueType)}
+}
+
+const (
+	PreferenceValueTypeInteger string = "integer"
+	PreferenceValueTypeFloat   string = "float"
+	PreferenceValueTypeString  string = "string"
+	PreferenceValueTypeBoolean string = "boolean"
+)
+
+const (
+	OrgAllowedScope  string = "org"
+	UserAllowedScope string = "user"
+)
+
+func (p *Preference) checkIfInAllowedValues(preferenceValue interface{}) (bool, *model.ApiError) {
+
+	switch p.ValueType {
+	case PreferenceValueTypeInteger:
+		_, ok := preferenceValue.(int64)
+		if !ok {
+			return false, p.ErrorValueTypeMismatch()
+		}
+	case PreferenceValueTypeFloat:
+		_, ok := preferenceValue.(float64)
+		if !ok {
+			return false, p.ErrorValueTypeMismatch()
+		}
+	case PreferenceValueTypeString:
+		_, ok := preferenceValue.(string)
+		if !ok {
+			return false, p.ErrorValueTypeMismatch()
+		}
+	case PreferenceValueTypeBoolean:
+		_, ok := preferenceValue.(bool)
+		if !ok {
+			return false, p.ErrorValueTypeMismatch()
+		}
+	}
+	isInAllowedValues := false
+	for _, value := range p.AllowedValues {
+		switch p.ValueType {
+		case PreferenceValueTypeInteger:
+			allowedValue, ok := value.(int64)
+			if !ok {
+				return false, p.ErrorValueTypeMismatch()
+			}
+
+			if allowedValue == preferenceValue {
+				isInAllowedValues = true
+			}
+		case PreferenceValueTypeFloat:
+			allowedValue, ok := value.(float64)
+			if !ok {
+				return false, p.ErrorValueTypeMismatch()
+			}
+
+			if allowedValue == preferenceValue {
+				isInAllowedValues = true
+			}
+		case PreferenceValueTypeString:
+			allowedValue, ok := value.(string)
+			if !ok {
+				return false, p.ErrorValueTypeMismatch()
+			}
+
+			if allowedValue == preferenceValue {
+				isInAllowedValues = true
+			}
+		case PreferenceValueTypeBoolean:
+			allowedValue, ok := value.(bool)
+			if !ok {
+				return false, p.ErrorValueTypeMismatch()
+			}
+
+			if allowedValue == preferenceValue {
+				isInAllowedValues = true
+			}
+		}
+	}
+	return isInAllowedValues, nil
+}
+
+func (p *Preference) IsValidValue(preferenceValue interface{}) *model.ApiError {
+	typeSafeValue := preferenceValue
+	switch p.ValueType {
+	case PreferenceValueTypeInteger:
+		val, ok := preferenceValue.(int64)
+		if !ok {
+			floatVal, ok := preferenceValue.(float64)
+			if !ok || floatVal != float64(int64(floatVal)) {
+				return p.ErrorValueTypeMismatch()
+			}
+			val = int64(floatVal)
+			typeSafeValue = val
+		}
+		if !p.IsDiscreteValues {
+			if val < p.Range.Min || val > p.Range.Max {
+				return &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("the preference value is not in the range specified, min: %v , max:%v", p.Range.Min, p.Range.Max)}
+			}
+		}
+	case PreferenceValueTypeString:
+		_, ok := preferenceValue.(string)
+		if !ok {
+			return p.ErrorValueTypeMismatch()
+		}
+	case PreferenceValueTypeFloat:
+		_, ok := preferenceValue.(float64)
+		if !ok {
+			return p.ErrorValueTypeMismatch()
+		}
+	case PreferenceValueTypeBoolean:
+		_, ok := preferenceValue.(bool)
+		if !ok {
+			return p.ErrorValueTypeMismatch()
+		}
+	}
+
+	// check the validity of the value being part of allowed values or the range specified if any
+	if p.IsDiscreteValues {
+		if p.AllowedValues != nil {
+			isInAllowedValues, valueMisMatchErr := p.checkIfInAllowedValues(typeSafeValue)
+
+			if valueMisMatchErr != nil {
+				return valueMisMatchErr
+			}
+			if !isInAllowedValues {
+				return &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("the preference value is not in the list of allowedValues: %v", p.AllowedValues)}
+			}
+		}
+	}
+	return nil
+}
+
+func (p *Preference) IsEnabledForScope(scope string) bool {
+	isPreferenceEnabledForGivenScope := false
+	if p.AllowedScopes != nil {
+		for _, allowedScope := range p.AllowedScopes {
+			if allowedScope == strings.ToLower(scope) {
+				isPreferenceEnabledForGivenScope = true
+			}
+		}
+	}
+	return isPreferenceEnabledForGivenScope
+}
+
+func (p *Preference) SanitizeValue(preferenceValue interface{}) interface{} {
+	switch p.ValueType {
+	case PreferenceValueTypeBoolean:
+		if preferenceValue == "1" || preferenceValue == true {
+			return true
+		} else {
+			return false
+		}
+	default:
+		return preferenceValue
+	}
+}
+
+type AllPreferences struct {
+	Preference
+	Value interface{} `json:"value"`
+}
+
+type PreferenceKV struct {
+	PreferenceId    string      `json:"preference_id" db:"preference_id"`
+	PreferenceValue interface{} `json:"preference_value" db:"preference_value"`
+}
+
+type UpdatePreference struct {
+	PreferenceValue interface{} `json:"preference_value"`
+}
+
+var db *sqlx.DB
+
+func InitDB(inputDB *sqlx.DB) error {
+	db = inputDB
+	return nil
+}
+
+// org preference functions
+func GetOrgPreference(ctx context.Context, preferenceId string, orgId string) (*PreferenceKV, *model.ApiError) {
+	// check if the preference key exists or not
+	preference, seen := preferenceMap[preferenceId]
+	if !seen {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no such preferenceId exists: %s", preferenceId)}
+	}
+
+	// check if the preference is enabled for org scope or not
+	isPreferenceEnabled := preference.IsEnabledForScope(OrgAllowedScope)
+	if !isPreferenceEnabled {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("preference is not enabled at org scope: %s", preferenceId)}
+	}
+
+	// fetch the value from the database
+	var orgPreference PreferenceKV
+	query := `SELECT preference_id , preference_value FROM org_preference WHERE preference_id=$1 AND org_id=$2;`
+	err := db.Get(&orgPreference, query, preferenceId, orgId)
+
+	// if the value doesn't exist in db then return the default value
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return &PreferenceKV{
+				PreferenceId:    preferenceId,
+				PreferenceValue: preference.DefaultValue,
+			}, nil
+		}
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in fetching the org preference: %s", err.Error())}
+
+	}
+
+	// else return the value fetched from the org_preference table
+	return &PreferenceKV{
+		PreferenceId:    preferenceId,
+		PreferenceValue: preference.SanitizeValue(orgPreference.PreferenceValue),
+	}, nil
+}
+
+func UpdateOrgPreference(ctx context.Context, preferenceId string, preferenceValue interface{}, orgId string) (*PreferenceKV, *model.ApiError) {
+	// check if the preference key exists or not
+	preference, seen := preferenceMap[preferenceId]
+	if !seen {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no such preferenceId exists: %s", preferenceId)}
+	}
+
+	// check if the preference is enabled at org scope or not
+	isPreferenceEnabled := preference.IsEnabledForScope(OrgAllowedScope)
+	if !isPreferenceEnabled {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("preference is not enabled at org scope: %s", preferenceId)}
+	}
+
+	err := preference.IsValidValue(preferenceValue)
+	if err != nil {
+		return nil, err
+	}
+
+	// update the values in the org_preference table and return the key and the value
+	query := `INSERT INTO org_preference(preference_id,preference_value,org_id) VALUES($1,$2,$3)
+	ON CONFLICT(preference_id,org_id) DO
+	UPDATE SET preference_value= $2 WHERE preference_id=$1 AND org_id=$3;`
+
+	_, dberr := db.Exec(query, preferenceId, preferenceValue, orgId)
+
+	if dberr != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in setting the preference value: %s", dberr.Error())}
+	}
+
+	return &PreferenceKV{
+		PreferenceId:    preferenceId,
+		PreferenceValue: preferenceValue,
+	}, nil
+}
+
+func GetAllOrgPreferences(ctx context.Context, orgId string) (*[]AllPreferences, *model.ApiError) {
+	// filter out all the org enabled preferences from the preference variable
+	allOrgPreferences := []AllPreferences{}
+
+	// fetch all the org preference values stored in org_preference table
+	orgPreferenceValues := []PreferenceKV{}
+
+	query := `SELECT preference_id,preference_value FROM org_preference WHERE org_id=$1;`
+	err := db.Select(&orgPreferenceValues, query, orgId)
+
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in getting all org preference values: %s", err)}
+	}
+
+	// create a map of key vs values from the above response
+	preferenceValueMap := map[string]interface{}{}
+
+	for _, preferenceValue := range orgPreferenceValues {
+		preferenceValueMap[preferenceValue.PreferenceId] = preferenceValue.PreferenceValue
+	}
+
+	// update in the above filtered list wherver value present in the map
+	for _, preference := range preferenceMap {
+		isEnabledForOrgScope := preference.IsEnabledForScope(OrgAllowedScope)
+		if isEnabledForOrgScope {
+			preferenceWithValue := AllPreferences{}
+			preferenceWithValue.Key = preference.Key
+			preferenceWithValue.Name = preference.Name
+			preferenceWithValue.Description = preference.Description
+			preferenceWithValue.AllowedScopes = preference.AllowedScopes
+			preferenceWithValue.AllowedValues = preference.AllowedValues
+			preferenceWithValue.DefaultValue = preference.DefaultValue
+			preferenceWithValue.Range = preference.Range
+			preferenceWithValue.ValueType = preference.ValueType
+			preferenceWithValue.IsDiscreteValues = preference.IsDiscreteValues
+			value, seen := preferenceValueMap[preference.Key]
+
+			if seen {
+				preferenceWithValue.Value = value
+			} else {
+				preferenceWithValue.Value = preference.DefaultValue
+			}
+
+			preferenceWithValue.Value = preference.SanitizeValue(preferenceWithValue.Value)
+			allOrgPreferences = append(allOrgPreferences, preferenceWithValue)
+		}
+	}
+	return &allOrgPreferences, nil
+}
+
+// user preference functions
+func GetUserPreference(ctx context.Context, preferenceId string, orgId string, userId string) (*PreferenceKV, *model.ApiError) {
+	// check if the preference key exists
+	preference, seen := preferenceMap[preferenceId]
+	if !seen {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no such preferenceId exists: %s", preferenceId)}
+	}
+
+	preferenceValue := PreferenceKV{
+		PreferenceId:    preferenceId,
+		PreferenceValue: preference.DefaultValue,
+	}
+
+	// check if the preference is enabled at user scope
+	isPreferenceEnabledAtUserScope := preference.IsEnabledForScope(UserAllowedScope)
+	if !isPreferenceEnabledAtUserScope {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("preference is not enabled at user scope: %s", preferenceId)}
+	}
+
+	isPreferenceEnabledAtOrgScope := preference.IsEnabledForScope(OrgAllowedScope)
+	// get the value from the org scope if enabled at org scope
+	if isPreferenceEnabledAtOrgScope {
+		orgPreference := PreferenceKV{}
+
+		query := `SELECT preference_id , preference_value FROM org_preference WHERE preference_id=$1 AND org_id=$2;`
+
+		err := db.Get(&orgPreference, query, preferenceId, orgId)
+
+		// if there is error in getting values and its not an empty rows error return from here
+		if err != nil && err != sql.ErrNoRows {
+			return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in getting org preference values: %s", err.Error())}
+		}
+
+		// if there is no error update the preference value with value from org preference
+		if err == nil {
+			preferenceValue.PreferenceValue = orgPreference.PreferenceValue
+		}
+	}
+
+	// get the value from the user_preference table, if exists return this value else the one calculated in the above step
+	userPreference := PreferenceKV{}
+
+	query := `SELECT preference_id, preference_value FROM user_preference WHERE preference_id=$1 AND user_id=$2;`
+	err := db.Get(&userPreference, query, preferenceId, userId)
+
+	if err != nil && err != sql.ErrNoRows {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in getting user preference values: %s", err.Error())}
+	}
+
+	if err == nil {
+		preferenceValue.PreferenceValue = userPreference.PreferenceValue
+	}
+
+	return &PreferenceKV{
+		PreferenceId:    preferenceValue.PreferenceId,
+		PreferenceValue: preference.SanitizeValue(preferenceValue.PreferenceValue),
+	}, nil
+}
+
+func UpdateUserPreference(ctx context.Context, preferenceId string, preferenceValue interface{}, userId string) (*PreferenceKV, *model.ApiError) {
+	// check if the preference id is valid
+	preference, seen := preferenceMap[preferenceId]
+	if !seen {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no such preferenceId exists: %s", preferenceId)}
+	}
+
+	// check if the preference is enabled at user scope
+	isPreferenceEnabledAtUserScope := preference.IsEnabledForScope(UserAllowedScope)
+	if !isPreferenceEnabledAtUserScope {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("preference is not enabled at user scope: %s", preferenceId)}
+	}
+
+	err := preference.IsValidValue(preferenceValue)
+	if err != nil {
+		return nil, err
+	}
+	// update the user preference values
+	query := `INSERT INTO user_preference(preference_id,preference_value,user_id) VALUES($1,$2,$3)
+	ON CONFLICT(preference_id,user_id) DO
+	UPDATE SET preference_value= $2 WHERE preference_id=$1 AND user_id=$3;`
+
+	_, dberrr := db.Exec(query, preferenceId, preferenceValue, userId)
+
+	if dberrr != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in setting the preference value: %s", dberrr.Error())}
+	}
+
+	return &PreferenceKV{
+		PreferenceId:    preferenceId,
+		PreferenceValue: preferenceValue,
+	}, nil
+}
+
+func GetAllUserPreferences(ctx context.Context, orgId string, userId string) (*[]AllPreferences, *model.ApiError) {
+	allUserPreferences := []AllPreferences{}
+
+	// fetch all the org preference values stored in org_preference table
+	orgPreferenceValues := []PreferenceKV{}
+
+	query := `SELECT preference_id,preference_value FROM org_preference WHERE org_id=$1;`
+	err := db.Select(&orgPreferenceValues, query, orgId)
+
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in getting all org preference values: %s", err)}
+	}
+
+	// create a map of key vs values from the above response
+	preferenceOrgValueMap := map[string]interface{}{}
+
+	for _, preferenceValue := range orgPreferenceValues {
+		preferenceOrgValueMap[preferenceValue.PreferenceId] = preferenceValue.PreferenceValue
+	}
+
+	// fetch all the user preference values stored in user_preference table
+	userPreferenceValues := []PreferenceKV{}
+
+	query = `SELECT preference_id,preference_value FROM user_preference WHERE user_id=$1;`
+	err = db.Select(&userPreferenceValues, query, userId)
+
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in getting all user preference values: %s", err)}
+	}
+
+	// create a map of key vs values from the above response
+	preferenceUserValueMap := map[string]interface{}{}
+
+	for _, preferenceValue := range userPreferenceValues {
+		preferenceUserValueMap[preferenceValue.PreferenceId] = preferenceValue.PreferenceValue
+	}
+
+	// update in the above filtered list wherver value present in the map
+	for _, preference := range preferenceMap {
+		isEnabledForUserScope := preference.IsEnabledForScope(UserAllowedScope)
+
+		if isEnabledForUserScope {
+			preferenceWithValue := AllPreferences{}
+			preferenceWithValue.Key = preference.Key
+			preferenceWithValue.Name = preference.Name
+			preferenceWithValue.Description = preference.Description
+			preferenceWithValue.AllowedScopes = preference.AllowedScopes
+			preferenceWithValue.AllowedValues = preference.AllowedValues
+			preferenceWithValue.DefaultValue = preference.DefaultValue
+			preferenceWithValue.Range = preference.Range
+			preferenceWithValue.ValueType = preference.ValueType
+			preferenceWithValue.IsDiscreteValues = preference.IsDiscreteValues
+			preferenceWithValue.Value = preference.DefaultValue
+
+			isEnabledForOrgScope := preference.IsEnabledForScope(OrgAllowedScope)
+			if isEnabledForOrgScope {
+				value, seen := preferenceOrgValueMap[preference.Key]
+				if seen {
+					preferenceWithValue.Value = value
+				}
+			}
+
+			value, seen := preferenceUserValueMap[preference.Key]
+
+			if seen {
+				preferenceWithValue.Value = value
+			}
+
+			preferenceWithValue.Value = preference.SanitizeValue(preferenceWithValue.Value)
+			allUserPreferences = append(allUserPreferences, preferenceWithValue)
+		}
+	}
+	return &allUserPreferences, nil
+}

pkg/query-service/app/server.go

@@ -14,8 +14,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/alertmanager"
 	"github.com/SigNoz/signoz/pkg/http/middleware"
-	"github.com/SigNoz/signoz/pkg/modules/preference"
-	preferencecore "github.com/SigNoz/signoz/pkg/modules/preference/core"
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
 	"github.com/SigNoz/signoz/pkg/query-service/app/clickhouseReader"
 	"github.com/SigNoz/signoz/pkg/query-service/app/cloudintegrations"
@@ -24,11 +22,11 @@ import (
 	"github.com/SigNoz/signoz/pkg/query-service/app/logparsingpipeline"
 	"github.com/SigNoz/signoz/pkg/query-service/app/opamp"
 	opAmpModel "github.com/SigNoz/signoz/pkg/query-service/app/opamp/model"
+	"github.com/SigNoz/signoz/pkg/query-service/app/preferences"
 	"github.com/SigNoz/signoz/pkg/signoz"
 	"github.com/SigNoz/signoz/pkg/sqlstore"
 	"github.com/SigNoz/signoz/pkg/types"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/types/preferencetypes"
 	"github.com/SigNoz/signoz/pkg/web"
 	"github.com/rs/cors"
 	"github.com/soheilhy/cmux"
@@ -99,6 +97,10 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 		return nil, err
 	}
 
+	if err := preferences.InitDB(serverOptions.SigNoz.SQLStore.SQLxDB()); err != nil {
+		return nil, err
+	}
+
 	if err := dashboards.InitDB(serverOptions.SigNoz.SQLStore); err != nil {
 		return nil, err
 	}
@@ -118,7 +120,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 	}
 
 	clickhouseReader := clickhouseReader.NewReader(
-		serverOptions.SigNoz.SQLStore,
+		serverOptions.SigNoz.SQLStore.SQLxDB(),
 		serverOptions.SigNoz.TelemetryStore.ClickHouseDB(),
 		serverOptions.PromConfigPath,
 		fm,
@@ -149,8 +151,6 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 		c = cache.NewCache(cacheOpts)
 	}
 
-	preference := preference.NewAPI(preferencecore.NewPreference(preferencecore.NewStore(serverOptions.SigNoz.SQLStore), preferencetypes.NewDefaultPreferenceMap()))
-
 	<-readerReady
 	rm, err := makeRulesManager(
 		serverOptions.RuleRepoURL,
@@ -207,7 +207,6 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 		JWT:                           serverOptions.Jwt,
 		AlertmanagerAPI:               alertmanager.NewAPI(serverOptions.SigNoz.Alertmanager),
 		Signoz:                        serverOptions.SigNoz,
-		Preference:                    preference,
 	})
 	if err != nil {
 		return nil, err
@@ -236,10 +235,13 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 
 	s.privateHTTP = privateServer
 
-	opAmpModel.InitDB(serverOptions.SigNoz.SQLStore)
+	_, err = opAmpModel.InitDB(serverOptions.SigNoz.SQLStore.SQLxDB())
+	if err != nil {
+		return nil, err
+	}
 
 	agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{
-		Store: serverOptions.SigNoz.SQLStore,
+		DB: serverOptions.SigNoz.SQLStore.SQLxDB(),
 		AgentFeatures: []agentConf.AgentFeature{
 			logParsingPipelineController,
 		},

pkg/query-service/auth/auth.go

@@ -329,9 +329,6 @@ func CreateResetPasswordToken(ctx context.Context, userId string) (*types.ResetP
 	}
 
 	req := &types.ResetPasswordRequest{
-		Identifiable: types.Identifiable{
-			ID: valuer.GenerateUUID(),
-		},
 		UserID: userId,
 		Token:  token,
 	}

pkg/query-service/dao/sqlite/apdex.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/query-service/model"
 	"github.com/SigNoz/signoz/pkg/types"
-	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/uptrace/bun"
 )
 
@@ -49,7 +48,6 @@ func (mds *ModelDaoSqlite) GetApdexSettings(ctx context.Context, orgID string, s
 func (mds *ModelDaoSqlite) SetApdexSettings(ctx context.Context, orgID string, apdexSettings *types.ApdexSettings) *model.ApiError {
 	// Set the org_id from the parameter since it's required for the foreign key constraint
 	apdexSettings.OrgID = orgID
-	apdexSettings.Identifiable.ID = valuer.GenerateUUID()
 
 	_, err := mds.bundb.NewInsert().
 		Model(apdexSettings).

pkg/query-service/interfaces/interface.go

@@ -23,7 +23,7 @@ type Reader interface {
 	GetServicesList(ctx context.Context) (*[]string, error)
 	GetDependencyGraph(ctx context.Context, query *model.GetServicesParams) (*[]model.ServiceMapDependencyResponseItem, error)
 
-	GetTTL(ctx context.Context, orgID string, ttlParams *model.GetTTLParams) (*model.GetTTLResponseItem, *model.ApiError)
+	GetTTL(ctx context.Context, ttlParams *model.GetTTLParams) (*model.GetTTLResponseItem, *model.ApiError)
 
 	// GetDisks returns a list of disks configured in the underlying DB. It is supported by
 	// clickhouse only.
@@ -45,7 +45,7 @@ type Reader interface {
 	GetFlamegraphSpansForTrace(ctx context.Context, traceID string, req *model.GetFlamegraphSpansForTraceParams) (*model.GetFlamegraphSpansForTraceResponse, *model.ApiError)
 
 	// Setter Interfaces
-	SetTTL(ctx context.Context, orgID string, ttlParams *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError)
+	SetTTL(ctx context.Context, ttlParams *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError)
 
 	FetchTemporality(ctx context.Context, metricNames []string) (map[string]map[v3.Temporality]bool, error)
 	GetMetricAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest, skipDotNames bool, skipSignozMetrics bool) (*v3.AggregateAttributeResponse, error)

pkg/query-service/model/featureSet.go

@@ -24,6 +24,7 @@ const AlertChannelOpsgenie = "ALERT_CHANNEL_OPSGENIE"
 const AlertChannelEmail = "ALERT_CHANNEL_EMAIL"
 const AnomalyDetection = "ANOMALY_DETECTION"
 const HostsInfraMonitoring = "HOSTS_INFRA_MONITORING"
+const TraceFunnels = "TRACE_FUNNELS"
 
 var BasicPlan = FeatureSet{
 	Feature{
@@ -124,4 +125,11 @@ var BasicPlan = FeatureSet{
 		UsageLimit: -1,
 		Route:      "",
 	},
+	Feature{
+		Name:       TraceFunnels,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
 }

pkg/query-service/telemetry/telemetry.go

@@ -317,10 +317,9 @@ func createTelemetry() {
 
 		getLogsInfoInLastHeartBeatInterval, _ := telemetry.reader.GetLogsInfoInLastHeartBeatInterval(ctx, HEART_BEAT_DURATION)
 
-		// TODO update this post bootstrap decision
-		traceTTL, _ := telemetry.reader.GetTTL(ctx, "", &model.GetTTLParams{Type: constants.TraceTTL})
-		metricsTTL, _ := telemetry.reader.GetTTL(ctx, "", &model.GetTTLParams{Type: constants.MetricsTTL})
-		logsTTL, _ := telemetry.reader.GetTTL(ctx, "", &model.GetTTLParams{Type: constants.LogsTTL})
+		traceTTL, _ := telemetry.reader.GetTTL(ctx, &model.GetTTLParams{Type: constants.TraceTTL})
+		metricsTTL, _ := telemetry.reader.GetTTL(ctx, &model.GetTTLParams{Type: constants.MetricsTTL})
+		logsTTL, _ := telemetry.reader.GetTTL(ctx, &model.GetTTLParams{Type: constants.LogsTTL})
 
 		userCount, _ := telemetry.userCountCallback(ctx)
 
@@ -741,7 +740,7 @@ func (a *Telemetry) SendEvent(event string, data map[string]interface{}, userEma
 
 	// zap.L().Info(data)
 	properties := analytics.NewProperties()
-	properties.Set("version", version.Info.Version)
+	properties.Set("version", version.Info.Version())
 	properties.Set("deploymentType", getDeploymentType())
 	properties.Set("companyDomain", a.getCompanyDomain())
 

pkg/query-service/tests/integration/filter_suggestions_test.go

@@ -293,7 +293,7 @@ func NewFilterSuggestionsTestBed(t *testing.T) *FilterSuggestionsTestBed {
 	testDB := utils.NewQueryServiceDBForTests(t)
 
 	fm := featureManager.StartManager()
-	reader, mockClickhouse := NewMockClickhouseReader(t, testDB, fm)
+	reader, mockClickhouse := NewMockClickhouseReader(t, testDB.SQLxDB(), fm)
 	mockClickhouse.MatchExpectationsInOrder(false)
 
 	apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{

pkg/query-service/tests/integration/logparsingpipeline_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/query-service/app/integrations"
 	"github.com/SigNoz/signoz/pkg/query-service/app/logparsingpipeline"
 	"github.com/SigNoz/signoz/pkg/query-service/app/opamp"
-	"github.com/SigNoz/signoz/pkg/query-service/app/opamp/model"
+	opampModel "github.com/SigNoz/signoz/pkg/query-service/app/opamp/model"
 	"github.com/SigNoz/signoz/pkg/query-service/constants"
 	"github.com/SigNoz/signoz/pkg/query-service/dao"
 	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
@@ -37,9 +37,6 @@ func TestLogPipelinesLifecycle(t *testing.T) {
 	testbed := NewLogPipelinesTestBed(t, nil)
 	require := require.New(t)
 
-	orgID, err := utils.GetTestOrgId(testbed.sqlStore)
-	require.Nil(err)
-
 	getPipelinesResp := testbed.GetPipelinesFromQS()
 	require.Equal(
 		0, len(getPipelinesResp.Pipelines),
@@ -110,7 +107,7 @@ func TestLogPipelinesLifecycle(t *testing.T) {
 		t, postablePipelines, createPipelinesResp,
 	)
 	testbed.assertPipelinesSentToOpampClient(createPipelinesResp.Pipelines)
-	testbed.assertNewAgentGetsPipelinesOnConnection(orgID, createPipelinesResp.Pipelines)
+	testbed.assertNewAgentGetsPipelinesOnConnection(createPipelinesResp.Pipelines)
 
 	// Should be able to get the configured pipelines.
 	getPipelinesResp = testbed.GetPipelinesFromQS()
@@ -124,7 +121,7 @@ func TestLogPipelinesLifecycle(t *testing.T) {
 		"pipelines config history should not be empty after 1st configuration",
 	)
 	require.Equal(
-		types.DeployInitiated, getPipelinesResp.History[0].DeployStatus,
+		agentConf.DeployInitiated, getPipelinesResp.History[0].DeployStatus,
 		"pipelines deployment should be in progress after 1st configuration",
 	)
 
@@ -136,7 +133,7 @@ func TestLogPipelinesLifecycle(t *testing.T) {
 		t, postablePipelines, getPipelinesResp,
 	)
 	require.Equal(
-		types.Deployed,
+		agentConf.Deployed,
 		getPipelinesResp.History[0].DeployStatus,
 		"pipeline deployment should be complete after acknowledgment from opamp client",
 	)
@@ -148,7 +145,7 @@ func TestLogPipelinesLifecycle(t *testing.T) {
 		t, postablePipelines, updatePipelinesResp,
 	)
 	testbed.assertPipelinesSentToOpampClient(updatePipelinesResp.Pipelines)
-	testbed.assertNewAgentGetsPipelinesOnConnection(orgID, updatePipelinesResp.Pipelines)
+	testbed.assertNewAgentGetsPipelinesOnConnection(updatePipelinesResp.Pipelines)
 
 	getPipelinesResp = testbed.GetPipelinesFromQS()
 	require.Equal(
@@ -156,7 +153,7 @@ func TestLogPipelinesLifecycle(t *testing.T) {
 		"there should be 2 history entries after posting pipelines config for the 2nd time",
 	)
 	require.Equal(
-		types.DeployInitiated, getPipelinesResp.History[0].DeployStatus,
+		agentConf.DeployInitiated, getPipelinesResp.History[0].DeployStatus,
 		"deployment should be in progress for latest pipeline config",
 	)
 
@@ -168,7 +165,7 @@ func TestLogPipelinesLifecycle(t *testing.T) {
 		t, postablePipelines, getPipelinesResp,
 	)
 	require.Equal(
-		types.Deployed,
+		agentConf.Deployed,
 		getPipelinesResp.History[0].DeployStatus,
 		"deployment for latest pipeline config should be complete after acknowledgment from opamp client",
 	)
@@ -222,7 +219,7 @@ func TestLogPipelinesHistory(t *testing.T) {
 	testbed.PostPipelinesToQS(postablePipelines)
 	getPipelinesResp = testbed.GetPipelinesFromQS()
 	require.Equal(1, len(getPipelinesResp.History))
-	require.Equal(types.DeployInitiated, getPipelinesResp.History[0].DeployStatus)
+	require.Equal(agentConf.DeployInitiated, getPipelinesResp.History[0].DeployStatus)
 
 	postablePipelines.Pipelines[0].Config = append(
 		postablePipelines.Pipelines[0].Config,
@@ -241,8 +238,8 @@ func TestLogPipelinesHistory(t *testing.T) {
 	getPipelinesResp = testbed.GetPipelinesFromQS()
 
 	require.Equal(2, len(getPipelinesResp.History))
-	require.Equal(types.DeployInitiated, getPipelinesResp.History[0].DeployStatus)
-	require.Equal(types.DeployStatusUnknown, getPipelinesResp.History[1].DeployStatus)
+	require.Equal(agentConf.DeployInitiated, getPipelinesResp.History[0].DeployStatus)
+	require.Equal(agentConf.DeployStatusUnknown, getPipelinesResp.History[1].DeployStatus)
 }
 
 func TestLogPipelinesValidation(t *testing.T) {
@@ -450,22 +447,24 @@ type LogPipelinesTestBed struct {
 	agentConfMgr    *agentConf.Manager
 	opampServer     *opamp.Server
 	opampClientConn *opamp.MockOpAmpConnection
-	sqlStore        sqlstore.SQLStore
 }
 
 // testDB can be injected for sharing a DB across multiple integration testbeds.
-func NewTestbedWithoutOpamp(t *testing.T, store sqlstore.SQLStore) *LogPipelinesTestBed {
-	if store == nil {
-		store = utils.NewQueryServiceDBForTests(t)
+func NewTestbedWithoutOpamp(t *testing.T, sqlStore sqlstore.SQLStore) *LogPipelinesTestBed {
+	if sqlStore == nil {
+		sqlStore = utils.NewQueryServiceDBForTests(t)
 	}
 
-	ic, err := integrations.NewController(store)
+	// create test org
+	// utils.CreateTestOrg(t, sqlStore)
+
+	ic, err := integrations.NewController(sqlStore)
 	if err != nil {
 		t.Fatalf("could not create integrations controller: %v", err)
 	}
 
 	controller, err := logparsingpipeline.NewLogParsingPipelinesController(
-		store, ic.GetPipelinesForInstalledIntegrations,
+		sqlStore, ic.GetPipelinesForInstalledIntegrations,
 	)
 	if err != nil {
 		t.Fatalf("could not create a logparsingpipelines controller: %v", err)
@@ -486,11 +485,11 @@ func NewTestbedWithoutOpamp(t *testing.T, store sqlstore.SQLStore) *LogPipelines
 	}
 
 	// Mock an available opamp agent
-	// testDB, err := opampModel.InitDB(sqlStore.SQLxDB())
+	testDB, err := opampModel.InitDB(sqlStore.SQLxDB())
 	require.Nil(t, err, "failed to init opamp model")
 
 	agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{
-		Store: store,
+		DB: testDB,
 		AgentFeatures: []agentConf.AgentFeature{
 			apiHandler.LogsParsingPipelineController,
 		}})
@@ -501,20 +500,14 @@ func NewTestbedWithoutOpamp(t *testing.T, store sqlstore.SQLStore) *LogPipelines
 		testUser:     user,
 		apiHandler:   apiHandler,
 		agentConfMgr: agentConfMgr,
-		sqlStore:     store,
 	}
 }
 
 func NewLogPipelinesTestBed(t *testing.T, testDB sqlstore.SQLStore) *LogPipelinesTestBed {
 	testbed := NewTestbedWithoutOpamp(t, testDB)
 
-	orgID, err := utils.GetTestOrgId(testbed.sqlStore)
-	require.Nil(t, err)
-
-	model.InitDB(testbed.sqlStore)
-
 	opampServer := opamp.InitializeServer(nil, testbed.agentConfMgr)
-	err = opampServer.Start(opamp.GetAvailableLocalAddress())
+	err := opampServer.Start(opamp.GetAvailableLocalAddress())
 	require.Nil(t, err, "failed to start opamp server")
 
 	t.Cleanup(func() {
@@ -529,16 +522,6 @@ func NewLogPipelinesTestBed(t *testing.T, testDB sqlstore.SQLStore) *LogPipeline
 			EffectiveConfig: &protobufs.EffectiveConfig{
 				ConfigMap: newInitialAgentConfigMap(),
 			},
-			AgentDescription: &protobufs.AgentDescription{
-				IdentifyingAttributes: []*protobufs.KeyValue{
-					{
-						Key: "orgId",
-						Value: &protobufs.AnyValue{
-							Value: &protobufs.AnyValue_StringValue{StringValue: orgID},
-						},
-					},
-				},
-			},
 		},
 	)
 
@@ -745,7 +728,6 @@ func (tb *LogPipelinesTestBed) simulateOpampClientAcknowledgementForLatestConfig
 }
 
 func (tb *LogPipelinesTestBed) assertNewAgentGetsPipelinesOnConnection(
-	orgID string,
 	pipelines []pipelinetypes.GettablePipeline,
 ) {
 	newAgentConn := &opamp.MockOpAmpConnection{}
@@ -756,16 +738,6 @@ func (tb *LogPipelinesTestBed) assertNewAgentGetsPipelinesOnConnection(
 			EffectiveConfig: &protobufs.EffectiveConfig{
 				ConfigMap: newInitialAgentConfigMap(),
 			},
-			AgentDescription: &protobufs.AgentDescription{
-				IdentifyingAttributes: []*protobufs.KeyValue{
-					{
-						Key: "orgId",
-						Value: &protobufs.AnyValue{
-							Value: &protobufs.AnyValue_StringValue{StringValue: orgID},
-						},
-					},
-				},
-			},
 		},
 	)
 	latestMsgFromServer := newAgentConn.LatestMsgFromServer()

pkg/query-service/tests/integration/signoz_cloud_integrations_test.go

@@ -355,7 +355,7 @@ func NewCloudIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *CloudI
 	}
 
 	fm := featureManager.StartManager()
-	reader, mockClickhouse := NewMockClickhouseReader(t, testDB, fm)
+	reader, mockClickhouse := NewMockClickhouseReader(t, testDB.SQLxDB(), fm)
 	mockClickhouse.MatchExpectationsInOrder(false)
 
 	apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{

pkg/query-service/tests/integration/signoz_integrations_test.go

@@ -32,9 +32,6 @@ func TestSignozIntegrationLifeCycle(t *testing.T) {
 	require := require.New(t)
 	testbed := NewIntegrationsTestBed(t, nil)
 
-	merr := utils.CreateTestOrg(t, testbed.store)
-	require.NoError(merr)
-
 	installedResp := testbed.GetInstalledIntegrationsFromQS()
 	require.Equal(
 		len(installedResp.Integrations), 0,
@@ -118,11 +115,6 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) {
 	require := require.New(t)
 
 	testDB := utils.NewQueryServiceDBForTests(t)
-	utils.CreateTestOrg(t, testDB)
-
-	orgID, err := utils.GetTestOrgId(testDB)
-	require.Nil(err)
-
 	integrationsTB := NewIntegrationsTestBed(t, testDB)
 	pipelinesTB := NewLogPipelinesTestBed(t, testDB)
 
@@ -180,7 +172,7 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) {
 	require.Equal(testIntegration.Id, *integrations.IntegrationIdForPipeline(lastPipeline))
 
 	pipelinesTB.assertPipelinesSentToOpampClient(getPipelinesResp.Pipelines)
-	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(orgID, getPipelinesResp.Pipelines)
+	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(getPipelinesResp.Pipelines)
 
 	// After saving a user created pipeline, pipelines response should include
 	// both user created pipelines and pipelines for installed integrations.
@@ -225,7 +217,7 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) {
 	getPipelinesResp = pipelinesTB.GetPipelinesFromQS()
 	require.Equal(1+len(testIntegrationPipelines), len(getPipelinesResp.Pipelines))
 	pipelinesTB.assertPipelinesSentToOpampClient(getPipelinesResp.Pipelines)
-	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(orgID, getPipelinesResp.Pipelines)
+	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(getPipelinesResp.Pipelines)
 
 	// Reordering integration pipelines should be possible.
 	postable := postableFromPipelines(getPipelinesResp.Pipelines)
@@ -242,7 +234,7 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) {
 	require.Equal(testIntegration.Id, *integrations.IntegrationIdForPipeline(firstPipeline))
 
 	pipelinesTB.assertPipelinesSentToOpampClient(getPipelinesResp.Pipelines)
-	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(orgID, getPipelinesResp.Pipelines)
+	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(getPipelinesResp.Pipelines)
 
 	// enabling/disabling integration pipelines should be possible.
 	require.True(firstPipeline.Enabled)
@@ -260,7 +252,7 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) {
 	require.False(firstPipeline.Enabled)
 
 	pipelinesTB.assertPipelinesSentToOpampClient(getPipelinesResp.Pipelines)
-	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(orgID, getPipelinesResp.Pipelines)
+	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(getPipelinesResp.Pipelines)
 
 	// should not be able to edit integrations pipeline.
 	require.Greater(len(postable.Pipelines[0].Config), 0)
@@ -299,7 +291,7 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) {
 		"Pipelines for uninstalled integrations should get removed from pipelines list",
 	)
 	pipelinesTB.assertPipelinesSentToOpampClient(getPipelinesResp.Pipelines)
-	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(orgID, getPipelinesResp.Pipelines)
+	pipelinesTB.assertNewAgentGetsPipelinesOnConnection(getPipelinesResp.Pipelines)
 }
 
 func TestDashboardsForInstalledIntegrationDashboards(t *testing.T) {
@@ -378,7 +370,6 @@ type IntegrationsTestBed struct {
 	testUser       *types.User
 	qsHttpHandler  http.Handler
 	mockClickhouse mockhouse.ClickConnMockCommon
-	store          sqlstore.SQLStore
 }
 
 func (tb *IntegrationsTestBed) GetAvailableIntegrationsFromQS() *integrations.IntegrationsListResponse {
@@ -566,7 +557,7 @@ func NewIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *Integration
 	}
 
 	fm := featureManager.StartManager()
-	reader, mockClickhouse := NewMockClickhouseReader(t, testDB, fm)
+	reader, mockClickhouse := NewMockClickhouseReader(t, testDB.SQLxDB(), fm)
 	mockClickhouse.MatchExpectationsInOrder(false)
 
 	cloudIntegrationsController, err := cloudintegrations.NewController(testDB)
@@ -602,7 +593,6 @@ func NewIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *Integration
 		testUser:       user,
 		qsHttpHandler:  router,
 		mockClickhouse: mockClickhouse,
-		store:          testDB,
 	}
 }