Merge branch 'main' into feat/update_hosts_list_error_messaging

frontend/src/api/infraMonitoring/getHostLists.ts

@@ -50,6 +50,7 @@ export interface HostListResponse {
 		total: number;
 		sentAnyHostMetricsData: boolean;
 		isSendingK8SAgentMetrics: boolean;
+		endTimeBeforeRetention: boolean;
 	};
 }
 

frontend/src/container/InfraMonitoringHosts/HostsListTable.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useMemo } from 'react';
+import React, { useCallback, useMemo } from 'react';
 import { LoadingOutlined } from '@ant-design/icons';
 import {
 	Skeleton,
@@ -14,12 +14,93 @@ import { InfraMonitoringEvents } from 'constants/events';
 
 import HostsEmptyOrIncorrectMetrics from './HostsEmptyOrIncorrectMetrics';
 import {
+	EmptyOrLoadingViewProps,
 	formatDataForTable,
 	getHostsListColumns,
 	HostRowData,
 	HostsListTableProps,
 } from './utils';
 
+function EmptyOrLoadingView(
+	viewState: EmptyOrLoadingViewProps,
+): React.ReactNode {
+	const { isError, errorMessage } = viewState;
+	if (isError) {
+		return <Typography>{errorMessage || 'Something went wrong'}</Typography>;
+	}
+	if (viewState.showHostsEmptyState) {
+		return (
+			<HostsEmptyOrIncorrectMetrics
+				noData={!viewState.sentAnyHostMetricsData}
+				incorrectData={viewState.isSendingIncorrectK8SAgentMetrics}
+			/>
+		);
+	}
+	if (viewState.showEndTimeBeforeRetentionMessage) {
+		return (
+			<div className="hosts-empty-state-container">
+				<div className="hosts-empty-state-container-content">
+					<img className="eyes-emoji" src="/Images/eyesEmoji.svg" alt="eyes emoji" />
+					<div className="no-hosts-message">
+						<Typography.Title level={5} className="no-hosts-message-title">
+							Queried time range is before earliest host metrics
+						</Typography.Title>
+						<Typography.Text className="no-hosts-message-text">
+							Your requested end time is earlier than the earliest detected time of
+							host metrics data, please adjust your end time.
+						</Typography.Text>
+					</div>
+				</div>
+			</div>
+		);
+	}
+	if (viewState.showNoRecordsInSelectedTimeRangeMessage) {
+		return (
+			<div className="no-filtered-hosts-message-container">
+				<div className="no-filtered-hosts-message-content">
+					<img
+						src="/Icons/emptyState.svg"
+						alt="thinking-emoji"
+						className="empty-state-svg"
+					/>
+					<Typography.Title level={5} className="no-filtered-hosts-title">
+						No host metrics found
+					</Typography.Title>
+					<Typography.Text className="no-filtered-hosts-message">
+						No host metrics in the selected time range and filters. Please adjust your
+						time range or filters.
+					</Typography.Text>
+				</div>
+			</div>
+		);
+	}
+	if (viewState.showTableLoadingState) {
+		return (
+			<div className="hosts-list-loading-state">
+				<Skeleton.Input
+					className="hosts-list-loading-state-item"
+					size="large"
+					block
+					active
+				/>
+				<Skeleton.Input
+					className="hosts-list-loading-state-item"
+					size="large"
+					block
+					active
+				/>
+				<Skeleton.Input
+					className="hosts-list-loading-state-item"
+					size="large"
+					block
+					active
+				/>
+			</div>
+		);
+	}
+	return null;
+}
+
 export default function HostsListTable({
 	isLoading,
 	isFetching,
@@ -46,6 +127,11 @@ export default function HostsListTable({
 		[data],
 	);
 
+	const endTimeBeforeRetention = useMemo(
+		() => data?.payload?.data?.endTimeBeforeRetention || false,
+		[data],
+	);
+
 	const formattedHostMetricsData = useMemo(
 		() => formatDataForTable(hostMetricsData),
 		[hostMetricsData],
@@ -84,12 +170,6 @@ export default function HostsListTable({
 		});
 	};
 
-	const showNoFilteredHostsMessage =
-		!isFetching &&
-		!isLoading &&
-		formattedHostMetricsData.length === 0 &&
-		filters.items.length > 0;
-
 	const showHostsEmptyState =
 		!isFetching &&
 		!isLoading &&
@@ -97,63 +177,36 @@ export default function HostsListTable({
 		(!sentAnyHostMetricsData || isSendingIncorrectK8SAgentMetrics) &&
 		!filters.items.length;
 
+	const showEndTimeBeforeRetentionMessage =
+		!isFetching &&
+		!isLoading &&
+		formattedHostMetricsData.length === 0 &&
+		endTimeBeforeRetention &&
+		!filters.items.length;
+
+	const showNoRecordsInSelectedTimeRangeMessage =
+		!isFetching &&
+		!isLoading &&
+		formattedHostMetricsData.length === 0 &&
+		!showEndTimeBeforeRetentionMessage &&
+		!showHostsEmptyState;
+
 	const showTableLoadingState =
 		(isLoading || isFetching) && formattedHostMetricsData.length === 0;
 
-	if (isError) {
-		return <Typography>{data?.error || 'Something went wrong'}</Typography>;
-	}
+	const emptyOrLoadingView = EmptyOrLoadingView({
+		isError,
+		errorMessage: data?.error ?? '',
+		showHostsEmptyState,
+		sentAnyHostMetricsData,
+		isSendingIncorrectK8SAgentMetrics,
+		showEndTimeBeforeRetentionMessage,
+		showNoRecordsInSelectedTimeRangeMessage,
+		showTableLoadingState,
+	});
 
-	if (showHostsEmptyState) {
-		return (
-			<HostsEmptyOrIncorrectMetrics
-				noData={!sentAnyHostMetricsData}
-				incorrectData={isSendingIncorrectK8SAgentMetrics}
-			/>
-		);
-	}
-
-	if (showNoFilteredHostsMessage) {
-		return (
-			<div className="no-filtered-hosts-message-container">
-				<div className="no-filtered-hosts-message-content">
-					<img
-						src="/Icons/emptyState.svg"
-						alt="thinking-emoji"
-						className="empty-state-svg"
-					/>
-
-					<Typography.Text className="no-filtered-hosts-message">
-						This query had no results. Edit your query and try again!
-					</Typography.Text>
-				</div>
-			</div>
-		);
-	}
-
-	if (showTableLoadingState) {
-		return (
-			<div className="hosts-list-loading-state">
-				<Skeleton.Input
-					className="hosts-list-loading-state-item"
-					size="large"
-					block
-					active
-				/>
-				<Skeleton.Input
-					className="hosts-list-loading-state-item"
-					size="large"
-					block
-					active
-				/>
-				<Skeleton.Input
-					className="hosts-list-loading-state-item"
-					size="large"
-					block
-					active
-				/>
-			</div>
-		);
+	if (emptyOrLoadingView) {
+		return <>{emptyOrLoadingView}</>;
 	}
 
 	return (

frontend/src/container/InfraMonitoringHosts/__tests__/HostsListTable.test.tsx

@@ -1,12 +1,16 @@
 /* eslint-disable react/jsx-props-no-spreading */
 import { render, screen } from '@testing-library/react';
+import { HostData, HostListResponse } from 'api/infraMonitoring/getHostLists';
+import { ErrorResponse, SuccessResponse } from 'types/api';
+import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
 
 import HostsListTable from '../HostsListTable';
+import { HostsListTableProps } from '../utils';
 
 const EMPTY_STATE_CONTAINER_CLASS = '.hosts-empty-state-container';
 
-describe('HostsListTable', () => {
-	const mockHost = {
+const createMockHost = (): HostData =>
+	({
 		hostName: 'test-host-1',
 		active: true,
 		cpu: 0.75,
@@ -14,20 +18,46 @@ describe('HostsListTable', () => {
 		wait: 0.03,
 		load15: 1.5,
 		os: 'linux',
-	};
+		cpuTimeSeries: { labels: {}, labelsArray: [], values: [] },
+		memoryTimeSeries: { labels: {}, labelsArray: [], values: [] },
+		waitTimeSeries: { labels: {}, labelsArray: [], values: [] },
+		load15TimeSeries: { labels: {}, labelsArray: [], values: [] },
+	} as HostData);
 
-	const mockTableData = {
+const createMockTableData = (
+	overrides: Partial<HostListResponse['data']> = {},
+): SuccessResponse<HostListResponse> => {
+	const mockHost = createMockHost();
+	return {
+		statusCode: 200,
+		message: 'Success',
+		error: null,
 		payload: {
+			status: 'success',
 			data: {
-				hosts: [mockHost],
+				type: 'list',
+				records: [mockHost],
+				groups: null,
+				total: 1,
+				sentAnyHostMetricsData: true,
+				isSendingK8SAgentMetrics: false,
+				endTimeBeforeRetention: false,
+				...overrides,
 			},
 		},
 	};
+};
+
+describe('HostsListTable', () => {
+	const mockHost = createMockHost();
+	const mockTableData = createMockTableData();
+
 	const mockOnHostClick = jest.fn();
 	const mockSetCurrentPage = jest.fn();
 	const mockSetOrderBy = jest.fn();
 	const mockSetPageSize = jest.fn();
-	const mockProps = {
+
+	const mockProps: HostsListTableProps = {
 		isLoading: false,
 		isError: false,
 		isFetching: false,
@@ -43,7 +73,7 @@ describe('HostsListTable', () => {
 		pageSize: 10,
 		setOrderBy: mockSetOrderBy,
 		setPageSize: mockSetPageSize,
-	} as any;
+	};
 
 	it('renders loading state if isLoading is true and tableData is empty', () => {
 		const { container } = render(
@@ -51,7 +81,7 @@ describe('HostsListTable', () => {
 				{...mockProps}
 				isLoading
 				hostMetricsData={[]}
-				tableData={{ payload: { data: { hosts: [] } } }}
+				tableData={createMockTableData({ records: [] })}
 			/>,
 		);
 		expect(container.querySelector('.hosts-list-loading-state')).toBeTruthy();
@@ -63,7 +93,7 @@ describe('HostsListTable', () => {
 				{...mockProps}
 				isFetching
 				hostMetricsData={[]}
-				tableData={{ payload: { data: { hosts: [] } } }}
+				tableData={createMockTableData({ records: [] })}
 			/>,
 		);
 		expect(container.querySelector('.hosts-list-loading-state')).toBeTruthy();
@@ -74,19 +104,56 @@ describe('HostsListTable', () => {
 		expect(screen.getByText('Something went wrong')).toBeTruthy();
 	});
 
+	it('renders "Something went wrong" fallback when isError is true and error message is empty', () => {
+		const tableDataWithEmptyError: ErrorResponse = {
+			statusCode: 500,
+			payload: null,
+			error: '',
+			message: null,
+		};
+		render(
+			<HostsListTable
+				{...mockProps}
+				isError
+				hostMetricsData={[]}
+				tableData={tableDataWithEmptyError}
+			/>,
+		);
+		expect(screen.getByText('Something went wrong')).toBeInTheDocument();
+	});
+
+	it('renders custom error message when isError is true and error message is provided', () => {
+		const customErrorMessage = 'Failed to fetch host metrics';
+		const tableDataWithError: ErrorResponse = {
+			statusCode: 500,
+			payload: null,
+			error: customErrorMessage,
+			message: null,
+		};
+		render(
+			<HostsListTable
+				{...mockProps}
+				isError
+				hostMetricsData={[]}
+				tableData={tableDataWithError}
+			/>,
+		);
+		expect(screen.getByText(customErrorMessage)).toBeInTheDocument();
+	});
+
 	it('renders empty state if no hosts are found', () => {
 		const { container } = render(
 			<HostsListTable
 				{...mockProps}
 				hostMetricsData={[]}
-				tableData={{
-					payload: {
-						data: { hosts: [] },
-					},
-				}}
+				tableData={createMockTableData({
+					records: [],
+				})}
 			/>,
 		);
-		expect(container.querySelector(EMPTY_STATE_CONTAINER_CLASS)).toBeTruthy();
+		expect(
+			container.querySelector('.no-filtered-hosts-message-container'),
+		).toBeTruthy();
 	});
 
 	it('renders empty state if sentAnyHostMetricsData is false', () => {
@@ -94,58 +161,114 @@ describe('HostsListTable', () => {
 			<HostsListTable
 				{...mockProps}
 				hostMetricsData={[]}
-				tableData={{
-					...mockTableData,
-					payload: {
-						...mockTableData.payload,
-						data: {
-							...mockTableData.payload.data,
-							sentAnyHostMetricsData: false,
-							hosts: [],
-						},
-					},
-				}}
+				tableData={createMockTableData({
+					sentAnyHostMetricsData: false,
+					records: [],
+				})}
 			/>,
 		);
 		expect(container.querySelector(EMPTY_STATE_CONTAINER_CLASS)).toBeTruthy();
 	});
 
-	it('renders empty state if isSendingIncorrectK8SAgentMetrics is true', () => {
+	it('renders empty state if isSendingK8SAgentMetrics is true', () => {
 		const { container } = render(
 			<HostsListTable
 				{...mockProps}
 				hostMetricsData={[]}
-				tableData={{
-					...mockTableData,
-					payload: {
-						...mockTableData.payload,
-						data: {
-							...mockTableData.payload.data,
-							isSendingIncorrectK8SAgentMetrics: true,
-							hosts: [],
-						},
-					},
-				}}
+				tableData={createMockTableData({
+					isSendingK8SAgentMetrics: true,
+					records: [],
+				})}
 			/>,
 		);
 		expect(container.querySelector(EMPTY_STATE_CONTAINER_CLASS)).toBeTruthy();
 	});
 
+	it('renders end time before retention message when endTimeBeforeRetention is true', () => {
+		const { container } = render(
+			<HostsListTable
+				{...mockProps}
+				hostMetricsData={[]}
+				tableData={createMockTableData({
+					sentAnyHostMetricsData: true,
+					isSendingK8SAgentMetrics: false,
+					endTimeBeforeRetention: true,
+					records: [],
+				})}
+			/>,
+		);
+		expect(container.querySelector(EMPTY_STATE_CONTAINER_CLASS)).toBeTruthy();
+		expect(
+			screen.getByText(
+				/Your requested end time is earlier than the earliest detected time of host metrics data, please adjust your end time\./,
+			),
+		).toBeInTheDocument();
+	});
+
+	it('renders no records message when noRecordsInSelectedTimeRangeAndFilters is true', () => {
+		const { container } = render(
+			<HostsListTable
+				{...mockProps}
+				hostMetricsData={[]}
+				tableData={createMockTableData({
+					sentAnyHostMetricsData: true,
+					isSendingK8SAgentMetrics: false,
+					records: [],
+				})}
+			/>,
+		);
+		expect(
+			container.querySelector('.no-filtered-hosts-message-container'),
+		).toBeTruthy();
+		expect(
+			screen.getByText(/No host metrics in the selected time range and filters/),
+		).toBeInTheDocument();
+	});
+
+	it('renders no filtered hosts message when filters are present and no hosts are found', () => {
+		const { container } = render(
+			<HostsListTable
+				{...mockProps}
+				hostMetricsData={[]}
+				filters={{
+					items: [
+						{
+							id: 'host_name',
+							key: {
+								key: 'host_name',
+								dataType: DataTypes.String,
+								type: 'tag',
+								isIndexed: true,
+							},
+							op: '=',
+							value: 'unknown',
+						},
+					],
+					op: 'AND',
+				}}
+				tableData={createMockTableData({
+					sentAnyHostMetricsData: true,
+					isSendingK8SAgentMetrics: false,
+					records: [],
+				})}
+			/>,
+		);
+		expect(container.querySelector('.no-filtered-hosts-message')).toBeTruthy();
+		expect(
+			screen.getByText(
+				/No host metrics in the selected time range and filters\. Please adjust your time range or filters\./,
+			),
+		).toBeInTheDocument();
+	});
+
 	it('renders table data', () => {
 		const { container } = render(
 			<HostsListTable
 				{...mockProps}
-				tableData={{
-					...mockTableData,
-					payload: {
-						...mockTableData.payload,
-						data: {
-							...mockTableData.payload.data,
-							isSendingIncorrectK8SAgentMetrics: false,
-							sentAnyHostMetricsData: true,
-						},
-					},
-				}}
+				tableData={createMockTableData({
+					isSendingK8SAgentMetrics: false,
+					sentAnyHostMetricsData: true,
+				})}
 			/>,
 		);
 		expect(container.querySelector('.hosts-list-table')).toBeTruthy();

frontend/src/container/InfraMonitoringHosts/utils.tsx

@@ -107,6 +107,17 @@ export interface HostsListTableProps {
 	setPageSize: (pageSize: number) => void;
 }
 
+export interface EmptyOrLoadingViewProps {
+	isError: boolean;
+	errorMessage: string;
+	showHostsEmptyState: boolean;
+	sentAnyHostMetricsData: boolean;
+	isSendingIncorrectK8SAgentMetrics: boolean;
+	showEndTimeBeforeRetentionMessage: boolean;
+	showNoRecordsInSelectedTimeRangeMessage: boolean;
+	showTableLoadingState: boolean;
+}
+
 export const getHostListsQuery = (): HostListPayload => ({
 	filters: {
 		items: [],

pkg/query-service/app/clickhouseReader/reader.go

@@ -4308,6 +4308,28 @@ func (r *ClickHouseReader) GetListResultV3(ctx context.Context, query string) ([
 
 }
 
+// GetHostMetricsExistenceAndEarliestTime returns (count, minFirstReportedUnixMilli, error) for the given host metric names
+// from distributed_metadata. When count is 0, minFirstReportedUnixMilli is 0.
+func (r *ClickHouseReader) GetHostMetricsExistenceAndEarliestTime(ctx context.Context, metricNames []string) (uint64, uint64, error) {
+	if len(metricNames) == 0 {
+		return 0, 0, nil
+	}
+
+	query := fmt.Sprintf(
+		`SELECT count(*) AS cnt, min(first_reported_unix_milli) AS min_first_reported
+		FROM %s.%s
+		WHERE metric_name IN @metric_names`,
+		constants.SIGNOZ_METRIC_DBNAME, constants.SIGNOZ_METADATA_TABLENAME)
+
+	var count, minFirstReported uint64
+	err := r.db.QueryRow(ctx, query, clickhouse.Named("metric_names", metricNames)).Scan(&count, &minFirstReported)
+	if err != nil {
+		zap.L().Error("error getting host metrics existence and earliest time", zap.Error(err))
+		return 0, 0, err
+	}
+	return count, minFirstReported, nil
+}
+
 func getPersonalisedError(err error) error {
 	if err == nil {
 		return nil

pkg/query-service/app/inframetrics/common.go

@@ -10,6 +10,7 @@ import (
 )
 
 var dotMetricMap = map[string]string{
+	"system_filesystem_usage":               "system.filesystem.usage",
 	"system_cpu_time":                       "system.cpu.time",
 	"system_memory_usage":                   "system.memory.usage",
 	"system_cpu_load_average_15m":           "system.cpu.load_average.15m",

pkg/query-service/app/inframetrics/hosts.go

@@ -67,10 +67,11 @@ var (
 		GetDotMetrics("os_type"),
 	}
 	metricNamesForHosts = map[string]string{
-		"cpu":    GetDotMetrics("system_cpu_time"),
-		"memory": GetDotMetrics("system_memory_usage"),
-		"load15": GetDotMetrics("system_cpu_load_average_15m"),
-		"wait":   GetDotMetrics("system_cpu_time"),
+		"filesystem": GetDotMetrics("system_filesystem_usage"),
+		"cpu":        GetDotMetrics("system_cpu_time"),
+		"memory":     GetDotMetrics("system_memory_usage"),
+		"load15":     GetDotMetrics("system_cpu_load_average_15m"),
+		"wait":       GetDotMetrics("system_cpu_time"),
 	}
 )
 
@@ -316,24 +317,15 @@ func (h *HostsRepo) getTopHostGroups(ctx context.Context, orgID valuer.UUID, req
 	return topHostGroups, allHostGroups, nil
 }
 
-func (h *HostsRepo) DidSendHostMetricsData(ctx context.Context, req model.HostListRequest) (bool, error) {
-
+// GetHostMetricsExistenceAndEarliestTime returns (count, minFirstReportedUnixMilli, error) for host metrics
+// in distributed_metadata. Uses metricNamesForHosts plus system.filesystem.usage.
+func (h *HostsRepo) GetHostMetricsExistenceAndEarliestTime(ctx context.Context, req model.HostListRequest) (uint64, uint64, error) {
 	names := []string{}
 	for _, metricName := range metricNamesForHosts {
 		names = append(names, metricName)
 	}
 
-	namesStr := "'" + strings.Join(names, "','") + "'"
-
-	query := fmt.Sprintf("SELECT count() FROM %s.%s WHERE metric_name IN (%s)",
-		constants.SIGNOZ_METRIC_DBNAME, constants.SIGNOZ_TIMESERIES_v4_1DAY_TABLENAME, namesStr)
-
-	count, err := h.reader.GetCountOfThings(ctx, query)
-	if err != nil {
-		return false, err
-	}
-
-	return count > 0, nil
+	return h.reader.GetHostMetricsExistenceAndEarliestTime(ctx, names)
 }
 
 func (h *HostsRepo) IsSendingK8SAgentMetrics(ctx context.Context, req model.HostListRequest) ([]string, []string, error) {
@@ -412,8 +404,25 @@ func (h *HostsRepo) GetHostList(ctx context.Context, orgID valuer.UUID, req mode
 		resp.ClusterNames = clusterNames
 		resp.NodeNames = nodeNames
 	}
-	if sentAnyHostMetricsData, err := h.DidSendHostMetricsData(ctx, req); err == nil {
-		resp.SentAnyHostMetricsData = sentAnyHostMetricsData
+
+	// 1. Check if any host metrics exist and get earliest retention time
+	// if no hosts metrics exist, that means we should show the onboarding guide on UI, and return early.
+	// 2. If host metrics exist, but req.End is earlier than the earliest time of host metrics as read from
+	// metadata table, then we should convey the same to the user and return early
+	if count, minFirstReportedUnixMilli, err := h.GetHostMetricsExistenceAndEarliestTime(ctx, req); err == nil {
+		if count == 0 {
+			resp.SentAnyHostMetricsData = false
+			resp.Records = []model.HostListRecord{}
+			resp.Total = 0
+			return resp, nil
+		}
+		resp.SentAnyHostMetricsData = true
+		if req.End < int64(minFirstReportedUnixMilli) {
+			resp.EndTimeBeforeRetention = true
+			resp.Records = []model.HostListRecord{}
+			resp.Total = 0
+			return resp, nil
+		}
 	}
 
 	step := int64(math.Max(float64(common.MinAllowedStepInterval(req.Start, req.End)), 60))

pkg/query-service/constants/constants.go

@@ -125,6 +125,8 @@ const (
 	SIGNOZ_TIMESERIES_v4_6HRS_TABLENAME        = "distributed_time_series_v4_6hrs"
 	SIGNOZ_ATTRIBUTES_METADATA_TABLENAME       = "distributed_attributes_metadata"
 	SIGNOZ_ATTRIBUTES_METADATA_LOCAL_TABLENAME = "attributes_metadata"
+	SIGNOZ_METADATA_TABLENAME                  = "distributed_metadata"
+	SIGNOZ_METADATA_LOCAL_TABLENAME            = "metadata"
 )
 
 // alert related constants

pkg/query-service/interfaces/interface.go

@@ -100,6 +100,8 @@ type Reader interface {
 
 	GetCountOfThings(ctx context.Context, query string) (uint64, error)
 
+	GetHostMetricsExistenceAndEarliestTime(ctx context.Context, metricNames []string) (uint64, uint64, error)
+
 	//trace
 	GetTraceFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError)
 	UpdateTraceField(ctx context.Context, field *model.UpdateField) *model.ApiError

pkg/query-service/model/infra.go

@@ -44,6 +44,7 @@ type HostListResponse struct {
 	IsSendingK8SAgentMetrics bool             `json:"isSendingK8SAgentMetrics"`
 	ClusterNames             []string         `json:"clusterNames"`
 	NodeNames                []string         `json:"nodeNames"`
+	EndTimeBeforeRetention   bool             `json:"endTimeBeforeRetention"`
 }
 
 func (r *HostListResponse) SortBy(orderBy *v3.OrderBy) {

pkg/signoz/provider.go

@@ -169,7 +169,6 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewAddAnonymousPublicDashboardTransactionFactory(sqlstore),
 		sqlmigration.NewAddRootUserFactory(sqlstore, sqlschema),
 		sqlmigration.NewAddUserEmailOrgIDIndexFactory(sqlstore, sqlschema),
-		sqlmigration.NewMigrateRulesV4ToV5Factory(sqlstore, telemetryStore),
 	)
 }
 

pkg/sqlmigration/066_migrate_rules_v4_to_v5_post_deprecation.go

@@ -1,209 +0,0 @@
-package sqlmigration
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"log/slog"
-
-	"github.com/SigNoz/signoz/pkg/factory"
-	"github.com/SigNoz/signoz/pkg/sqlstore"
-	"github.com/SigNoz/signoz/pkg/telemetrystore"
-	"github.com/SigNoz/signoz/pkg/transition"
-	"github.com/uptrace/bun"
-	"github.com/uptrace/bun/migrate"
-)
-
-type migrateRulesV4ToV5 struct {
-	store          sqlstore.SQLStore
-	telemetryStore telemetrystore.TelemetryStore
-	logger         *slog.Logger
-}
-
-func NewMigrateRulesV4ToV5Factory(
-	store sqlstore.SQLStore,
-	telemetryStore telemetrystore.TelemetryStore,
-) factory.ProviderFactory[SQLMigration, Config] {
-	return factory.NewProviderFactory(
-		factory.MustNewName("migrate_rules_post_deprecation"),
-		func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
-			return &migrateRulesV4ToV5{
-				store:          store,
-				telemetryStore: telemetryStore,
-				logger:         ps.Logger,
-			}, nil
-		})
-}
-
-func (migration *migrateRulesV4ToV5) Register(migrations *migrate.Migrations) error {
-	if err := migrations.Register(migration.Up, migration.Down); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (migration *migrateRulesV4ToV5) getLogDuplicateKeys(ctx context.Context) ([]string, error) {
-	query := `
-		SELECT name
-		FROM (
-			SELECT DISTINCT name FROM signoz_logs.distributed_logs_attribute_keys
-			INTERSECT
-			SELECT DISTINCT name FROM signoz_logs.distributed_logs_resource_keys
-		)
-		ORDER BY name
-	`
-
-	rows, err := migration.telemetryStore.ClickhouseDB().Query(ctx, query)
-	if err != nil {
-		migration.logger.WarnContext(ctx, "failed to query log duplicate keys", "error", err)
-		return nil, nil
-	}
-	defer rows.Close()
-
-	var keys []string
-	for rows.Next() {
-		var key string
-		if err := rows.Scan(&key); err != nil {
-			migration.logger.WarnContext(ctx, "failed to scan log duplicate key", "error", err)
-			continue
-		}
-		keys = append(keys, key)
-	}
-
-	return keys, nil
-}
-
-func (migration *migrateRulesV4ToV5) getTraceDuplicateKeys(ctx context.Context) ([]string, error) {
-	query := `
-		SELECT tagKey
-		FROM signoz_traces.distributed_span_attributes_keys
-		WHERE tagType IN ('tag', 'resource')
-		GROUP BY tagKey
-		HAVING COUNT(DISTINCT tagType) > 1
-		ORDER BY tagKey
-	`
-
-	rows, err := migration.telemetryStore.ClickhouseDB().Query(ctx, query)
-	if err != nil {
-		migration.logger.WarnContext(ctx, "failed to query trace duplicate keys", "error", err)
-		return nil, nil
-	}
-	defer rows.Close()
-
-	var keys []string
-	for rows.Next() {
-		var key string
-		if err := rows.Scan(&key); err != nil {
-			migration.logger.WarnContext(ctx, "failed to scan trace duplicate key", "error", err)
-			continue
-		}
-		keys = append(keys, key)
-	}
-
-	return keys, nil
-}
-
-func (migration *migrateRulesV4ToV5) Up(ctx context.Context, db *bun.DB) error {
-	logsKeys, err := migration.getLogDuplicateKeys(ctx)
-	if err != nil {
-		return err
-	}
-
-	tracesKeys, err := migration.getTraceDuplicateKeys(ctx)
-	if err != nil {
-		return err
-	}
-
-	tx, err := db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		_ = tx.Rollback()
-	}()
-
-	var rules []struct {
-		ID   string         `bun:"id"`
-		Data map[string]any `bun:"data"`
-	}
-
-	err = tx.NewSelect().
-		Table("rule").
-		Column("id", "data").
-		Scan(ctx, &rules)
-	if err != nil {
-		if err == sql.ErrNoRows {
-			return nil
-		}
-		return err
-	}
-
-	alertsMigrator := transition.NewAlertMigrateV5(migration.logger, logsKeys, tracesKeys)
-
-	count := 0
-
-	for _, rule := range rules {
-		version, _ := rule.Data["version"].(string)
-
-		if version == "v5" {
-			continue
-		}
-
-		if version == "" {
-			migration.logger.WarnContext(ctx, "unexpected empty version for rule", "rule_id", rule.ID)
-		}
-
-		migration.logger.InfoContext(ctx, "migrating rule v4 to v5", "rule_id", rule.ID, "current_version", version)
-
-		// Check if the queries envelope already exists and is non-empty
-		hasQueriesEnvelope := false
-		if condition, ok := rule.Data["condition"].(map[string]any); ok {
-			if compositeQuery, ok := condition["compositeQuery"].(map[string]any); ok {
-				if queries, ok := compositeQuery["queries"].([]any); ok && len(queries) > 0 {
-					hasQueriesEnvelope = true
-				}
-			}
-		}
-
-		if hasQueriesEnvelope {
-			// already has queries envelope, just bump version
-			// this is because user made a mistake of choosing version
-			migration.logger.InfoContext(ctx, "rule already has queries envelope, bumping version", "rule_id", rule.ID)
-			rule.Data["version"] = "v5"
-		} else {
-			// old format, run full migration
-			migration.logger.InfoContext(ctx, "rule has old format, running full migration", "rule_id", rule.ID)
-			updated := alertsMigrator.Migrate(ctx, rule.Data)
-			if !updated {
-				migration.logger.WarnContext(ctx, "expected updated to be true but got false", "rule_id", rule.ID)
-				continue
-			}
-			rule.Data["version"] = "v5"
-		}
-
-		dataJSON, err := json.Marshal(rule.Data)
-		if err != nil {
-			return err
-		}
-
-		_, err = tx.NewUpdate().
-			Table("rule").
-			Set("data = ?", string(dataJSON)).
-			Where("id = ?", rule.ID).
-			Exec(ctx)
-		if err != nil {
-			return err
-		}
-		count++
-	}
-	if count != 0 {
-		migration.logger.InfoContext(ctx, "migrate v4 alerts", "count", count)
-	}
-
-	return tx.Commit()
-}
-
-func (migration *migrateRulesV4ToV5) Down(ctx context.Context, db *bun.DB) error {
-	return nil
-}

pkg/types/ruletypes/api_params.go

@@ -355,10 +355,6 @@ func (r *PostableRule) validate() error {
 		errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "composite query is required"))
 	}
 
-	if r.Version != "v5" {
-		errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "only version v5 is supported, got %q", r.Version))
-	}
-
 	if isAllQueriesDisabled(r.RuleCondition.CompositeQuery) {
 		errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "all queries are disabled in rule condition"))
 	}

pkg/types/ruletypes/api_params_test.go

@@ -108,7 +108,6 @@ func TestParseIntoRule(t *testing.T) {
 				"ruleType": "threshold_rule",
 				"evalWindow": "5m",
 				"frequency": "1m",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -151,7 +150,6 @@ func TestParseIntoRule(t *testing.T) {
 			content: []byte(`{
 				"alert": "DefaultsRule",
 				"ruleType": "threshold_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -189,7 +187,6 @@ func TestParseIntoRule(t *testing.T) {
 			initRule: PostableRule{},
 			content: []byte(`{
 				"alert": "PromQLRule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "promql",
@@ -259,7 +256,6 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			content: []byte(`{
 				"alert": "SeverityLabelTest",
 				"schemaVersion": "v1",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -348,7 +344,6 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			content: []byte(`{
 				"alert": "NoLabelsTest",
 				"schemaVersion": "v1",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -389,7 +384,6 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			content: []byte(`{
 				"alert": "OverwriteTest",
 				"schemaVersion": "v1",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -480,7 +474,6 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			content: []byte(`{
 				"alert": "V2Test",
 				"schemaVersion": "v2",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -524,7 +517,6 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			initRule: PostableRule{},
 			content: []byte(`{
 				"alert": "DefaultSchemaTest",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -577,7 +569,6 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 func TestParseIntoRuleThresholdGeneration(t *testing.T) {
 	content := []byte(`{
 		"alert": "TestThresholds",
-		"version": "v5",
 		"condition": {
 			"compositeQuery": {
 				"queryType": "builder",
@@ -648,7 +639,6 @@ func TestParseIntoRuleMultipleThresholds(t *testing.T) {
 		"schemaVersion": "v2",
 		"alert": "MultiThresholdAlert",
 		"ruleType": "threshold_rule",
-		"version": "v5",
 		"condition": {
 			"compositeQuery": {
 				"queryType": "builder",
@@ -742,7 +732,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyBelowTest",
 				"ruleType": "anomaly_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -777,7 +766,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyBelowTest",
 				"ruleType": "anomaly_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -811,7 +799,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyAboveTest",
 				"ruleType": "anomaly_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -846,7 +833,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyAboveTest",
 				"ruleType": "anomaly_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -880,7 +866,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyBelowAllTest",
 				"ruleType": "anomaly_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -916,7 +901,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyBelowAllTest",
 				"ruleType": "anomaly_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -951,7 +935,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyOutOfBoundsTest",
 				"ruleType": "anomaly_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -986,7 +969,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "ThresholdTest",
 				"ruleType": "threshold_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -1021,7 +1003,6 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "ThresholdTest",
 				"ruleType": "threshold_rule",
-				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",