chore: deprecates generator from deploy

* fix: add missing filtering for ip address for scalar data

In domain listing api for external api monitoring,
we have option to filter out the IP address but
it only handles timeseries and raw type data while
domain list handler returns scalar data.

* fix: switch to new derived attributes for ip filtering

---------

Co-authored-by: Nityananda Gohain <nityanandagohain@gmail.com>

deploy/docker-swarm/generator/hotrod/docker-compose.yaml

@@ -1,38 +0,0 @@
-version: "3"
-x-common: &common
-  networks:
-    - signoz-net
-  extra_hosts:
-    - host.docker.internal:host-gateway
-  logging:
-    options:
-      max-size: 50m
-      max-file: "3"
-  deploy:
-    restart_policy:
-      condition: on-failure
-services:
-  hotrod:
-    <<: *common
-    image: jaegertracing/example-hotrod:1.61.0
-    command: [ "all" ]
-    environment:
-      - OTEL_EXPORTER_OTLP_ENDPOINT=http://host.docker.internal:4318  #
-  load-hotrod:
-    <<: *common
-    image: "signoz/locust:1.2.3"
-    environment:
-      ATTACKED_HOST: http://hotrod:8080
-      LOCUST_MODE: standalone
-      NO_PROXY: standalone
-      TASK_DELAY_FROM: 5
-      TASK_DELAY_TO: 30
-      QUIET_MODE: "${QUIET_MODE:-false}"
-      LOCUST_OPTS: "--headless -u 10 -r 1"
-    volumes:
-      - ../../../common/locust-scripts:/locust
-
-networks:
-  signoz-net:
-    name: signoz-net
-    external: true

deploy/docker-swarm/generator/infra/docker-compose.yaml

@@ -1,69 +0,0 @@
-version: "3"
-x-common: &common
-  networks:
-    - signoz-net
-  extra_hosts:
-    - host.docker.internal:host-gateway
-  logging:
-    options:
-      max-size: 50m
-      max-file: "3"
-  deploy:
-    mode: global
-    restart_policy:
-      condition: on-failure
-services:
-  otel-agent:
-    <<: *common
-    image: otel/opentelemetry-collector-contrib:0.111.0
-    command:
-      - --config=/etc/otel-collector-config.yaml
-    volumes:
-      - ./otel-agent-config.yaml:/etc/otel-collector-config.yaml
-      - /:/hostfs:ro
-    environment:
-      - SIGNOZ_COLLECTOR_ENDPOINT=http://host.docker.internal:4317    # In case of external SigNoz or cloud, update the endpoint and access token
-      - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}}
-      # - SIGNOZ_ACCESS_TOKEN="<your-access-token>"
-    # Before exposing the ports, make sure the ports are not used by other services
-    # ports:
-    #   - "4317:4317"
-    #   - "4318:4318"
-  otel-metrics:
-    <<: *common
-    image: otel/opentelemetry-collector-contrib:0.111.0
-    user: 0:0   #  If you have security concerns, you can replace this with your `UID:GID` that has necessary permissions to docker.sock
-    command:
-      - --config=/etc/otel-collector-config.yaml
-    volumes:
-      - ./otel-metrics-config.yaml:/etc/otel-collector-config.yaml
-      - /var/run/docker.sock:/var/run/docker.sock
-    environment:
-      - SIGNOZ_COLLECTOR_ENDPOINT=http://host.docker.internal:4317    # In case of external SigNoz or cloud, update the endpoint and access token
-      - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}}
-      # - SIGNOZ_ACCESS_TOKEN="<your-access-token>"
-    # Before exposing the ports, make sure the ports are not used by other services
-    # ports:
-    #   - "4317:4317"
-    #   - "4318:4318"
-    deploy:
-      mode: replicated
-      replicas: 1
-      placement:
-        constraints:
-          - node.role == manager
-  logspout:
-    <<: *common
-    image: "gliderlabs/logspout:v3.2.14"
-    command: syslog+tcp://otel-agent:2255
-    user: root
-    volumes:
-      - /etc/hostname:/etc/host_hostname:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-    depends_on:
-      - otel-agent
-
-networks:
-  signoz-net:
-    name: signoz-net
-    external: true

deploy/docker-swarm/generator/infra/otel-agent-config.yaml

@@ -1,102 +0,0 @@
-receivers:
-  hostmetrics:
-    collection_interval: 30s
-    root_path: /hostfs
-    scrapers:
-      cpu: {}
-      load: {}
-      memory: {}
-      disk: {}
-      filesystem: {}
-      network: {}
-  otlp:
-    protocols:
-      grpc:
-        endpoint: 0.0.0.0:4317
-      http:
-        endpoint: 0.0.0.0:4318
-  prometheus:
-    config:
-      global:
-        scrape_interval: 60s
-      scrape_configs:
-        - job_name: otel-agent
-          static_configs:
-          - targets:
-              - localhost:8888
-            labels:
-              job_name: otel-agent
-  tcplog/docker:
-    listen_address: "0.0.0.0:2255"
-    operators:
-      - type: regex_parser
-        regex: '^<([0-9]+)>[0-9]+ (?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P<container_id>\S+) (?P<container_name>\S+) [0-9]+ - -( (?P<body>.*))?'
-        timestamp:
-          parse_from: attributes.timestamp
-          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
-      - type: move
-        from: attributes["body"]
-        to: body
-      - type: remove
-        field: attributes.timestamp
-      # please remove names from below if you want to collect logs from them
-      - type: filter
-        id: signoz_logs_filter
-        expr: 'attributes.container_name matches "^(signoz_(logspout|signoz|otel-collector|clickhouse|zookeeper))|(infra_(logspout|otel-agent|otel-metrics)).*"'
-processors:
-  batch:
-    send_batch_size: 10000
-    send_batch_max_size: 11000
-    timeout: 10s
-  resourcedetection:
-    # Using OTEL_RESOURCE_ATTRIBUTES envvar, env detector adds custom labels.
-    detectors:
-      # - ec2
-      # - gcp
-      # - azure
-      - env
-      - system
-    timeout: 2s
-extensions:
-  health_check:
-    endpoint: 0.0.0.0:13133
-  pprof:
-    endpoint: 0.0.0.0:1777
-exporters:
-  otlp:
-    endpoint: ${env:SIGNOZ_COLLECTOR_ENDPOINT}
-    tls:
-      insecure: true
-    headers:
-      signoz-access-token: ${env:SIGNOZ_ACCESS_TOKEN}
-  # debug: {}
-service:
-  telemetry:
-    logs:
-      encoding: json
-    metrics:
-      address: 0.0.0.0:8888
-  extensions:
-    - health_check
-    - pprof
-  pipelines:
-    traces:
-      receivers: [otlp]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]
-    metrics:
-      receivers: [otlp]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]
-    metrics/hostmetrics:
-      receivers: [hostmetrics]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]
-    metrics/prometheus:
-      receivers: [prometheus]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]
-    logs:
-      receivers: [otlp, tcplog/docker]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]

deploy/docker-swarm/generator/infra/otel-metrics-config.yaml

@@ -1,103 +0,0 @@
-receivers:
-  prometheus:
-    config:
-      global:
-        scrape_interval: 60s
-      scrape_configs:
-        - job_name: otel-metrics
-          static_configs:
-          - targets:
-              - localhost:8888
-            labels:
-              job_name: otel-metrics
-        # For Docker daemon metrics to be scraped, it must be configured to expose
-        # Prometheus metrics, as documented here: https://docs.docker.com/config/daemon/prometheus/
-        # - job_name: docker-daemon
-        #   dockerswarm_sd_configs:
-        #     - host: unix:///var/run/docker.sock
-        #       role: nodes
-        #   relabel_configs:
-        #     - source_labels: [__meta_dockerswarm_node_address]
-        #       target_label: __address__
-        #       replacement: $1:9323
-        - job_name: "dockerswarm"
-          dockerswarm_sd_configs:
-            - host: unix:///var/run/docker.sock
-              role: tasks
-          relabel_configs:
-            - action: keep
-              regex: running
-              source_labels:
-                - __meta_dockerswarm_task_desired_state
-            - action: keep
-              regex: true
-              source_labels:
-                - __meta_dockerswarm_service_label_signoz_io_scrape
-            - regex: ([^:]+)(?::\d+)?
-              replacement: $1
-              source_labels:
-                - __address__
-              target_label: swarm_container_ip
-            - separator: .
-              source_labels:
-                - __meta_dockerswarm_service_name
-                - __meta_dockerswarm_task_slot
-                - __meta_dockerswarm_task_id
-              target_label: swarm_container_name
-            - target_label: __address__
-              source_labels:
-                - swarm_container_ip
-                - __meta_dockerswarm_service_label_signoz_io_port
-              separator: ":"
-            - source_labels:
-                - __meta_dockerswarm_service_label_signoz_io_path
-              target_label: __metrics_path__
-            - source_labels:
-              - __meta_dockerswarm_service_label_com_docker_stack_namespace
-              target_label: namespace
-            - source_labels:
-                - __meta_dockerswarm_service_name
-              target_label: service_name
-            - source_labels:
-                - __meta_dockerswarm_task_id
-              target_label: service_instance_id
-            - source_labels:
-                - __meta_dockerswarm_node_hostname
-              target_label: host_name
-processors:
-  batch:
-    send_batch_size: 10000
-    send_batch_max_size: 11000
-    timeout: 10s
-  resourcedetection:
-    detectors:
-      - env
-      - system
-    timeout: 2s
-extensions:
-  health_check:
-    endpoint: 0.0.0.0:13133
-  pprof:
-    endpoint: 0.0.0.0:1777
-exporters:
-  otlp:
-    endpoint: ${env:SIGNOZ_COLLECTOR_ENDPOINT}
-    tls:
-      insecure: true
-    headers:
-      signoz-access-token: ${env:SIGNOZ_ACCESS_TOKEN}
-  # debug: {}
-service:
-  telemetry:
-    logs:
-      encoding: json
-    metrics:
-      address: 0.0.0.0:8888
-  extensions:
-    - health_check
-    - pprof
-  pipelines:
-    metrics:
-      receivers: [prometheus]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]

deploy/docker/generator/hotrod/docker-compose.yaml

@@ -1,39 +0,0 @@
-version: "3"
-x-common: &common
-  networks:
-    - signoz-net
-  extra_hosts:
-    - host.docker.internal:host-gateway
-  logging:
-    options:
-      max-size: 50m
-      max-file: "3"
-  restart: unless-stopped
-services:
-  hotrod:
-    <<: *common
-    image: jaegertracing/example-hotrod:1.61.0
-    container_name: hotrod
-    command: [ "all" ]
-    environment:
-      - OTEL_EXPORTER_OTLP_ENDPOINT=http://host.docker.internal:4318  # In case of external SigNoz or cloud, update the endpoint and access token
-      # - OTEL_OTLP_HEADERS=signoz-access-token=<your-access-token>
-  load-hotrod:
-    <<: *common
-    image: "signoz/locust:1.2.3"
-    container_name: load-hotrod
-    environment:
-      ATTACKED_HOST: http://hotrod:8080
-      LOCUST_MODE: standalone
-      NO_PROXY: standalone
-      TASK_DELAY_FROM: 5
-      TASK_DELAY_TO: 30
-      QUIET_MODE: "${QUIET_MODE:-false}"
-      LOCUST_OPTS: "--headless -u 10 -r 1"
-    volumes:
-      - ../../../common/locust-scripts:/locust
-
-networks:
-  signoz-net:
-    name: signoz-net
-    external: true

deploy/docker/generator/infra/docker-compose.yaml

@@ -1,43 +0,0 @@
-version: "3"
-x-common: &common
-  networks:
-    - signoz-net
-  extra_hosts:
-    - host.docker.internal:host-gateway
-  logging:
-    options:
-      max-size: 50m
-      max-file: "3"
-  restart: unless-stopped
-services:
-  otel-agent:
-    <<: *common
-    image: otel/opentelemetry-collector-contrib:0.111.0
-    command:
-      - --config=/etc/otel-collector-config.yaml
-    volumes:
-      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
-      - /:/hostfs:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-    environment:
-      - SIGNOZ_COLLECTOR_ENDPOINT=http://host.docker.internal:4317    # In case of external SigNoz or cloud, update the endpoint and access token
-      - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux  # Replace signoz-host with the actual hostname
-      # - SIGNOZ_ACCESS_TOKEN="<your-access-token>"
-    # Before exposing the ports, make sure the ports are not used by other services
-    # ports:
-    #   - "4317:4317"
-    #   - "4318:4318"
-  logspout:
-    <<: *common
-    image: "gliderlabs/logspout:v3.2.14"
-    volumes:
-      - /etc/hostname:/etc/host_hostname:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-    command: syslog+tcp://otel-agent:2255
-    depends_on:
-      - otel-agent
-
-networks:
-  signoz-net:
-    name: signoz-net
-    external: true

deploy/docker/generator/infra/otel-collector-config.yaml

@@ -1,139 +0,0 @@
-receivers:
-  hostmetrics:
-    collection_interval: 30s
-    root_path: /hostfs
-    scrapers:
-      cpu: {}
-      load: {}
-      memory: {}
-      disk: {}
-      filesystem: {}
-      network: {}
-  otlp:
-    protocols:
-      grpc:
-        endpoint: 0.0.0.0:4317
-      http:
-        endpoint: 0.0.0.0:4318
-  prometheus:
-    config:
-      global:
-        scrape_interval: 60s
-      scrape_configs:
-        - job_name: otel-collector
-          static_configs:
-          - targets:
-              - localhost:8888
-            labels:
-              job_name: otel-collector
-        # For Docker daemon metrics to be scraped, it must be configured to expose
-        # Prometheus metrics, as documented here: https://docs.docker.com/config/daemon/prometheus/
-        # - job_name: docker-daemon
-        #   static_configs:
-        #   - targets:
-        #       - host.docker.internal:9323
-        #     labels:
-        #       job_name: docker-daemon
-        - job_name: docker-container
-          docker_sd_configs:
-            - host: unix:///var/run/docker.sock
-          relabel_configs:
-            - action: keep
-              regex: true
-              source_labels:
-                - __meta_docker_container_label_signoz_io_scrape
-            - regex: true
-              source_labels:
-                - __meta_docker_container_label_signoz_io_path
-              target_label: __metrics_path__
-            - regex: (.+)
-              source_labels:
-                - __meta_docker_container_label_signoz_io_path
-              target_label: __metrics_path__
-            - separator: ":"
-              source_labels:
-                - __meta_docker_network_ip
-                - __meta_docker_container_label_signoz_io_port
-              target_label: __address__
-            - regex: '/(.*)'
-              replacement: '$1'
-              source_labels:
-                - __meta_docker_container_name
-              target_label: container_name
-            - regex: __meta_docker_container_label_signoz_io_(.+)
-              action: labelmap
-              replacement: $1
-  tcplog/docker:
-    listen_address: "0.0.0.0:2255"
-    operators:
-      - type: regex_parser
-        regex: '^<([0-9]+)>[0-9]+ (?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P<container_id>\S+) (?P<container_name>\S+) [0-9]+ - -( (?P<body>.*))?'
-        timestamp:
-          parse_from: attributes.timestamp
-          layout: '%Y-%m-%dT%H:%M:%S.%LZ'
-      - type: move
-        from: attributes["body"]
-        to: body
-      - type: remove
-        field: attributes.timestamp
-      # please remove names from below if you want to collect logs from them
-      - type: filter
-        id: signoz_logs_filter
-        expr: 'attributes.container_name matches "^signoz|(signoz-(|otel-collector|clickhouse|zookeeper))|(infra-(logspout|otel-agent)-.*)"'
-processors:
-  batch:
-    send_batch_size: 10000
-    send_batch_max_size: 11000
-    timeout: 10s
-  resourcedetection:
-    # Using OTEL_RESOURCE_ATTRIBUTES envvar, env detector adds custom labels.
-    detectors:
-      # - ec2
-      # - gcp
-      # - azure
-      - env
-      - system
-    timeout: 2s
-extensions:
-  health_check:
-    endpoint: 0.0.0.0:13133
-  pprof:
-    endpoint: 0.0.0.0:1777
-exporters:
-  otlp:
-    endpoint: ${env:SIGNOZ_COLLECTOR_ENDPOINT}
-    tls:
-      insecure: true
-    headers:
-      signoz-access-token: ${env:SIGNOZ_ACCESS_TOKEN}
-  # debug: {}
-service:
-  telemetry:
-    logs:
-      encoding: json
-    metrics:
-      address: 0.0.0.0:8888
-  extensions:
-    - health_check
-    - pprof
-  pipelines:
-    traces:
-      receivers: [otlp]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]
-    metrics:
-      receivers: [otlp]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]
-    metrics/hostmetrics:
-      receivers: [hostmetrics]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]
-    metrics/prometheus:
-      receivers: [prometheus]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]
-    logs:
-      receivers: [otlp, tcplog/docker]
-      processors: [resourcedetection, batch]
-      exporters: [otlp]

frontend/src/AppRoutes/pageComponents.ts

@@ -308,3 +308,15 @@ export const PublicDashboardPage = Loadable(
 			/* webpackChunkName: "Public Dashboard Page" */ 'pages/PublicDashboard'
 		),
 );
+
+export const AlertTypeSelectionPage = Loadable(
+	() =>
+		import(
+			/* webpackChunkName: "Alert Type Selection Page" */ 'pages/AlertTypeSelection'
+		),
+);
+
+export const MeterExplorerPage = Loadable(
+	() =>
+		import(/* webpackChunkName: "Meter Explorer Page" */ 'pages/MeterExplorer'),
+);

frontend/src/AppRoutes/routes.ts

@@ -1,12 +1,10 @@
 import { RouteProps } from 'react-router-dom';
 import ROUTES from 'constants/routes';
-import AlertTypeSelectionPage from 'pages/AlertTypeSelection';
-import MessagingQueues from 'pages/MessagingQueues';
-import MeterExplorer from 'pages/MeterExplorer';
 
 import {
 	AlertHistory,
 	AlertOverview,
+	AlertTypeSelectionPage,
 	AllAlertChannels,
 	AllErrors,
 	ApiMonitoring,
@@ -29,6 +27,8 @@ import {
 	LogsExplorer,
 	LogsIndexToFields,
 	LogsSaveViews,
+	MessagingQueuesMainPage,
+	MeterExplorerPage,
 	MetricsExplorer,
 	OldLogsExplorer,
 	Onboarding,
@@ -399,28 +399,28 @@ const routes: AppRoutes[] = [
 	{
 		path: ROUTES.MESSAGING_QUEUES_KAFKA,
 		exact: true,
-		component: MessagingQueues,
+		component: MessagingQueuesMainPage,
 		key: 'MESSAGING_QUEUES_KAFKA',
 		isPrivate: true,
 	},
 	{
 		path: ROUTES.MESSAGING_QUEUES_CELERY_TASK,
 		exact: true,
-		component: MessagingQueues,
+		component: MessagingQueuesMainPage,
 		key: 'MESSAGING_QUEUES_CELERY_TASK',
 		isPrivate: true,
 	},
 	{
 		path: ROUTES.MESSAGING_QUEUES_OVERVIEW,
 		exact: true,
-		component: MessagingQueues,
+		component: MessagingQueuesMainPage,
 		key: 'MESSAGING_QUEUES_OVERVIEW',
 		isPrivate: true,
 	},
 	{
 		path: ROUTES.MESSAGING_QUEUES_KAFKA_DETAIL,
 		exact: true,
-		component: MessagingQueues,
+		component: MessagingQueuesMainPage,
 		key: 'MESSAGING_QUEUES_KAFKA_DETAIL',
 		isPrivate: true,
 	},
@@ -463,21 +463,21 @@ const routes: AppRoutes[] = [
 	{
 		path: ROUTES.METER,
 		exact: true,
-		component: MeterExplorer,
+		component: MeterExplorerPage,
 		key: 'METER',
 		isPrivate: true,
 	},
 	{
 		path: ROUTES.METER_EXPLORER,
 		exact: true,
-		component: MeterExplorer,
+		component: MeterExplorerPage,
 		key: 'METER_EXPLORER',
 		isPrivate: true,
 	},
 	{
 		path: ROUTES.METER_EXPLORER_VIEWS,
 		exact: true,
-		component: MeterExplorer,
+		component: MeterExplorerPage,
 		key: 'METER_EXPLORER_VIEWS',
 		isPrivate: true,
 	},

pkg/modules/thirdpartyapi/translator.go

@@ -120,6 +120,8 @@ func FilterResponse(results []*qbtypes.QueryRangeResponse) []*qbtypes.QueryRange
 					}
 				}
 				resultData.Rows = filteredRows
+			case *qbtypes.ScalarData:
+				resultData.Data = filterScalarDataIPs(resultData.Columns, resultData.Data)
 			}
 
 			filteredData = append(filteredData, result)
@@ -145,6 +147,39 @@ func shouldIncludeSeries(series *qbtypes.TimeSeries) bool {
 	return true
 }
 
+func filterScalarDataIPs(columns []*qbtypes.ColumnDescriptor, data [][]any) [][]any {
+	// Find column indices for server address fields
+	serverColIndices := make([]int, 0)
+	for i, col := range columns {
+		if col.Name == derivedKeyHTTPHost {
+			serverColIndices = append(serverColIndices, i)
+		}
+	}
+
+	if len(serverColIndices) == 0 {
+		return data
+	}
+
+	filtered := make([][]any, 0, len(data))
+	for _, row := range data {
+		includeRow := true
+		for _, colIdx := range serverColIndices {
+			if colIdx < len(row) {
+				if strVal, ok := row[colIdx].(string); ok {
+					if net.ParseIP(strVal) != nil {
+						includeRow = false
+						break
+					}
+				}
+			}
+		}
+		if includeRow {
+			filtered = append(filtered, row)
+		}
+	}
+	return filtered
+}
+
 func shouldIncludeRow(row *qbtypes.RawRow) bool {
 	if row.Data != nil {
 		if domainVal, ok := row.Data[derivedKeyHTTPHost]; ok {

pkg/modules/thirdpartyapi/translator_test.go

@@ -117,6 +117,59 @@ func TestFilterResponse(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "should filter out IP addresses from scalar data",
+			input: []*qbtypes.QueryRangeResponse{
+				{
+					Data: qbtypes.QueryData{
+						Results: []any{
+							&qbtypes.ScalarData{
+								QueryName: "endpoints",
+								Columns: []*qbtypes.ColumnDescriptor{
+									{
+										TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: derivedKeyHTTPHost},
+										Type:              qbtypes.ColumnTypeGroup,
+									},
+									{
+										TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "endpoints"},
+										Type:              qbtypes.ColumnTypeAggregation,
+									},
+								},
+								Data: [][]any{
+									{"192.168.1.1", 10},
+									{"example.com", 20},
+									{"10.0.0.1", 5},
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: []*qbtypes.QueryRangeResponse{
+				{
+					Data: qbtypes.QueryData{
+						Results: []any{
+							&qbtypes.ScalarData{
+								QueryName: "endpoints",
+								Columns: []*qbtypes.ColumnDescriptor{
+									{
+										TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: derivedKeyHTTPHost},
+										Type:              qbtypes.ColumnTypeGroup,
+									},
+									{
+										TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "endpoints"},
+										Type:              qbtypes.ColumnTypeAggregation,
+									},
+								},
+								Data: [][]any{
+									{"example.com", 20},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

pkg/signoz/provider.go

@@ -169,6 +169,7 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewAddAnonymousPublicDashboardTransactionFactory(sqlstore),
 		sqlmigration.NewAddRootUserFactory(sqlstore, sqlschema),
 		sqlmigration.NewAddUserEmailOrgIDIndexFactory(sqlstore, sqlschema),
+		sqlmigration.NewMigrateRulesV4ToV5Factory(sqlstore, telemetryStore),
 	)
 }
 

pkg/sqlmigration/066_migrate_rules_v4_to_v5_post_deprecation.go

@@ -0,0 +1,209 @@
+package sqlmigration
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"log/slog"
+
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/sqlstore"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/SigNoz/signoz/pkg/transition"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/migrate"
+)
+
+type migrateRulesV4ToV5 struct {
+	store          sqlstore.SQLStore
+	telemetryStore telemetrystore.TelemetryStore
+	logger         *slog.Logger
+}
+
+func NewMigrateRulesV4ToV5Factory(
+	store sqlstore.SQLStore,
+	telemetryStore telemetrystore.TelemetryStore,
+) factory.ProviderFactory[SQLMigration, Config] {
+	return factory.NewProviderFactory(
+		factory.MustNewName("migrate_rules_post_deprecation"),
+		func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
+			return &migrateRulesV4ToV5{
+				store:          store,
+				telemetryStore: telemetryStore,
+				logger:         ps.Logger,
+			}, nil
+		})
+}
+
+func (migration *migrateRulesV4ToV5) Register(migrations *migrate.Migrations) error {
+	if err := migrations.Register(migration.Up, migration.Down); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (migration *migrateRulesV4ToV5) getLogDuplicateKeys(ctx context.Context) ([]string, error) {
+	query := `
+		SELECT name
+		FROM (
+			SELECT DISTINCT name FROM signoz_logs.distributed_logs_attribute_keys
+			INTERSECT
+			SELECT DISTINCT name FROM signoz_logs.distributed_logs_resource_keys
+		)
+		ORDER BY name
+	`
+
+	rows, err := migration.telemetryStore.ClickhouseDB().Query(ctx, query)
+	if err != nil {
+		migration.logger.WarnContext(ctx, "failed to query log duplicate keys", "error", err)
+		return nil, nil
+	}
+	defer rows.Close()
+
+	var keys []string
+	for rows.Next() {
+		var key string
+		if err := rows.Scan(&key); err != nil {
+			migration.logger.WarnContext(ctx, "failed to scan log duplicate key", "error", err)
+			continue
+		}
+		keys = append(keys, key)
+	}
+
+	return keys, nil
+}
+
+func (migration *migrateRulesV4ToV5) getTraceDuplicateKeys(ctx context.Context) ([]string, error) {
+	query := `
+		SELECT tagKey
+		FROM signoz_traces.distributed_span_attributes_keys
+		WHERE tagType IN ('tag', 'resource')
+		GROUP BY tagKey
+		HAVING COUNT(DISTINCT tagType) > 1
+		ORDER BY tagKey
+	`
+
+	rows, err := migration.telemetryStore.ClickhouseDB().Query(ctx, query)
+	if err != nil {
+		migration.logger.WarnContext(ctx, "failed to query trace duplicate keys", "error", err)
+		return nil, nil
+	}
+	defer rows.Close()
+
+	var keys []string
+	for rows.Next() {
+		var key string
+		if err := rows.Scan(&key); err != nil {
+			migration.logger.WarnContext(ctx, "failed to scan trace duplicate key", "error", err)
+			continue
+		}
+		keys = append(keys, key)
+	}
+
+	return keys, nil
+}
+
+func (migration *migrateRulesV4ToV5) Up(ctx context.Context, db *bun.DB) error {
+	logsKeys, err := migration.getLogDuplicateKeys(ctx)
+	if err != nil {
+		return err
+	}
+
+	tracesKeys, err := migration.getTraceDuplicateKeys(ctx)
+	if err != nil {
+		return err
+	}
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = tx.Rollback()
+	}()
+
+	var rules []struct {
+		ID   string         `bun:"id"`
+		Data map[string]any `bun:"data"`
+	}
+
+	err = tx.NewSelect().
+		Table("rule").
+		Column("id", "data").
+		Scan(ctx, &rules)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil
+		}
+		return err
+	}
+
+	alertsMigrator := transition.NewAlertMigrateV5(migration.logger, logsKeys, tracesKeys)
+
+	count := 0
+
+	for _, rule := range rules {
+		version, _ := rule.Data["version"].(string)
+
+		if version == "v5" {
+			continue
+		}
+
+		if version == "" {
+			migration.logger.WarnContext(ctx, "unexpected empty version for rule", "rule_id", rule.ID)
+		}
+
+		migration.logger.InfoContext(ctx, "migrating rule v4 to v5", "rule_id", rule.ID, "current_version", version)
+
+		// Check if the queries envelope already exists and is non-empty
+		hasQueriesEnvelope := false
+		if condition, ok := rule.Data["condition"].(map[string]any); ok {
+			if compositeQuery, ok := condition["compositeQuery"].(map[string]any); ok {
+				if queries, ok := compositeQuery["queries"].([]any); ok && len(queries) > 0 {
+					hasQueriesEnvelope = true
+				}
+			}
+		}
+
+		if hasQueriesEnvelope {
+			// already has queries envelope, just bump version
+			// this is because user made a mistake of choosing version
+			migration.logger.InfoContext(ctx, "rule already has queries envelope, bumping version", "rule_id", rule.ID)
+			rule.Data["version"] = "v5"
+		} else {
+			// old format, run full migration
+			migration.logger.InfoContext(ctx, "rule has old format, running full migration", "rule_id", rule.ID)
+			updated := alertsMigrator.Migrate(ctx, rule.Data)
+			if !updated {
+				migration.logger.WarnContext(ctx, "expected updated to be true but got false", "rule_id", rule.ID)
+				continue
+			}
+			rule.Data["version"] = "v5"
+		}
+
+		dataJSON, err := json.Marshal(rule.Data)
+		if err != nil {
+			return err
+		}
+
+		_, err = tx.NewUpdate().
+			Table("rule").
+			Set("data = ?", string(dataJSON)).
+			Where("id = ?", rule.ID).
+			Exec(ctx)
+		if err != nil {
+			return err
+		}
+		count++
+	}
+	if count != 0 {
+		migration.logger.InfoContext(ctx, "migrate v4 alerts", "count", count)
+	}
+
+	return tx.Commit()
+}
+
+func (migration *migrateRulesV4ToV5) Down(ctx context.Context, db *bun.DB) error {
+	return nil
+}

pkg/types/ruletypes/api_params.go

@@ -355,6 +355,10 @@ func (r *PostableRule) validate() error {
 		errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "composite query is required"))
 	}
 
+	if r.Version != "v5" {
+		errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "only version v5 is supported, got %q", r.Version))
+	}
+
 	if isAllQueriesDisabled(r.RuleCondition.CompositeQuery) {
 		errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "all queries are disabled in rule condition"))
 	}

pkg/types/ruletypes/api_params_test.go

@@ -108,6 +108,7 @@ func TestParseIntoRule(t *testing.T) {
 				"ruleType": "threshold_rule",
 				"evalWindow": "5m",
 				"frequency": "1m",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -150,6 +151,7 @@ func TestParseIntoRule(t *testing.T) {
 			content: []byte(`{
 				"alert": "DefaultsRule",
 				"ruleType": "threshold_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -187,6 +189,7 @@ func TestParseIntoRule(t *testing.T) {
 			initRule: PostableRule{},
 			content: []byte(`{
 				"alert": "PromQLRule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "promql",
@@ -256,6 +259,7 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			content: []byte(`{
 				"alert": "SeverityLabelTest",
 				"schemaVersion": "v1",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -344,6 +348,7 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			content: []byte(`{
 				"alert": "NoLabelsTest",
 				"schemaVersion": "v1",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -384,6 +389,7 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			content: []byte(`{
 				"alert": "OverwriteTest",
 				"schemaVersion": "v1",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -474,6 +480,7 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			content: []byte(`{
 				"alert": "V2Test",
 				"schemaVersion": "v2",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -517,6 +524,7 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 			initRule: PostableRule{},
 			content: []byte(`{
 				"alert": "DefaultSchemaTest",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -569,6 +577,7 @@ func TestParseIntoRuleSchemaVersioning(t *testing.T) {
 func TestParseIntoRuleThresholdGeneration(t *testing.T) {
 	content := []byte(`{
 		"alert": "TestThresholds",
+		"version": "v5",
 		"condition": {
 			"compositeQuery": {
 				"queryType": "builder",
@@ -639,6 +648,7 @@ func TestParseIntoRuleMultipleThresholds(t *testing.T) {
 		"schemaVersion": "v2",
 		"alert": "MultiThresholdAlert",
 		"ruleType": "threshold_rule",
+		"version": "v5",
 		"condition": {
 			"compositeQuery": {
 				"queryType": "builder",
@@ -732,6 +742,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyBelowTest",
 				"ruleType": "anomaly_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -766,6 +777,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyBelowTest",
 				"ruleType": "anomaly_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -799,6 +811,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyAboveTest",
 				"ruleType": "anomaly_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -833,6 +846,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyAboveTest",
 				"ruleType": "anomaly_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -866,6 +880,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyBelowAllTest",
 				"ruleType": "anomaly_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -901,6 +916,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyBelowAllTest",
 				"ruleType": "anomaly_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -935,6 +951,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "AnomalyOutOfBoundsTest",
 				"ruleType": "anomaly_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -969,6 +986,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "ThresholdTest",
 				"ruleType": "threshold_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",
@@ -1003,6 +1021,7 @@ func TestAnomalyNegationEval(t *testing.T) {
 			ruleJSON: []byte(`{
 				"alert": "ThresholdTest",
 				"ruleType": "threshold_rule",
+				"version": "v5",
 				"condition": {
 					"compositeQuery": {
 						"queryType": "builder",