test: fix error msg assertion

.claude/opencode.json

@@ -1,7 +0,0 @@
-{
-  "$schema": "https://opencode.ai/config.json",
-  "lsp": true,
-  "experimental": {
-    "disable_paste_summary": true
-  }
-}

.claude/rules/comments.md

@@ -7,6 +7,5 @@ Applies to everything in the repo — code, config, workflows.
 - **Rationale goes in prose, not source.** Why a version is pinned, why a job exists, how a subsystem fits together — that belongs in the README or the PR.
 - **Never remove pre-existing comments** when editing code. The bar above applies to comments you write, not comments already there.
 - **Never talk to the reviewer.** No comments about where a change came from, what was changed, or why the change is correct — that belongs in the PR description and is noise the moment it merges.
-- **Less is more.** When writing something intended for human consumption, (comment, commit message, reply to prompt) use as few words as possible. Pick every word meticulously to reduce the volume to a strict minimum. Be down to the point. Less is more.
 
 Language rules build on this one: [`go-comments`](go-comments.md), [`py-comments`](py-comments.md).

.claude/rules/go-contrib.md

@@ -1,16 +0,0 @@
----
-paths:
-  - "**/*.go"
----
-
-# Contribution guidelines
-
-- When making Go changes, always ensure they follow the contributing guildelines in [`docs/contributing/go/`](../../docs/contributing/go/).
-- Look for existing patterns in the codebase for any change before implementing the changes.
-- If any API contract is modified, generate the OpenAPI specs with `make gen-openapi-specs`.
-- Always keep the OpenAPI spec generated in a separate commit, so the whole commit can be dropped in case of conflicts during merge. Do not try to resolve conflict in generated files, instead just generate them again.
-- Avoid breaking function calls unncessarily into multilines for couple of arguments.
-- Try to keep most computational only logic in types package itself related to a domain type, use modules as the orchestraction layer cordinating different layers and all db queries in store layer. Check the serviceaccount modules for inspiration when confused.
-- When defining types, keep the structure of file to have any constants and variables first, then exported types and exported methods and then finally the unexported types and methods.
-- Never import types or other modules in migration files, duplicate the required type or method to keep migration free from changes.
-- Always run the gofmt tool for formating beforing commiting any changes.

.claude/rules/go-test.md

@@ -1,12 +0,0 @@
----
-paths:
-  - "**/*_test.go"
----
-
-# Go tests
-
-- **testify + table-driven.** Use `assert` / `require`; prefer table-driven cases. Tests live next to the source file.
-- **`require` vs `assert`.** `require` for anything the rest of the test cannot proceed without — setup, `require.NoError(t, err)`, nil/length checks before indexing or dereferencing. `assert` for the actual expectations, so one failed check still reports the rest.
-- **Mock with mockery.** When an interface needs mocking, list it in `.mockery.yml` and run `mockery`; never hand-write mocks. Generated mocks live in the source package's `<pkg>test` sibling (e.g. `resourcestest.NewMockAdapter(t)`).
-- **Table format.** Declare cases as `testCases := []struct{ name string; ... }` and iterate with `for _, testCase := range testCases { t.Run(testCase.name, ...) }` — the variables are named `testCases` / `testCase`. Case names are PascalCase segments joined by `_`, one segment per aspect (scenario, condition, expectation): `TimestampNotNullNoDefault`, `DropPrimaryKeyConstraint_AlterColumnNullable`, `ForeignKeyConstraint_DoesNotExist_SCreateAndDropConstraintTrue`.
-- **No hoisted test constants.** When goconst flags a repeated literal in a test, vary the fixture strings across cases instead of hoisting a constant — never introduce a shared const for test data. 

.claude/rules/pull-requests.md

@@ -2,10 +2,6 @@
 
 - **Follow the template** (`.github/pull_request_template.md`): fill in its headings (Description / Issues closed by this PR / Screenshots / Additional Information). Don't add sections the template doesn't have.
 - **Keep only the headings that apply.** Delete every heading that has nothing under it, along with its `<!--...-->` placeholder comment. The body must never contain an empty heading — if only Description applies, the body has exactly that one heading.
-- **Keep the description concise and human-readable.** A few non repetitive bullets saying what changed and why, for a reviewer skimming it — not a wall of text, not a restatement of the diff, not generated boilerplate and not the user agent conversation details.
+- **Keep the description concise and human-readable.** A few plain bullets saying what changed and why, for a reviewer skimming it — not a wall of text, not a restatement of the diff, not generated boilerplate.
 - **Reference issues with `Closes #issue-number`** under "Issues closed by this PR" so they auto-close on merge. This goes in the PR description only — never in commit messages.
-- **Breaking changes can be added in additional information section** if any.
 - **AI assistance in commits may optionally be disclosed with an `Assisted-by:` trailer** naming the model (e.g. `Assisted-by: Claude Opus 4.5`) — do NOT use a `Co-authored-by:` trailer for this.
-- **Keep the commit body short and human readable** focused on decision made if any. Commit body must not re-iterate the changes done, skip if title is sufficient in conveying the change.
-- **Use convensional commit format** for commits and PR title.
-- **Do not amend the commits once pushed.** Always create a new commit once changes are pushed to remote.

.gitignore

@@ -232,4 +232,3 @@ pyrightconfig.json
 # dev
 .dev/
 .claude/worktrees/
-.claude/settings.local.json

Makefile

@@ -81,13 +81,10 @@ devenv-clickhouse-clean: ## Clean all ClickHouse data from filesystem
 ##############################################################
 # go commands
 ##############################################################
-SIGNOZ_SQLSTORE_SQLITE_PATH ?= signoz.db
-SIGNOZ_APISERVER_ADDRESS    ?= 0.0.0.0:8080
-
 .PHONY: go-run-enterprise
 go-run-enterprise: ## Runs the enterprise go backend server
 	@SIGNOZ_INSTRUMENTATION_LOGS_LEVEL=debug \
-	SIGNOZ_SQLSTORE_SQLITE_PATH=$(SIGNOZ_SQLSTORE_SQLITE_PATH) \
+	SIGNOZ_SQLSTORE_SQLITE_PATH=signoz.db \
 	SIGNOZ_WEB_ENABLED=false \
 	SIGNOZ_TOKENIZER_JWT_SECRET=secret \
 	SIGNOZ_ALERTMANAGER_PROVIDER=signoz \
@@ -104,7 +101,7 @@ go-test: ## Runs go unit tests
 .PHONY: go-run-community
 go-run-community: ## Runs the community go backend server
 	@SIGNOZ_INSTRUMENTATION_LOGS_LEVEL=debug \
-	SIGNOZ_SQLSTORE_SQLITE_PATH=$(SIGNOZ_SQLSTORE_SQLITE_PATH) \
+	SIGNOZ_SQLSTORE_SQLITE_PATH=signoz.db \
 	SIGNOZ_WEB_ENABLED=false \
 	SIGNOZ_TOKENIZER_JWT_SECRET=secret \
 	SIGNOZ_ALERTMANAGER_PROVIDER=signoz \
@@ -114,28 +111,6 @@ go-run-community: ## Runs the community go backend server
 	go run -race \
 		$(GO_BUILD_CONTEXT_COMMUNITY)/*.go server
 
-.PHONY: go-stop
-go-stop: ## Stops the go backend server listening on SIGNOZ_APISERVER_ADDRESS, waiting for it to release every port it holds
-	@PORT=$(lastword $(subst :, ,$(SIGNOZ_APISERVER_ADDRESS))); \
-	PIDS=$$(lsof -ti tcp:$$PORT); \
-	if [ -z "$$PIDS" ]; then \
-		echo "No signoz server running on port $$PORT."; \
-		echo "If it's running on a different port, rerun as: make go-stop SIGNOZ_APISERVER_ADDRESS=host:port"; \
-		exit 0; \
-	fi; \
-	kill $$PIDS 2>/dev/null; \
-	for i in $$(seq 1 10); do \
-		alive=$$(for p in $$PIDS; do kill -0 $$p 2>/dev/null && echo $$p; done); \
-		[ -z "$$alive" ] && break; \
-		sleep 1; \
-	done; \
-	alive=$$(for p in $$PIDS; do kill -0 $$p 2>/dev/null && echo $$p; done); \
-	if [ -n "$$alive" ]; then \
-		echo "Graceful shutdown did not finish in 10s, sending SIGKILL to $$alive"; \
-		kill -9 $$alive 2>/dev/null; \
-	fi; \
-	echo "Stopped signoz server on port $$PORT (pid $$PIDS)"
-
 .PHONY: go-build-community $(GO_BUILD_ARCHS_COMMUNITY)
 go-build-community: ## Builds the go backend server for community
 go-build-community: $(GO_BUILD_ARCHS_COMMUNITY)
@@ -266,8 +241,3 @@ semconv-generate: ## Regenerate semantic-convention families for Go and TypeScri
 gen-mocks:
 	@echo ">> Generating mocks"
 	@mockery --config .mockery.yml
-
-.PHONY: gen-openapi-specs
-gen-openapi-specs:
-	@go run cmd/enterprise/*.go generate openapi
-	cd frontend && pnpm generate:api && cd -

conf/example.yaml

@@ -138,18 +138,6 @@ sqlstore:
 
 ##################### APIServer #####################
 apiserver:
-  # The TCP address the API server listens on, in the form "host:port".
-  address: 0.0.0.0:8080
-  # Maximum duration for reading an entire request, including the body.
-  read_timeout: 60s
-  # Keep at 0; any value cuts off streaming endpoints (livetail, SSE, export_raw_data).
-  write_timeout: 0
-  # tls:
-  #   enabled: true
-  #   cert_file: /path/to/server.crt
-  #   key_file: /path/to/server.key
-  #   # Minimum TLS version: "1.2" or "1.3". Defaults to "1.2".
-  #   min_version: "1.2"
   timeout:
     # Default request timeout.
     default: 60s

docs/api/openapi.yml

@@ -3474,6 +3474,79 @@ components:
       - tags
       - spec
       type: object
+    DashboardtypesHeatmapAxes:
+      properties:
+        yScale:
+          $ref: '#/components/schemas/DashboardtypesHeatmapYScale'
+      type: object
+    DashboardtypesHeatmapChartAppearance:
+      properties:
+        colors:
+          $ref: '#/components/schemas/DashboardtypesHeatmapColors'
+      type: object
+    DashboardtypesHeatmapColorMode:
+      enum:
+      - palette
+      - opacity
+      type: string
+    DashboardtypesHeatmapColorScale:
+      enum:
+      - log
+      - sqrt
+      - linear
+      type: string
+    DashboardtypesHeatmapColors:
+      properties:
+        fill:
+          type: string
+        maxCount:
+          nullable: true
+          type: number
+        minCount:
+          nullable: true
+          type: number
+        mode:
+          $ref: '#/components/schemas/DashboardtypesHeatmapColorMode'
+        palette:
+          $ref: '#/components/schemas/DashboardtypesHeatmapPalette'
+        scale:
+          $ref: '#/components/schemas/DashboardtypesHeatmapColorScale'
+        steps:
+          type: integer
+      type: object
+    DashboardtypesHeatmapPalette:
+      enum:
+      - ice
+      - moss
+      - rust
+      - graphite
+      - ember
+      - lagoon
+      - orchid
+      - verdant
+      - lava
+      - beacon
+      type: string
+    DashboardtypesHeatmapPanelSpec:
+      properties:
+        axes:
+          $ref: '#/components/schemas/DashboardtypesHeatmapAxes'
+        chartAppearance:
+          $ref: '#/components/schemas/DashboardtypesHeatmapChartAppearance'
+        formatting:
+          $ref: '#/components/schemas/DashboardtypesPanelFormatting'
+        legend:
+          $ref: '#/components/schemas/DashboardtypesLegend'
+        visualization:
+          $ref: '#/components/schemas/DashboardtypesBasicVisualization'
+      type: object
+    DashboardtypesHeatmapYScale:
+      enum:
+      - auto
+      - linear
+      - log
+      - symlog
+      type: string
     DashboardtypesHistogramBuckets:
       properties:
         bucketCount:
@@ -3826,6 +3899,7 @@ components:
       discriminator:
         mapping:
           signoz/BarChartPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesBarChartPanelSpec'
+          signoz/HeatmapPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec'
           signoz/HistogramPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec'
           signoz/ListPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec'
           signoz/NumberPanel: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesNumberPanelSpec'
@@ -3841,6 +3915,7 @@ components:
       - $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTablePanelSpec'
       - $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec'
       - $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec'
+      - $ref: '#/components/schemas/DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec'
       type: object
     DashboardtypesPanelPluginKind:
       enum:
@@ -3851,6 +3926,7 @@ components:
       - signoz/TablePanel
       - signoz/HistogramPanel
       - signoz/ListPanel
+      - signoz/HeatmapPanel
       type: string
     DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesBarChartPanelSpec:
       properties:
@@ -3864,6 +3940,18 @@ components:
       - kind
       - spec
       type: object
+    DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec:
+      properties:
+        kind:
+          enum:
+          - signoz/HeatmapPanel
+          type: string
+        spec:
+          $ref: '#/components/schemas/DashboardtypesHeatmapPanelSpec'
+      required:
+      - kind
+      - spec
+      type: object
     DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec:
       properties:
         kind:
@@ -7395,10 +7483,7 @@ components:
             $ref: '#/components/schemas/Querybuildertypesv5TimeSeries'
           type: array
         meta:
-          properties:
-            unit:
-              type: string
-          type: object
+          $ref: '#/components/schemas/Querybuildertypesv5AggregationMeta'
         predictedSeries:
           items:
             $ref: '#/components/schemas/Querybuildertypesv5TimeSeries'
@@ -7413,12 +7498,51 @@ components:
             $ref: '#/components/schemas/Querybuildertypesv5TimeSeries'
           type: array
       type: object
-    Querybuildertypesv5Bucket:
+    Querybuildertypesv5AggregationMeta:
       properties:
-        step:
-          format: double
-          type: number
+        buckets:
+          items:
+            format: double
+            type: number
+          type: array
+        unit:
+          type: string
       type: object
+    Querybuildertypesv5BucketOptions:
+      discriminator:
+        mapping:
+          linear: '#/components/schemas/Querybuildertypesv5BucketOptionsLinear'
+          log: '#/components/schemas/Querybuildertypesv5BucketOptionsLog'
+        propertyName: kind
+      oneOf:
+      - $ref: '#/components/schemas/Querybuildertypesv5BucketOptionsLinear'
+      - $ref: '#/components/schemas/Querybuildertypesv5BucketOptionsLog'
+      type: object
+    Querybuildertypesv5BucketOptionsLinear:
+      properties:
+        kind:
+          $ref: '#/components/schemas/Querybuildertypesv5BucketsKind'
+        spec:
+          $ref: '#/components/schemas/Querybuildertypesv5LinearBucketsSpec'
+      required:
+      - kind
+      - spec
+      type: object
+    Querybuildertypesv5BucketOptionsLog:
+      properties:
+        kind:
+          $ref: '#/components/schemas/Querybuildertypesv5BucketsKind'
+        spec:
+          $ref: '#/components/schemas/Querybuildertypesv5LogBucketsSpec'
+      required:
+      - kind
+      - spec
+      type: object
+    Querybuildertypesv5BucketsKind:
+      enum:
+      - linear
+      - log
+      type: string
     Querybuildertypesv5BuilderQuerySpec:
       discriminator:
         mapping:
@@ -7599,6 +7723,16 @@ components:
         value:
           type: string
       type: object
+    Querybuildertypesv5LinearBucketsSpec:
+      properties:
+        maxValue:
+          format: double
+          type: number
+        numBuckets:
+          type: integer
+      required:
+      - maxValue
+      type: object
     Querybuildertypesv5LogAggregation:
       properties:
         alias:
@@ -7606,6 +7740,12 @@ components:
         expression:
           type: string
       type: object
+    Querybuildertypesv5LogBucketsSpec:
+      properties:
+        scale:
+          nullable: true
+          type: integer
+      type: object
     Querybuildertypesv5MetricAggregation:
       properties:
         comparisonSpaceAggregationParam:
@@ -7686,6 +7826,8 @@ components:
       type: object
     Querybuildertypesv5QueryBuilderFormula:
       properties:
+        bucketOptions:
+          $ref: '#/components/schemas/Querybuildertypesv5BucketOptions'
         disabled:
           type: boolean
         expression:
@@ -7716,6 +7858,8 @@ components:
             $ref: '#/components/schemas/Querybuildertypesv5LogAggregation'
           nullable: true
           type: array
+        bucketOptions:
+          $ref: '#/components/schemas/Querybuildertypesv5BucketOptions'
         cursor:
           type: string
         disabled:
@@ -7777,6 +7921,8 @@ components:
             $ref: '#/components/schemas/Querybuildertypesv5MetricAggregation'
           nullable: true
           type: array
+        bucketOptions:
+          $ref: '#/components/schemas/Querybuildertypesv5BucketOptions'
         cursor:
           type: string
         disabled:
@@ -7838,6 +7984,8 @@ components:
             $ref: '#/components/schemas/Querybuildertypesv5TraceAggregation'
           nullable: true
           type: array
+        bucketOptions:
+          $ref: '#/components/schemas/Querybuildertypesv5BucketOptions'
         cursor:
           type: string
         disabled:
@@ -8163,6 +8311,7 @@ components:
       - raw
       - raw_stream
       - trace
+      - heatmap
       type: string
     Querybuildertypesv5ScalarData:
       properties:
@@ -8237,8 +8386,6 @@ components:
       type: object
     Querybuildertypesv5TimeSeriesValue:
       properties:
-        bucket:
-          $ref: '#/components/schemas/Querybuildertypesv5Bucket'
         partial:
           type: boolean
         timestamp:

docs/contributing/development.md

@@ -83,13 +83,7 @@ This command:
 
    You should see: `{"status":"ok"}`
 
-3. Stop it when you're done:
-   ```bash
-   make go-stop
-   ```
-
-> 💡 **Tip**: The API server runs at `http://localhost:8080/` by default. You can configure this using `apiserver.address` configuration option. See
-> [running more than one instance](#how-do-i-run-more-than-one-instance) if you need that for agentic testing.
+> 💡 **Tip**: The API server runs at `http://localhost:8080/` by default
 
 ### 4. Setting up the Frontend
 
@@ -125,36 +119,6 @@ To verify everything is working correctly:
 3. **Check Backend**: `curl http://localhost:8080/api/v1/health` (should return `{"status":"ok"}`)
 4. **Check Frontend**: Open `http://localhost:3301` in your browser
 
-## How do I run more than one instance?
-
-Handy when you keep several branches checked out as separate git worktrees. Every port
-and path below is read from the environment, so set them on the `make` call:
-
-```bash
-SIGNOZ_APISERVER_ADDRESS=0.0.0.0:8081 \
-SIGNOZ_SQLSTORE_SQLITE_PATH=/path/to/main/sqlite.db \
-SIGNOZ_INSTRUMENTATION_METRICS_READERS_PULL_EXPORTER_PROMETHEUS_PORT=9091 \
-make go-run-community
-```
-
-| Variable | Default | Why you'd change it |
-| --- | --- | --- |
-| `SIGNOZ_APISERVER_ADDRESS` | `0.0.0.0:8080` | Address the API server listens on |
-| `SIGNOZ_SQLSTORE_SQLITE_PATH` | `signoz.db` in worktree | To reuse same database |
-| `SIGNOZ_INSTRUMENTATION_METRICS_READERS_PULL_EXPORTER_PROMETHEUS_PORT` | `9090` | Bound by the Prometheus metrics exporter on startup |
-
-Point the frontend at whichever backend you want, in `frontend/.env`:
-
-```env
-VITE_FRONTEND_API_ENDPOINT=http://localhost:8081
-```
-
-Stop an instance using the address it was started on:
-
-```bash
-make go-stop SIGNOZ_APISERVER_ADDRESS=0.0.0.0:8081
-```
-
 ## How to send test data?
 
 You can now send telemetry data to your local SigNoz instance:

docs/contributing/go/provider.md

@@ -17,7 +17,7 @@ For example, the [prometheus](/pkg/prometheus) provider delivers a prometheus en
 
 - `pkg/prometheus/prometheus.go` - Interface definition
 - `pkg/prometheus/config.go` - Configuration
-- `pkg/prometheus/clickhouseprometheusv2/provider.go` - Clickhouse-powered implementation
+- `pkg/prometheus/clickhouseprometheus/provider.go` - Clickhouse-powered implementation
 - `pkg/prometheus/prometheustest/provider.go` - Mock implementation
 
 ## How to wire it up?

docs/contributing/go/service.md

@@ -191,7 +191,7 @@ A standalone service only has the `factory.Service` lifecycle i.e it does not se
         // ... dependencies ...
     ) user.Service {
         return &service{
-            settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/user"),
+            settings: factory.NewScopedProviderSettings(providerSettings, "go.signoz.io/pkg/modules/user"),
             // ... dependencies ...
             stopC:    make(chan struct{}),
         }

docs/contributing/prometheus.md

@@ -9,16 +9,15 @@ change breaks an invariant, flag it and discuss it first.
 
 ---
 
-## Why the provider looks like this
+## Why a second provider
 
-The removed v1 provider served the promql engine through the remote-read
-protobuf adapter. It fetched every raw sample of a query's union window,
-serialized all of them, and gave them to the engine. The cost followed the
-ingested data, not the question. This is how a dashboard of PromQL panels
-could take an instance down. v2 replaced it after a byte-level parity
-rollout, and v1 was then deleted.
+The v1 provider (`pkg/prometheus/clickhouseprometheus`) serves the promql
+engine through the remote-read protobuf adapter. It fetches every raw sample
+of a query's union window. It serializes all of them and gives them to the
+engine. The cost follows the ingested data, not the question. This is how a
+dashboard of PromQL panels can take an instance down.
 
-Each query runs in one of two ways. The classifier decides per query:
+In v2, each query runs in one of two ways. The classifier decides per query:
 
 - **Transpiled**: ClickHouse evaluates the query. Only final (or near-final)
   per-group grid arrays come back. The statements use the
@@ -31,7 +30,7 @@ Each query runs in one of two ways. The classifier decides per query:
 lost user. A construct that cannot reproduce engine semantics exactly falls
 back. It does not approximate.** The conformance suite
 (`tests/integration/tests/promqlconformance/`) replays Prometheus' own test
-corpus against the provider. It is the arbiter. The classification golden
+corpus against both providers. It is the arbiter. The classification golden
 (`testdata/classification_golden.json`) freezes the route of each corpus
 expression. The rest of this document is the PromQL-to-SQL story. That
 mapping is where correctness is won or lost.
@@ -264,8 +263,7 @@ per-thread partials scaled memory with the thread count. The slide then
 combines each slot's at-most-W bucket partials by direct aggregation
 (`arraySum(arraySlice(...))`). Window sums are added the way the engine adds
 them. There is no prefix-sum differencing: its large-minus-large
-cancellation would drift past the conformance tolerance on counter-sized
-values.
+cancellation would drift past the shadow tolerance on counter-sized values.
 This is correct per slot because the bucket union is the exact window
 multiset, and avg/min/max/sum/count are order-insensitive on a multiset
 (sum/avg up to summation order; see the float caveat above). A slot with
@@ -335,7 +333,7 @@ can carry them.
 ## The engine path
 
 Queries that do not transpile run in the stock engine over this package's
-`storage.Querier`. Samples are fetched per
+`storage.Querier`. This is still not the v1 path. Samples are fetched per
 selector with the engine's per-selector hints, not the query-wide union
 window. So `foo / foo offset 1d` reads two narrow windows, not the widest
 one twice. Instant selectors of subquery-free queries fetch only the last
@@ -369,9 +367,9 @@ same predicates as a shard-local semi-join, not a GLOBAL broadcast of the
 matched set. The temporality filter on every samples statement is a
 semantic no-op: the matched fingerprints already come from those
 temporalities. It engages the leading samples primary-key column.
-Delta-temporality series stay invisible to PromQL here, as they were before
-v2. To make Delta visible is its own change with its own semantics to
-design. A Delta stream fed to `rate()`
+Delta-temporality series stay invisible to PromQL here, exactly as in v1.
+The rollout gate is parity with v1. To make Delta visible is its own change
+with its own semantics to design. A Delta stream fed to `rate()`
 as-if-cumulative would be wrong, not just new.
 
 ## Observability

ee/query-service/app/server.go

@@ -3,29 +3,56 @@ package app
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/http"
+	"slices"
+
+	"go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
+	"go.opentelemetry.io/otel/propagation"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+
+	"github.com/gorilla/handlers"
+
+	"github.com/rs/cors"
+	"github.com/soheilhy/cmux"
 
 	"github.com/SigNoz/signoz/ee/query-service/app/api"
 	"github.com/SigNoz/signoz/ee/query-service/usage"
 	"github.com/SigNoz/signoz/pkg/http/middleware"
 	"github.com/SigNoz/signoz/pkg/signoz"
+	"github.com/SigNoz/signoz/pkg/web"
 
 	"log/slog"
 
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
+	baseapp "github.com/SigNoz/signoz/pkg/query-service/app"
 	"github.com/SigNoz/signoz/pkg/query-service/app/clickhouseReader"
 	"github.com/SigNoz/signoz/pkg/query-service/app/integrations"
 	"github.com/SigNoz/signoz/pkg/query-service/app/logparsingpipeline"
 	"github.com/SigNoz/signoz/pkg/query-service/app/opamp"
 	opAmpModel "github.com/SigNoz/signoz/pkg/query-service/app/opamp/model"
 	baseconst "github.com/SigNoz/signoz/pkg/query-service/constants"
+	"github.com/SigNoz/signoz/pkg/query-service/healthcheck"
+	"github.com/SigNoz/signoz/pkg/query-service/utils"
 )
 
-// Server runs auxiliary servers (opamp) alongside the signoz apiserver
+// Server runs HTTP, Mux and a grpc server
 type Server struct {
+	config signoz.Config
+	signoz *signoz.SigNoz
+
+	// public http router
+	httpConn     net.Listener
+	httpServer   *http.Server
+	httpHostPort string
+
 	opampServer *opamp.Server
 
 	// Usage manager
 	usageManager *usage.Manager
+
+	unavailableChannel chan healthcheck.Status
 }
 
 // NewServer creates and initializes Server
@@ -100,11 +127,57 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 		return nil, err
 	}
 
-	// Register the legacy query-service routes on the apiserver router. The
-	// apiserver owns the HTTP server and applies the middleware chain at serve
-	// time, so these routes get the same treatment as the apiserver routes.
-	r := signoz.APIServer.Router()
-	am := middleware.NewAuthZ(signoz.Instrumentation.Logger(), signoz.Modules.OrgGetter, signoz.Authz)
+	s := &Server{
+		config:             config,
+		signoz:             signoz,
+		httpHostPort:       baseconst.HTTPHostPort,
+		unavailableChannel: make(chan healthcheck.Status),
+		usageManager:       usageManager,
+	}
+
+	httpServer, err := s.createPublicServer(apiHandler, signoz.Web)
+
+	if err != nil {
+		return nil, err
+	}
+
+	s.httpServer = httpServer
+
+	s.opampServer = opamp.InitializeServer(
+		&opAmpModel.AllAgents, agentConfMgr, signoz.Instrumentation,
+	)
+
+	return s, nil
+}
+
+// HealthCheckStatus returns health check status channel a client can subscribe to
+func (s Server) HealthCheckStatus() chan healthcheck.Status {
+	return s.unavailableChannel
+}
+
+func (s *Server) createPublicServer(apiHandler *api.APIHandler, web web.Web) (*http.Server, error) {
+	r := baseapp.NewRouter()
+	am := middleware.NewAuthZ(s.signoz.Instrumentation.Logger(), s.signoz.Modules.OrgGetter, s.signoz.Authz)
+
+	r.Use(middleware.NewRecovery(s.signoz.Instrumentation.Logger()).Wrap)
+	r.Use(otelmux.Middleware(
+		"apiserver",
+		otelmux.WithMeterProvider(s.signoz.Instrumentation.MeterProvider()),
+		otelmux.WithTracerProvider(s.signoz.Instrumentation.TracerProvider()),
+		otelmux.WithPropagators(propagation.NewCompositeTextMapPropagator(propagation.Baggage{}, propagation.TraceContext{})),
+		otelmux.WithFilter(func(r *http.Request) bool {
+			return !slices.Contains([]string{"/api/v1/health"}, r.URL.Path)
+		}),
+	))
+	r.Use(middleware.NewIdentN(s.signoz.IdentNResolver, s.signoz.Sharder, s.signoz.Instrumentation.Logger()).Wrap)
+	r.Use(middleware.NewTimeout(s.signoz.Instrumentation.Logger(),
+		s.config.APIServer.Timeout.ExcludedRoutes,
+		s.config.APIServer.Timeout.Default,
+		s.config.APIServer.Timeout.Max,
+	).Wrap)
+	r.Use(middleware.NewResource(s.signoz.Instrumentation.Logger()).Wrap)
+	r.Use(middleware.NewAudit(s.signoz.Instrumentation.Logger(), s.config.APIServer.Logging.ExcludedRoutes, s.signoz.Auditor).Wrap)
+	r.Use(middleware.NewComment().Wrap)
 
 	apiHandler.RegisterRoutes(r, am)
 	apiHandler.RegisterLogsRoutes(r, am)
@@ -115,29 +188,107 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 	apiHandler.RegisterThirdPartyApiRoutes(r, am)
 	apiHandler.RegisterTraceFunnelsRoutes(r, am)
 
-	s := &Server{
-		usageManager: usageManager,
+	err := s.signoz.APIServer.AddToRouter(r)
+	if err != nil {
+		return nil, err
 	}
 
-	s.opampServer = opamp.InitializeServer(
-		&opAmpModel.AllAgents, agentConfMgr, signoz.Instrumentation,
-	)
+	c := cors.New(cors.Options{
+		AllowedOrigins: []string{"*"},
+		AllowedMethods: []string{"GET", "DELETE", "POST", "PUT", "PATCH", "OPTIONS"},
+		AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "cache-control"},
+	})
 
-	return s, nil
+	handler := c.Handler(r)
+
+	handler = handlers.CompressHandler(handler)
+
+	err = web.AddToRouter(r)
+	if err != nil {
+		return nil, err
+	}
+
+	routePrefix := s.config.Global.ExternalPath()
+	if routePrefix != "" {
+		prefixed := http.StripPrefix(routePrefix, handler)
+		handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			switch req.URL.Path {
+			case "/api/v1/health", "/api/v2/healthz", "/api/v2/readyz", "/api/v2/livez":
+				r.ServeHTTP(w, req)
+				return
+			}
+
+			prefixed.ServeHTTP(w, req)
+		})
+	}
+
+	return &http.Server{
+		Handler: handler,
+	}, nil
 }
 
-// Start starts the opamp websocket server. The HTTP API server is started by
-// the signoz registry.
-func (s *Server) Start(ctx context.Context) error {
-	slog.Info("Starting OpAmp Websocket server", "addr", baseconst.OpAmpWsEndpoint)
-	if err := s.opampServer.Start(baseconst.OpAmpWsEndpoint); err != nil {
+// initListeners initialises listeners of the server
+func (s *Server) initListeners() error {
+	// listen on public port
+	var err error
+	publicHostPort := s.httpHostPort
+	if publicHostPort == "" {
+		return fmt.Errorf("baseconst.HTTPHostPort is required")
+	}
+
+	s.httpConn, err = net.Listen("tcp", publicHostPort)
+	if err != nil {
 		return err
 	}
 
+	slog.Info(fmt.Sprintf("Query server started listening on %s...", s.httpHostPort))
+
+	return nil
+}
+
+// Start listening on http and private http port concurrently
+func (s *Server) Start(ctx context.Context) error {
+	err := s.initListeners()
+	if err != nil {
+		return err
+	}
+
+	var httpPort int
+	if port, err := utils.GetPort(s.httpConn.Addr()); err == nil {
+		httpPort = port
+	}
+
+	go func() {
+		slog.Info("Starting HTTP server", "port", httpPort, "addr", s.httpHostPort)
+
+		switch err := s.httpServer.Serve(s.httpConn); err {
+		case nil, http.ErrServerClosed, cmux.ErrListenerClosed:
+			// normal exit, nothing to do
+		default:
+			slog.Error("Could not start HTTP server", errors.Attr(err))
+		}
+		s.unavailableChannel <- healthcheck.Unavailable
+	}()
+
+	go func() {
+		slog.Info("Starting OpAmp Websocket server", "addr", baseconst.OpAmpWsEndpoint)
+		err := s.opampServer.Start(baseconst.OpAmpWsEndpoint)
+		if err != nil {
+			slog.Error("opamp ws server failed to start", errors.Attr(err))
+			s.unavailableChannel <- healthcheck.Unavailable
+		}
+	}()
+
 	return nil
 }
 
 func (s *Server) Stop(ctx context.Context) error {
+	if s.httpServer != nil {
+		if err := s.httpServer.Shutdown(ctx); err != nil {
+			return err
+		}
+	}
+
 	s.opampServer.Stop()
 
 	// stop usage manager

ee/query-service/rules/manager_test.go

@@ -160,7 +160,7 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 			triggeredTestAlerts := []map[*alertmanagertypes.PostableAlert][]string{}
 
 			// Variable to store promProvider for cleanup
-			var promProvider prometheus.Prometheus
+			var promProvider *prometheustest.Provider
 
 			// Create manager using test factory with hooks
 			mgr := rules.NewTestManager(t, &rules.TestManagerOptions{
@@ -185,29 +185,76 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 				TelemetryStoreHook: func(store telemetrystore.TelemetryStore) {
 					mockStore := store.(*telemetrystoretest.Provider)
 
-					// Grid the TestNotification eval computes over (see
-					// Timestamps on base_rule); nil args match any window.
+					// Set up Prometheus-specific mock data
+					// Fingerprint columns for Prometheus queries
+					fingerprintCols := []cmock.ColumnType{
+						{Name: "fingerprint", Type: "UInt64"},
+						{Name: "any(labels)", Type: "String"},
+					}
+
+					// Samples columns for Prometheus queries
+					samplesCols := []cmock.ColumnType{
+						{Name: "metric_name", Type: "String"},
+						{Name: "fingerprint", Type: "UInt64"},
+						{Name: "unix_milli", Type: "Int64"},
+						{Name: "value", Type: "Float64"},
+						{Name: "flags", Type: "UInt32"},
+					}
+
+					// Calculate query time range similar to Prometheus rule tests
+					// TestNotification uses time.Now().UTC() for evaluation
+					// We calculate the query window based on current time to match what the actual evaluation will use
 					evalTime := baseTime
 					evalWindowMs := int64(5 * 60 * 1000) // 5 minutes in ms
-					gridEnd := (evalTime.UnixMilli() / 60000) * 60000
-					gridStart := gridEnd - evalWindowMs
+					evalTimeMs := evalTime.UnixMilli()
+					queryStart := ((evalTimeMs-2*evalWindowMs)/60000)*60000 + 1 // truncate to minute + 1ms
+					queryEnd := (evalTimeMs / 60000) * 60000                    // truncate to minute
 
-					tsList := make([]int64, 0, len(tc.Values))
-					vList := make([]float64, 0, len(tc.Values))
+					// Create fingerprint data
+					fingerprint := uint64(12345)
+					labelsJSON := `{"__name__":"test_metric"}`
+					fingerprintData := [][]interface{}{
+						{fingerprint, labelsJSON},
+					}
+					fingerprintRows := cmock.NewRows(fingerprintCols, fingerprintData)
+
+					// Create samples data from test case values, calculating timestamps relative to baseTime
+					validSamplesData := make([][]interface{}, 0)
 					for _, v := range tc.Values {
 						// Skip NaN and Inf values in the samples data
 						if math.IsNaN(v.Value) || math.IsInf(v.Value, 0) {
 							continue
 						}
-						tsList = append(tsList, baseTime.Add(v.Offset).UnixMilli())
-						vList = append(vList, v.Value)
+						// Calculate timestamp relative to baseTime
+						sampleTimestamp := baseTime.Add(v.Offset).UnixMilli()
+						validSamplesData = append(validSamplesData, []interface{}{
+							"test_metric",
+							fingerprint,
+							sampleTimestamp,
+							v.Value,
+							uint32(0), // flags - 0 means normal value
+						})
 					}
-					grid := prometheustest.LastSampleGrid(tsList, vList, gridStart, gridEnd, 60_000, 300_000)
+					samplesRows := cmock.NewRows(samplesCols, validSamplesData)
 
 					mock := mockStore.Mock()
-					mock.ExpectQuery("SELECT gkey").
-						WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-						WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{{`[["__name__","test_metric"]]`, grid}}))
+
+					// Mock the fingerprint query (for Prometheus label matching)
+					// args: $1=metric_name (the __name__ matcher maps onto the column)
+					mock.ExpectQuery("SELECT fingerprint, any").
+						WithArgs("test_metric").
+						WillReturnRows(fingerprintRows)
+
+					// Mock the samples query (for Prometheus metric data)
+					// args: metric_name IN (discovered names), subquery metric_name, start, end
+					mock.ExpectQuery("SELECT metric_name, fingerprint, unix_milli").
+						WithArgs(
+							"test_metric",
+							"test_metric",
+							queryStart,
+							queryEnd,
+						).
+						WillReturnRows(samplesRows)
 
 					// Create Prometheus provider for this test
 					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, store)
@@ -242,6 +289,7 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 				assert.Empty(t, triggeredTestAlerts)
 			}
 
+			promProvider.Close()
 		})
 	}
 }

frontend/src/constants/queryBuilder.ts

@@ -614,6 +614,18 @@ export const listViewInitialLogQuery: Query = {
 	},
 };
 
+export const PANEL_TYPES_INITIAL_QUERY: Record<PANEL_TYPES, Query> = {
+	[PANEL_TYPES.TIME_SERIES]: initialQueriesMap.metrics,
+	[PANEL_TYPES.VALUE]: initialQueriesMap.metrics,
+	[PANEL_TYPES.TABLE]: initialQueriesMap.metrics,
+	[PANEL_TYPES.LIST]: listViewInitialLogQuery,
+	[PANEL_TYPES.TRACE]: initialQueriesMap.traces,
+	[PANEL_TYPES.BAR]: initialQueriesMap.metrics,
+	[PANEL_TYPES.PIE]: initialQueriesMap.metrics,
+	[PANEL_TYPES.HISTOGRAM]: initialQueriesMap.metrics,
+	[PANEL_TYPES.EMPTY_WIDGET]: initialQueriesMap.metrics,
+};
+
 export const listViewInitialTraceQuery: Query = {
 	// it should be the above commented query
 	...initialQueriesMap.traces,

frontend/src/pages/DashboardPage/DashboardContainer/Panels/utils/drilldown/__tests__/drilldown.test.ts

@@ -426,30 +426,6 @@ describe('resolvePanelContextLinks', () => {
 
 		expect(resolved[0].url).toBe('https://wiki/{{_service.name}}');
 	});
-
-	it('carries targetBlank through, defaulting to true when unset', () => {
-		const resolved = resolvePanelContextLinks(
-			[
-				{ name: 'Same tab', url: 'https://wiki/a', targetBlank: false },
-				{ name: 'New tab', url: 'https://wiki/b', targetBlank: true },
-				{ name: 'Unset', url: 'https://wiki/c' },
-				{
-					name: 'Literal',
-					url: 'https://wiki/d',
-					targetBlank: false,
-					renderVariables: false,
-				},
-			],
-			{},
-		);
-
-		expect(resolved.map((link) => link.targetBlank)).toStrictEqual([
-			false,
-			true,
-			true,
-			false,
-		]);
-	});
 });
 
 describe('stepClickTimeRange', () => {

frontend/src/pages/DashboardPage/DashboardContainer/Panels/utils/drilldown/resolvePanelContextLinks.ts

@@ -8,8 +8,6 @@ export interface ResolvedDrilldownLink {
 	id: string;
 	label: string;
 	url: string;
-	/** Opens in a new tab; links saved before the toggle existed default to true. */
-	targetBlank: boolean;
 }
 
 /**
@@ -28,16 +26,14 @@ export function resolvePanelContextLinks(
 	return usable.map((link, index) => {
 		const rawLabel = link.name || link.url || '';
 		const rawUrl = link.url ?? '';
-		const targetBlank = link.targetBlank ?? true;
 		// Only an explicit `false` opts out; undefined defaults to substitution on.
 		if (link.renderVariables === false) {
-			return { id: String(index), label: rawLabel, url: rawUrl, targetBlank };
+			return { id: String(index), label: rawLabel, url: rawUrl };
 		}
 		return {
 			id: String(index),
 			label: resolveTexts({ texts: [rawLabel], processedVariables }).fullTexts[0],
 			url: resolveContextLinkUrl(rawUrl, processedVariables),
-			targetBlank,
 		};
 	});
 }

frontend/src/pages/DashboardPage/DashboardContainer/PanelsAndSectionsLayout/Panel/DrilldownMenu/DrilldownAggregateMenu.tsx

@@ -173,7 +173,7 @@ function DrilldownAggregateMenu({
 						void logEvent(DashboardDetailEvents.DrilldownAction, {
 							action: 'contextLink',
 						});
-						openInNewTab(link.url, !!link.targetBlank);
+						openInNewTab(link.url);
 						onClose();
 					}}
 				>

frontend/src/providers/QueryBuilder.tsx

@@ -767,15 +767,10 @@ export function QueryBuilderProvider({
 							queryItem.dataSource
 						].builder.queryData;
 
-					// `dataSource` travels with the panel type's fields, but is appended to a
-					// copy: `propsRequired` is the list held in
-					// `panelTypeDataSourceFormValuesMap`, and pushing onto it grew that
-					// module-level array by one entry on every call.
-					if (propsRequired) {
-						[...propsRequired, 'dataSource'].forEach((p: any) => {
-							set(queryItem, p, get(newQueryItem, p));
-						});
-					}
+					propsRequired?.push('dataSource');
+					propsRequired?.forEach((p: any) => {
+						set(queryItem, p, get(newQueryItem, p));
+					});
 					return queryItem;
 				}
 

frontend/src/types/common/queryBuilder.ts

@@ -211,11 +211,13 @@ export enum QueryFunctionsTypes {
 	FILL_ZERO = 'fillZero',
 }
 
-/**
- * Key names of {@link PANEL_TYPES}. Derived rather than listed: the hand-written
- * version had fallen behind the enum by three members (`BAR`, `PIE`, `HISTOGRAM`).
- */
-export type PanelTypeKeys = keyof typeof PANEL_TYPES;
+export type PanelTypeKeys =
+	| 'TIME_SERIES'
+	| 'VALUE'
+	| 'TABLE'
+	| 'LIST'
+	| 'TRACE'
+	| 'EMPTY_WIDGET';
 
 export enum ReduceOperators {
 	LAST = 'last',

frontend/src/utils/navigation.ts

@@ -1,9 +1,5 @@
 import { withBasePath } from 'utils/basePath';
 
-export const openInNewTab = (path: string, newTab = true): void => {
-	if (newTab) {
-		window.open(withBasePath(path), '_blank');
-	} else {
-		window.location.assign(withBasePath(path));
-	}
+export const openInNewTab = (path: string): void => {
+	window.open(withBasePath(path), '_blank');
 };

go.mod

@@ -57,6 +57,7 @@ require (
 	github.com/segmentio/analytics-go/v3 v3.2.1
 	github.com/sethvargo/go-password v0.2.0
 	github.com/smartystreets/goconvey v1.8.1
+	github.com/soheilhy/cmux v0.1.5
 	github.com/spf13/cobra v1.10.2
 	github.com/stretchr/testify v1.11.1
 	github.com/swaggest/jsonschema-go v0.3.78
@@ -131,6 +132,7 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/goccy/go-yaml v1.19.2 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/hashicorp/go-metrics v0.5.4 // indirect
 	github.com/huandu/go-clone v1.7.3 // indirect
@@ -138,7 +140,8 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/nxadm/tail v1.4.11 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.148.0 // indirect
+	github.com/prometheus/client_golang/exp v0.0.0-20260325093428-d8591d0db856 // indirect
+	github.com/puzpuzpuz/xsync/v4 v4.4.0 // indirect
 	github.com/redis/go-redis/extra/rediscmd/v9 v9.15.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/swaggest/refl v1.4.0 // indirect
@@ -149,6 +152,7 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/collector/internal/componentalias v0.148.0 // indirect
 	go.opentelemetry.io/collector/pdata/xpdata v0.148.0 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	golang.org/x/arch v0.20.0 // indirect
 	golang.org/x/term v0.43.0 // indirect
@@ -168,6 +172,10 @@ require (
 	cloud.google.com/go/auth v0.18.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
 	github.com/ClickHouse/ch-go v0.71.0
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Yiling-J/theine-go v0.6.2 // indirect
@@ -213,6 +221,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gojek/valkyrie v0.0.0-20180215180059-6aee720afcdf // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/cel-go v0.29.0 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
@@ -245,6 +254,7 @@ require (
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
 	github.com/klauspost/compress v1.18.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/leodido/go-syslog/v4 v4.3.0 // indirect
@@ -269,10 +279,14 @@ require (
 	github.com/oklog/ulid/v2 v2.1.1
 	github.com/open-feature/go-sdk v1.17.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.144.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.148.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.148.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.148.0 // indirect
 	github.com/openfga/openfga v1.14.1
 	github.com/paulmach/orb v0.12.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pierrec/lz4/v4 v4.1.25 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/pressly/goose/v3 v3.27.0 // indirect

go.sum

@@ -73,8 +73,16 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEB
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 h1:LkHbJbgF3YyvC53aqYGR+wWQDn2Rdp9AQdGndf9QvY4=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0/go.mod h1:QyiQdW4f4/BIfB8ZutZ2s+28RAgfa/pT+zS++ZHyM1I=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 h1:bXwSugBiSbgtz7rOtbfGf+woewp4f06orW9OP5BjHLA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0/go.mod h1:Y/HgrePTmGy9HjdSGTqZNa+apUpTVIEVKXJyARP2lrk=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
+github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -83,6 +91,8 @@ github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoy
 github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw=
 github.com/ClickHouse/clickhouse-go/v2 v2.44.0 h1:9pxs5pRwIvhni5BDRPn/n5A8DeUod5TnBaeulFBX8EQ=
 github.com/ClickHouse/clickhouse-go/v2 v2.44.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c=
+github.com/Code-Hex/go-generics-cache v1.5.1 h1:6vhZGc5M7Y/YD8cIUcY8kcuQLB4cHR7U+0KMqAA0KcU=
+github.com/Code-Hex/go-generics-cache v1.5.1/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4=
 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
 github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
@@ -152,11 +162,23 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
 github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0 h1:98Miqj16un1WLNyM1RjVDhXYumhqZrQfAeG8i4jPG6o=
+github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0/go.mod h1:T6ndRfdhnXLIY5oKBHjYZDVj706los2zGdpThppquvA=
+github.com/aws/aws-sdk-go-v2/service/ecs v1.74.0 h1:YS5TXaEvzDb+sV+wdQFUtuCAk0GeFR9Ai6HFdxpz6q8=
+github.com/aws/aws-sdk-go-v2/service/ecs v1.74.0/go.mod h1:10kBgdaNJz0FO/+JWDUH+0rtSjkn5yafgavDDmmhFzs=
+github.com/aws/aws-sdk-go-v2/service/elasticache v1.51.12 h1:S066ajzfPRCSW4lsSHOYglne6SNi2CHt1u5omzW1RBg=
+github.com/aws/aws-sdk-go-v2/service/elasticache v1.51.12/go.mod h1:86SE4NcXxbxr8KTG3yOyDmd4HyiFmKl8TexXnhYJ+Bw=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
+github.com/aws/aws-sdk-go-v2/service/kafka v1.49.1 h1:BgBatWcQIFqF1l6KGHjv66V0d/ISnWrTwxDx/Jf6EJM=
+github.com/aws/aws-sdk-go-v2/service/kafka v1.49.1/go.mod h1:pMpys+PlrN//vj8j5s0oOAMJjauj81VkHzIZxPVWOro=
+github.com/aws/aws-sdk-go-v2/service/lightsail v1.51.0 h1:cg6PxzoIide2wiEyLfikOFN+XwHafwR8p5+L9U1E8dQ=
+github.com/aws/aws-sdk-go-v2/service/lightsail v1.51.0/go.mod h1:YvX7hjUWecrKX8fBkbEncyddEW85xjNH+u5JHioITOw=
+github.com/aws/aws-sdk-go-v2/service/rds v1.117.0 h1:T1Xe9sYxSUUQOvd1RsFeVk/IXFPdqSiN0atXu/Hy/8A=
+github.com/aws/aws-sdk-go-v2/service/rds v1.117.0/go.mod h1:QbXW4coAMakHQhf1qhE0eVVCen9gwB/Kvn+HHHKhpGY=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
 github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
 github.com/aws/aws-sdk-go-v2/service/sns v1.39.11 h1:Ke7RS0NuP9Xwk31prXYcFGA1Qfn8QmNWcxyjKPcXZdc=
@@ -225,6 +247,8 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik=
+github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4=
 github.com/coder/quartz v0.3.0 h1:bUoSEJ77NBfKtUqv6CPSC0AS8dsjqAqqAv7bN02m1mg=
 github.com/coder/quartz v0.3.0/go.mod h1:BgE7DOj/8NfvRgvKw0jPLDQH/2Lya2kxcTaNJ8X0rZk=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -251,8 +275,12 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa5
 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/digitalocean/godo v1.178.0 h1:+B4xGOaoFwwwpM7TKhoyGHdmFg5eF9zDB1YfOLvNJ2E=
+github.com/digitalocean/godo v1.178.0/go.mod h1:xQsWpVCCbkDrWisHA72hPzPlnC+4W5w/McZY5ij9uvU=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
+github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
 github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -285,6 +313,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
+github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
+github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=
+github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds=
 github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
@@ -296,6 +327,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -405,12 +438,16 @@ github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHO
 github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
 github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw=
 github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0=
+github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
+github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
 github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
 github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
 github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
+github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
@@ -490,6 +527,8 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0=
+github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
 github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -532,6 +571,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
 github.com/googleapis/gax-go/v2 v2.18.0 h1:jxP5Uuo3bxm3M6gGtV94P4lliVetoCB4Wk2x8QA86LI=
 github.com/googleapis/gax-go/v2 v2.18.0/go.mod h1:uSzZN4a356eRG985CzJ3WfbFSpqkLTjsnhWGJR6EwrE=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
+github.com/gophercloud/gophercloud/v2 v2.11.1 h1:jCs4vLH8sJgRqrPzqVfWgl7uI6JnIIlsgeIRM0uHjxY=
+github.com/gophercloud/gophercloud/v2 v2.11.1/go.mod h1:Rm0YvKQ4QYX2rY9XaDKnjRzSGwlG5ge4h6ABYnmkKQM=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
 github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
@@ -553,7 +594,11 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
 github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0=
 github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ=
+github.com/hashicorp/consul/api v1.32.1 h1:0+osr/3t/aZNAdJX558crU3PEjVrG4x6715aZHRgceE=
+github.com/hashicorp/consul/api v1.32.1/go.mod h1:mXUWLnxftwTmDv4W3lzxYCPD199iNLLUyLfLGFJbtl4=
 github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
+github.com/hashicorp/cronexpr v1.1.3 h1:rl5IkxXN2m681EfivTlccqIryzYJSXRGRNa0xeG7NA4=
+github.com/hashicorp/cronexpr v1.1.3/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -565,6 +610,8 @@ github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9
 github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
+github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
+github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
@@ -583,6 +630,7 @@ github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es
 github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
 github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
 github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
@@ -610,12 +658,18 @@ github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/
 github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/hashicorp/memberlist v0.5.4 h1:40YY+3qq2tAUhZIMEK8kqusKZBBjdwJ3NUjvYkcxh74=
 github.com/hashicorp/memberlist v0.5.4/go.mod h1:OgN6xiIo6RlHUWk+ALjP9e32xWCoQrsOCmHrWCm2MWA=
+github.com/hashicorp/nomad/api v0.0.0-20260324203407-b27b0c2e019a h1:HGwfgBNl90YBiHdbzZ/+8aMxO1UL9B/yNTAXa8iB8z8=
+github.com/hashicorp/nomad/api v0.0.0-20260324203407-b27b0c2e019a/go.mod h1:KkLNLU0Nyfh5jWsFoF/PsmMbKpRIAoIV4lmQoJWgKCk=
 github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
 github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
+github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
+github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
 github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q=
 github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
+github.com/hetznercloud/hcloud-go/v2 v2.36.0 h1:HlLL/aaVXUulqe+rsjoJmrxKhPi1MflL5O9iq5QEtvo=
+github.com/hetznercloud/hcloud-go/v2 v2.36.0/go.mod h1:MnN/QJEa/RYNQiiVoJjNHPntM7Z1wlYPgJ2HA40/cDE=
 github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs=
 github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E=
 github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
@@ -635,6 +689,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/ionos-cloud/sdk-go/v6 v6.3.6 h1:l/TtKgdQ1wUH3DDe2SfFD78AW+TJWdEbDpQhHkWd6CM=
+github.com/ionos-cloud/sdk-go/v6 v6.3.6/go.mod h1:nUGHP4kZHAZngCVr4v6C8nuargFrtvt7GrzH/hqn7c4=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -671,6 +727,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
+github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
@@ -683,6 +741,8 @@ github.com/knadh/koanf v1.5.0 h1:q2TSd/3Pyc/5yP9ldIrSdIz26MCcyNQzW0pEAugLPNs=
 github.com/knadh/koanf v1.5.0/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs=
 github.com/knadh/koanf/v2 v2.3.3 h1:jLJC8XCRfLC7n4F+ZKKdBsbq1bfXTpuFhf4L7t94D94=
 github.com/knadh/koanf/v2 v2.3.3/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
+github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00=
+github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
@@ -710,6 +770,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b h1:11UHH39z1RhZ5dc4y4r/4koJo6IYFgTRMe/LlwRTEw0=
 github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg=
+github.com/linode/linodego v1.66.0 h1:rK8QJFaV53LWOEJvb/evhTg/dP5ElvtuZmx4iv4RJds=
+github.com/linode/linodego v1.66.0/go.mod h1:12ykGs9qsvxE+OU3SXuW2w+DTruWF35FPlXC7gGk2tU=
 github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
 github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
 github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
@@ -725,6 +787,8 @@ github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
@@ -751,6 +815,7 @@ github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXx
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -810,6 +875,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.144
 github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.144.0/go.mod h1:R0go5FMmUe51VpKl8YCk/rUxibA+U3lfPYMoihQ/nhw=
 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.144.0 h1:Qv3nLVGKJ9LQCGwxteJxjSNyQ5CP99QRvYPFn6d8Y60=
 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.144.0/go.mod h1:O2rZKRXk1WeYhzfJBVXES/g7+PlIds/TzPZW/4NfTNA=
+github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.148.0 h1:CiTjQE/Hh5xK2t56ogrDK4nl0+tJPNmASCs4zEYZ/xU=
+github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.148.0/go.mod h1:WUFkzTiOpt7EYyL67gv1GOf3RD8qKWGtin3lY9LYzW4=
 github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.144.0 h1:Ywu5mU4K5TMJigiXdyZloCRs/cq3/2OnoK3WjxNHWJo=
 github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.144.0/go.mod h1:iebqlu6UvpiV1hO37r1sXA9fXaCaA8sQXilG0///xss=
 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.144.0 h1:TMRTvQSAeeLtkKwSrqcbectxDRPiqB6yYM3IvjC75es=
@@ -822,6 +889,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.144.0 h1
 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.144.0/go.mod h1:3Y6ctEEwRg19B0jqsrQH6Hiquqte+zC0ZxpXLLSa5sA=
 github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.144.0 h1:gRk73SsIJv3q/HI0kxMRN5TiIiJj+MRxrz0GEYx3jZw=
 github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.144.0/go.mod h1:7b8ZcPdN6JSm6+LUq3rienoqHHiX60kyoOHDPCMM+L8=
+github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.148.0 h1:xgD/kNGp/wWY+bwY599Pc01OamYN17phRiTP934bM5Y=
+github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.148.0/go.mod h1:ZK7wvaefla9lB3bAW0rNKt7IzRPcTRQoOFqr4sZy/XM=
 github.com/open-telemetry/opentelemetry-collector-contrib/processor/logstransformprocessor v0.144.0 h1:HbmpzTixpQG/xGhQuQoiJTXQPrixe+yivAsF6tl2o4g=
 github.com/open-telemetry/opentelemetry-collector-contrib/processor/logstransformprocessor v0.144.0/go.mod h1:32jR9iqxVozOJ/Lg5RkCNoW18uCNwpKSbs6h5c28Ep4=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -837,6 +906,8 @@ github.com/openfga/openfga v1.14.1/go.mod h1:AqMyFFi3y24Hko1mIME6ctOdCsCru2HA3uH
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/ovh/go-ovh v1.9.0 h1:6K8VoL3BYjVV3In9tPJUdT7qMx9h0GExN9EXx1r2kKE=
+github.com/ovh/go-ovh v1.9.0/go.mod h1:cTVDnl94z4tl8pP1uZ/8jlVxntjSIf09bNcQ5TJSC7c=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -863,6 +934,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
+github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -912,6 +985,8 @@ github.com/prometheus/sigv4 v0.4.1 h1:EIc3j+8NBea9u1iV6O5ZAN8uvPq2xOIUPcqCTivHuX
 github.com/prometheus/sigv4 v0.4.1/go.mod h1:eu+ZbRvsc5TPiHwqh77OWuCnWK73IdkETYY46P4dXOU=
 github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg=
 github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
+github.com/puzpuzpuz/xsync/v4 v4.4.0 h1:vlSN6/CkEY0pY8KaB0yqo/pCLZvp9nhdbBdjipT4gWo=
+github.com/puzpuzpuz/xsync/v4 v4.4.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/redis/go-redis/extra/rediscmd/v9 v9.15.1 h1:G3pzZlMvMX9VX9TBB8zr03CAkeyMtbyW2D59PdyaGkM=
 github.com/redis/go-redis/extra/rediscmd/v9 v9.15.1/go.mod h1:JiJ4f0bngycE8LQqzY/4TB23witBbFnlUS6hPvHn6Zc=
@@ -947,6 +1022,8 @@ github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc=
 github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
 github.com/santhosh-tekuri/jsonschema/v3 v3.1.0 h1:levPcBfnazlA1CyCMC3asL/QLZkq9pa8tQZOH513zQw=
 github.com/santhosh-tekuri/jsonschema/v3 v3.1.0/go.mod h1:8kzK2TC0k0YjOForaAHdNEa7ik0fokNa2k30BKJ/W7Y=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.36 h1:ObX9hZmK+VmijreZO/8x9pQ8/P/ToHD/bdSb4Eg4tUo=
+github.com/scaleway/scaleway-sdk-go v1.0.0-beta.36/go.mod h1:LEsDu4BubxK7/cWhtlQWfuxwL4rf/2UEpxXz1o1EMtM=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
@@ -980,6 +1057,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
 github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
+github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
+github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -1001,6 +1080,8 @@ github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
 github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
 github.com/srikanthccv/ClickHouse-go-mock v0.12.0 h1:KUzaWTwuqMc2uf5FylM/oAcTFdE2DdZjvISm9V0/NAA=
 github.com/srikanthccv/ClickHouse-go-mock v0.12.0/go.mod h1:1oUmLtXEXOyS0EEWVKlKEfLfv9y02agCMAvD3tVnhlo=
+github.com/stackitcloud/stackit-sdk-go/core v0.23.0 h1:zPrOhf3Xe47rKRs1fg/AqKYUiJJRYjdcv+3qsS50mEs=
+github.com/stackitcloud/stackit-sdk-go/core v0.23.0/go.mod h1:osMglDby4csGZ5sIfhNyYq1bS1TxIdPY88+skE/kkmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
@@ -1079,6 +1160,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+github.com/vultr/govultr/v3 v3.28.1 h1:KR3LhppYARlBujY7+dcrE7YKL0Yo9qXL+msxykKQrLI=
+github.com/vultr/govultr/v3 v3.28.1/go.mod h1:2zyUw9yADQaGwKnwDesmIOlBNLrm7edsCfWHFJpWKf8=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
@@ -1410,6 +1493,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
@@ -1840,6 +1924,8 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWM
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k=
+gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
 gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/telebot.v3 v3.3.8 h1:uVDGjak9l824FN9YARWUHMsiNZnlohAVwUycw21k6t8=
 gopkg.in/telebot.v3 v3.3.8/go.mod h1:1mlbqcLTVSfK9dx7fdp+Nb5HZsy4LLPtpZTKmwhwtzM=

heatmap-poc/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"embed"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+)
+
+//go:embed static
+var embedded embed.FS
+
+// diskAssets is preferred over the embedded copy when it exists, so editing the
+// UI and reloading the page needs no rebuild. It resolves against the launch
+// config's cwd, the repo root.
+const diskAssets = "heatmap-poc/static"
+
+func main() {
+	addr := envOr("HEATMAP_POC_ADDR", "localhost:8099")
+	upstream, err := url.Parse(envOr("SIGNOZ_URL", "http://localhost:8080"))
+	if err != nil {
+		slog.Error("SIGNOZ_URL is not a URL", "error", err)
+		os.Exit(1)
+	}
+
+	apiKey := os.Getenv("SIGNOZ_API_KEY")
+	if apiKey == "" {
+		slog.Warn("SIGNOZ_API_KEY is unset, every upstream call will be rejected as unauthenticated")
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle("/api/", &httputil.ReverseProxy{
+		Rewrite: func(r *httputil.ProxyRequest) {
+			r.SetURL(upstream)
+			r.Out.Host = upstream.Host
+			r.Out.Header.Set("SIGNOZ-API-KEY", apiKey)
+		},
+		ErrorHandler: func(rw http.ResponseWriter, _ *http.Request, err error) {
+			slog.Error("upstream call failed", "error", err)
+			http.Error(rw, err.Error(), http.StatusBadGateway)
+		},
+	})
+	mux.Handle("/", http.FileServerFS(assets()))
+
+	slog.Info("listening", "url", "http://"+addr, "upstream", upstream.String())
+	if err := http.ListenAndServe(addr, mux); err != nil {
+		slog.Error("server stopped", "error", err)
+		os.Exit(1)
+	}
+}
+
+func assets() fs.FS {
+	if stat, err := os.Stat(diskAssets); err == nil && stat.IsDir() {
+		slog.Info("serving the UI from disk", "dir", diskAssets)
+		return os.DirFS(diskAssets)
+	}
+	slog.Info("serving the embedded UI")
+	sub, err := fs.Sub(embedded, "static")
+	if err != nil {
+		slog.Error("embedded assets are unreadable", "error", err)
+		os.Exit(1)
+	}
+	return sub
+}
+
+func envOr(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}

heatmap-poc/static/app.js

@@ -0,0 +1,788 @@
+const TIME_AGGREGATIONS = ["latest", "sum", "avg", "min", "max", "count", "count_distinct", "rate", "increase"];
+const SPACE_AGGREGATIONS = ["sum", "avg", "min", "max", "count"];
+
+// the types whose samples reach the reader as plain values, so a bucket axis has
+// to be chosen for them
+const VALUE_TYPES = new Set(["gauge", "sum", "summary"]);
+
+// single hue, dark to light: a count's magnitude is the only thing it encodes
+const RAMP = ["#1c3557", "#22406c", "#284c82", "#2e5998", "#3668ae", "#4a80c4", "#689dd6", "#8dbbe6", "#b9d8f5"];
+const ZERO_FILL = "#0e1016";
+
+const PAD = { left: 78, right: 8, top: 8, bottom: 22 };
+const MAX_CHART_HEIGHT = 560;
+const MAX_JSON_CHARS = 300_000;
+
+const state = {
+	mode: "metric",
+	rows: [],
+	groupBy: [],
+	bucketKind: "default",
+	colorScale: "linear",
+	grid: null,
+	emptyReason: "Run a query to draw the heatmap.",
+	hidden: new Set(),
+	requestText: "",
+	responseText: "",
+};
+
+const catalogue = new Map();
+const attributesByMetric = new Map();
+let nextRowId = 0;
+
+const $ = (selector) => document.querySelector(selector);
+const esc = (value) => String(value).replace(/[&<>"]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]);
+
+function timeWindow() {
+	const end = Date.now();
+	return { start: end - Number($("#range").value) * 60_000, end };
+}
+
+function rowLetter(index) {
+	return String.fromCharCode(65 + index);
+}
+
+function typeSupport(type) {
+	if (VALUE_TYPES.has(type)) {
+		return { aggregations: true, buckets: true };
+	}
+	if (type === "histogram") {
+		return { aggregations: false, buckets: false, note: "Read with increase/sum over its own le labels. Bucket options are rejected." };
+	}
+	if (type === "exponentialhistogram") {
+		return { aggregations: false, buckets: false, bad: true, note: "Exponential histograms are not supported yet — this comes back 501." };
+	}
+	return { aggregations: false, buckets: false, bad: true, note: "No type is recorded for this metric, so no bucket axis can be chosen — this comes back 400." };
+}
+
+function scaleHint(scale) {
+	const perTwice = 2 ** scale;
+	return perTwice >= 1 ? `${perTwice} bucket${perTwice === 1 ? "" : "s"} per 2x` : `1 bucket per ${2 ** -scale}x`;
+}
+
+function formatNumber(value) {
+	if (!Number.isFinite(value)) {
+		return value > 0 ? "∞" : "-∞";
+	}
+	if (value === 0) {
+		return "0";
+	}
+	const magnitude = Math.abs(value);
+	if (magnitude >= 1e6 || magnitude < 1e-3) {
+		return value.toExponential(1).replace("e+", "e");
+	}
+	if (Number.isInteger(value)) {
+		return String(value);
+	}
+	const text = value.toPrecision(magnitude >= 1 ? 4 : 3);
+	return text.includes(".") ? text.replace(/0+$/, "").replace(/\.$/, "") : text;
+}
+
+function formatTime(ms, spanMs) {
+	const at = new Date(ms);
+	const clock = at.toTimeString().slice(0, 5);
+	return spanMs > 24 * 3600_000 ? `${String(at.getMonth() + 1).padStart(2, "0")}-${String(at.getDate()).padStart(2, "0")} ${clock}` : clock;
+}
+
+/* ---------- upstream ---------- */
+
+async function getJSON(path, params) {
+	const response = await fetch(`${path}?${new URLSearchParams(params)}`);
+	if (!response.ok) {
+		throw new Error(`${path} came back ${response.status}: ${(await response.text()).slice(0, 400)}`);
+	}
+	return response.json();
+}
+
+async function searchMetrics(searchText) {
+	const { start, end } = timeWindow();
+	const body = await getJSON("/api/v2/metrics", { start, end, limit: 60, searchText });
+	const metrics = body?.data?.metrics ?? [];
+	for (const metric of metrics) {
+		catalogue.set(metric.metricName, metric);
+	}
+	return metrics;
+}
+
+async function metricAttributes(metricName) {
+	if (attributesByMetric.has(metricName)) {
+		return attributesByMetric.get(metricName);
+	}
+	const { start, end } = timeWindow();
+	const body = await getJSON("/api/v2/metrics/attributes", { metricName, start, end });
+	const keys = (body?.data?.attributes ?? []).map((attribute) => attribute.key);
+	attributesByMetric.set(metricName, keys);
+	return keys;
+}
+
+/* ---------- metric rows ---------- */
+
+function addRow() {
+	state.rows.push({ id: nextRowId++, metric: "", type: "", timeAggregation: "max", spaceAggregation: "max", filter: "" });
+	renderRows();
+}
+
+function renderRows() {
+	const host = $("#rows");
+	host.textContent = "";
+
+	state.rows.forEach((row, index) => {
+		const node = $("#row-template").content.firstElementChild.cloneNode(true);
+		const input = node.querySelector(".metric-input");
+		const list = node.querySelector("datalist");
+		const badge = node.querySelector(".badge");
+		const aggregations = node.querySelector(".agg-fields");
+		const note = node.querySelector(".row-note");
+		const listId = `metrics-${row.id}`;
+
+		node.querySelector(".row-name").textContent = rowLetter(index);
+		list.id = listId;
+		input.setAttribute("list", listId);
+		input.value = row.metric;
+		node.querySelector(".filter-input").value = row.filter;
+		node.querySelector(".remove-row").hidden = state.mode !== "formula" || state.rows.length < 2;
+
+		const support = typeSupport(row.type);
+		if (row.metric) {
+			badge.hidden = false;
+			badge.textContent = row.type || "no type";
+			badge.classList.toggle("bad", Boolean(support.bad));
+			aggregations.hidden = !support.aggregations;
+			note.hidden = !support.note;
+			note.textContent = support.note ?? "";
+			note.classList.toggle("warn", Boolean(support.bad));
+		}
+
+		for (const [select, options, chosen] of [
+			[node.querySelector(".time-agg"), TIME_AGGREGATIONS, row.timeAggregation],
+			[node.querySelector(".space-agg"), SPACE_AGGREGATIONS, row.spaceAggregation],
+		]) {
+			select.innerHTML = options.map((option) => `<option value="${option}"${option === chosen ? " selected" : ""}>${option}</option>`).join("");
+		}
+
+		const fillOptions = async () => {
+			try {
+				const metrics = await searchMetrics(input.value.trim());
+				list.innerHTML = metrics.map((metric) => `<option value="${esc(metric.metricName)}" label="${esc(metric.type || "no type")}"></option>`).join("");
+			} catch (error) {
+				showBanner(error.message);
+			}
+		};
+
+		let searchTimer = 0;
+		input.addEventListener("input", () => {
+			clearTimeout(searchTimer);
+			searchTimer = setTimeout(fillOptions, 220);
+		});
+		input.addEventListener("focus", () => {
+			if (!list.children.length) {
+				fillOptions();
+			}
+		});
+
+		input.addEventListener("change", async () => {
+			const name = input.value.trim();
+			row.metric = name;
+			row.type = catalogue.get(name)?.type ?? "";
+			if (name && !catalogue.has(name)) {
+				try {
+					await searchMetrics(name);
+					row.type = catalogue.get(name)?.type ?? "";
+				} catch (error) {
+					showBanner(error.message);
+				}
+			}
+			renderRows();
+			refreshPanes();
+			refreshGroupOptions();
+		});
+
+		node.querySelector(".time-agg").addEventListener("change", (event) => {
+			row.timeAggregation = event.target.value;
+		});
+		node.querySelector(".space-agg").addEventListener("change", (event) => {
+			row.spaceAggregation = event.target.value;
+		});
+		node.querySelector(".filter-input").addEventListener("change", (event) => {
+			row.filter = event.target.value.trim();
+		});
+		node.querySelector(".remove-row").addEventListener("click", () => {
+			state.rows = state.rows.filter((candidate) => candidate.id !== row.id);
+			renderRows();
+			refreshPanes();
+			refreshGroupOptions();
+		});
+
+		host.append(node);
+	});
+}
+
+/* ---------- group by ---------- */
+
+function renderGroupChips() {
+	$("#group-chips").innerHTML = state.groupBy
+		.map((key) => `<span class="chip">${esc(key)}<button type="button" data-key="${esc(key)}" title="Remove">&times;</button></span>`)
+		.join("");
+}
+
+async function refreshGroupOptions() {
+	const metrics = state.rows.map((row) => row.metric).filter(Boolean);
+	const hint = $("#group-hint");
+	if (!metrics.length) {
+		$("#group-options").innerHTML = "";
+		hint.hidden = false;
+		hint.textContent = "Pick a metric to load its attributes.";
+		return;
+	}
+
+	try {
+		const keys = new Set();
+		for (const row of state.rows.filter((candidate) => candidate.metric)) {
+			for (const key of await metricAttributes(row.metric)) {
+				// the builder strips `le` from a histogram's group by and reads the
+				// bucket axis off it instead, so offering it here would do nothing
+				if (key !== "le" || row.type !== "histogram") {
+					keys.add(key);
+				}
+			}
+		}
+		const available = [...keys].filter((key) => !state.groupBy.includes(key)).sort();
+		$("#group-options").innerHTML = available.map((key) => `<option value="${esc(key)}"></option>`).join("");
+		hint.hidden = available.length > 0;
+		hint.textContent = available.length ? "" : "No further attributes on the selected metrics in this window.";
+	} catch (error) {
+		showBanner(error.message);
+	}
+}
+
+/* ---------- panes ---------- */
+
+// A formula is bucketed from its own output, so its inputs may be any type. In
+// metric mode the one metric decides, and an unpicked one keeps the pane up.
+function bucketsAllowed() {
+	if (state.mode === "promql") {
+		return false;
+	}
+	if (state.mode === "formula") {
+		return true;
+	}
+	const row = state.rows[0];
+	return !row?.metric || typeSupport(row.type).buckets;
+}
+
+function refreshPanes() {
+	const isPromql = state.mode === "promql";
+	const isFormula = state.mode === "formula";
+
+	$("#builder-pane").hidden = isPromql;
+	$("#promql-pane").hidden = !isPromql;
+	$("#add-row").hidden = !isFormula;
+	$("#formula-field").hidden = !isFormula;
+
+	const allowed = bucketsAllowed();
+	$("#bucket-pane").hidden = !allowed;
+	if (!allowed) {
+		setBucketKind("default");
+	}
+}
+
+function setBucketKind(kind) {
+	state.bucketKind = kind;
+	for (const button of $("#bucket-kinds").children) {
+		button.classList.toggle("on", button.dataset.kind === kind);
+	}
+	$("#bucket-default-hint").hidden = kind !== "default";
+	$("#scale-field").hidden = kind !== "log";
+	$("#linear-fields").hidden = kind !== "linear";
+	$("#scale-hint").textContent = scaleHint(Number($("#scale").value));
+}
+
+function setMode(mode) {
+	state.mode = mode;
+	for (const button of $("#modes").children) {
+		button.classList.toggle("on", button.dataset.mode === mode);
+	}
+	if (mode === "metric") {
+		state.rows = state.rows.slice(0, 1);
+	}
+	if (!state.rows.length) {
+		addRow();
+	}
+	if (mode === "formula" && state.rows.length < 2) {
+		addRow();
+	}
+	renderRows();
+	refreshPanes();
+	refreshGroupOptions();
+}
+
+/* ---------- request ---------- */
+
+function buildRequest() {
+	const { start, end } = timeWindow();
+	const step = Number($("#step").value) || 60;
+	const request = {
+		schemaVersion: "v1",
+		start,
+		end,
+		requestType: "heatmap",
+		compositeQuery: { queries: [] },
+		formatOptions: { formatTableResultForUI: false, fillGaps: false },
+		noCache: $("#no-cache").checked,
+	};
+
+	if (state.mode === "promql") {
+		const query = $("#promql").value.trim();
+		if (!query) {
+			throw new Error("Enter a PromQL query.");
+		}
+		request.compositeQuery.queries.push({ type: "promql", spec: { name: "A", query, step, disabled: false } });
+		return request;
+	}
+
+	const inFormula = state.mode === "formula";
+	state.rows.forEach((row, index) => {
+		if (!row.metric) {
+			throw new Error(`Query ${rowLetter(index)} has no metric selected.`);
+		}
+		const histogram = row.type === "histogram";
+		const spec = {
+			name: rowLetter(index),
+			signal: "metrics",
+			aggregations: [
+				{
+					metricName: row.metric,
+					timeAggregation: histogram ? "increase" : row.timeAggregation,
+					spaceAggregation: histogram ? "sum" : row.spaceAggregation,
+				},
+			],
+			stepInterval: step,
+			// only the enabled query renders the heatmap, so a formula's inputs ride
+			// along disabled
+			disabled: inFormula,
+		};
+		if (state.groupBy.length) {
+			spec.groupBy = state.groupBy.map((name) => ({ name }));
+		}
+		if (row.filter) {
+			spec.filter = { expression: row.filter };
+		}
+		request.compositeQuery.queries.push({ type: "builder_query", spec });
+	});
+
+	if (inFormula) {
+		const expression = $("#formula").value.trim();
+		if (!expression) {
+			throw new Error("A formula heatmap needs an expression.");
+		}
+		request.compositeQuery.queries.push({ type: "builder_formula", spec: { name: "F1", expression, disabled: false } });
+	}
+
+	if (state.bucketKind === "log") {
+		request.bucketOptions = { kind: "log", spec: { scale: Number($("#scale").value) } };
+	} else if (state.bucketKind === "linear") {
+		request.bucketOptions = { kind: "linear", spec: { maxValue: Number($("#max-value").value), numBuckets: Number($("#num-buckets").value) } };
+	}
+
+	return request;
+}
+
+async function run() {
+	let request;
+	try {
+		request = buildRequest();
+	} catch (error) {
+		showBanner(error.message);
+		return;
+	}
+
+	state.window = { start: request.start, end: request.end };
+	state.requestText = JSON.stringify(request, null, "\t");
+	renderJSON("#request", state.requestText);
+	$("#run").disabled = true;
+	$("#run").textContent = "Running…";
+	showBanner("");
+
+	try {
+		const response = await fetch("/api/v5/query_range", {
+			method: "POST",
+			headers: { "content-type": "application/json" },
+			body: JSON.stringify(request),
+		});
+		const text = await response.text();
+		let parsed = null;
+		try {
+			parsed = JSON.parse(text);
+			state.responseText = JSON.stringify(parsed, null, "\t");
+		} catch {
+			state.responseText = text;
+		}
+		renderJSON("#response", state.responseText);
+
+		if (!response.ok) {
+			const problem = parsed?.error;
+			const detail = (problem?.errors ?? []).map((entry) => entry.message ?? JSON.stringify(entry)).join("\n");
+			showBanner(`${response.status} ${problem?.code ?? ""}\n${problem?.message ?? text.slice(0, 600)}${detail ? `\n${detail}` : ""}`.trim());
+			setGrid(null, "The request was rejected — see the message above.");
+			return;
+		}
+
+		const warning = parsed?.data?.warning;
+		if (warning?.message) {
+			showBanner([warning.message, ...(warning.warnings ?? []).map((entry) => entry.message)].join("\n"));
+		}
+		const grid = buildGrid(parsed);
+		setGrid(grid, "The response carried no series, so there is nothing to draw.");
+	} catch (error) {
+		showBanner(error.message);
+		setGrid(null, "The request could not be sent — see the message above.");
+	} finally {
+		$("#run").disabled = false;
+		$("#run").textContent = "Run query";
+	}
+}
+
+/* ---------- response ---------- */
+
+function buildGrid(body) {
+	const results = body?.data?.data?.results ?? [];
+	const result = results.find((entry) => Array.isArray(entry?.aggregations) && entry.aggregations.length);
+	if (!result) {
+		return null;
+	}
+
+	const aggregation = result.aggregations[0];
+	const buckets = aggregation.meta?.buckets ?? [];
+	const rows = buckets.length + 1;
+	const partial = new Set();
+
+	const series = (aggregation.series ?? []).map((entry) => {
+		const labels = (entry.labels ?? []).map((label) => `${label.key?.name ?? "?"}=${label.value}`);
+		const byTs = new Map();
+		let total = 0;
+		for (const point of entry.values ?? []) {
+			const counts = point.values ?? [];
+			byTs.set(point.timestamp, counts);
+			total += counts.reduce((sum, count) => sum + count, 0);
+			if (point.partial) {
+				partial.add(point.timestamp);
+			}
+		}
+		return { key: labels.join(", ") || "(no labels)", byTs, total };
+	});
+
+	const timestamps = [...new Set(series.flatMap((entry) => [...entry.byTs.keys()]))].sort((a, b) => a - b);
+	series.sort((a, b) => b.total - a.total || a.key.localeCompare(b.key));
+
+	return { queryName: result.queryName, buckets, rows, timestamps, partial, series };
+}
+
+function setGrid(grid, emptyReason) {
+	state.grid = grid;
+	state.emptyReason = emptyReason;
+	state.hidden = new Set();
+	renderGroups();
+	renderChart();
+}
+
+function visibleSeries() {
+	return state.grid.series.filter((entry) => !state.hidden.has(entry.key));
+}
+
+function renderGroups() {
+	const grid = state.grid;
+	const card = $("#groups-card");
+	if (!grid || grid.series.length < 2) {
+		card.hidden = true;
+		return;
+	}
+
+	card.hidden = false;
+	$("#group-count").textContent = `${grid.series.length - state.hidden.size} of ${grid.series.length} shown`;
+	$("#groups").innerHTML = grid.series
+		.map(
+			(entry) => `<label><input type="checkbox" data-key="${esc(entry.key)}"${state.hidden.has(entry.key) ? "" : " checked"}>
+				<span class="name" title="${esc(entry.key)}">${esc(entry.key)}</span>
+				<span class="total">${formatNumber(entry.total)}</span></label>`,
+		)
+		.join("");
+}
+
+/* ---------- chart ---------- */
+
+function bucketRange(grid, row) {
+	if (row === grid.rows - 1) {
+		return grid.buckets.length ? `> ${formatNumber(grid.buckets[grid.buckets.length - 1])}` : "overflow, the response carried no bucket bounds";
+	}
+	const upper = formatNumber(grid.buckets[row]);
+	return row === 0 ? `<= ${upper}` : `(${formatNumber(grid.buckets[row - 1])}, ${upper}]`;
+}
+
+function colorFor(count, max) {
+	if (count <= 0) {
+		return ZERO_FILL;
+	}
+	const fraction = max <= 0 ? 1 : state.colorScale === "log" ? Math.log1p(count) / Math.log1p(max) : count / max;
+	return RAMP[Math.min(RAMP.length - 1, Math.max(0, Math.round(fraction * (RAMP.length - 1))))];
+}
+
+function renderChart() {
+	const host = $("#chart");
+	const legend = $("#legend");
+	const grid = state.grid;
+	legend.textContent = "";
+
+	if (!grid) {
+		host.innerHTML = `<p class="empty">${esc(state.emptyReason)}</p>`;
+		return;
+	}
+	if (!grid.timestamps.length) {
+		host.innerHTML = '<p class="empty">The query returned no columns.</p>';
+		return;
+	}
+
+	const columns = grid.timestamps.length;
+	const shown = visibleSeries();
+	const matrix = grid.timestamps.map((ts) => {
+		const column = new Array(grid.rows).fill(0);
+		for (const entry of shown) {
+			const counts = entry.byTs.get(ts);
+			if (!counts) {
+				continue;
+			}
+			for (let row = 0; row < grid.rows; row++) {
+				column[row] += counts[row] ?? 0;
+			}
+		}
+		return column;
+	});
+
+	const max = Math.max(0, ...matrix.flat());
+	// clientWidth carries the 12px padding on either side of #chart
+	const available = host.clientWidth - 24 - PAD.left - PAD.right;
+	const cellWidth = Math.max(3, available / columns);
+	const cellHeight = Math.min(22, Math.max(4, MAX_CHART_HEIGHT / grid.rows));
+	const plotWidth = cellWidth * columns;
+	const plotHeight = cellHeight * grid.rows;
+	const gap = cellWidth >= 7 && cellHeight >= 7 ? 1 : 0;
+	const span = grid.timestamps[columns - 1] - grid.timestamps[0];
+
+	const cells = [];
+	for (let column = 0; column < columns; column++) {
+		for (let row = 0; row < grid.rows; row++) {
+			const x = PAD.left + column * cellWidth;
+			const y = PAD.top + (grid.rows - 1 - row) * cellHeight;
+			cells.push(`<rect x="${x.toFixed(2)}" y="${y.toFixed(2)}" width="${(cellWidth - gap).toFixed(2)}" height="${(cellHeight - gap).toFixed(2)}" fill="${colorFor(matrix[column][row], max)}"/>`);
+		}
+	}
+
+	const rowStride = Math.max(1, Math.ceil(13 / cellHeight));
+	const rowLabels = [];
+	for (let row = 0; row < grid.rows; row++) {
+		if (row % rowStride !== 0 && row !== grid.rows - 1) {
+			continue;
+		}
+		const y = PAD.top + (grid.rows - 1 - row) * cellHeight + cellHeight / 2;
+		const text = row === grid.rows - 1 ? "∞" : formatNumber(grid.buckets[row]);
+		rowLabels.push(`<text class="axis-label" x="${PAD.left - 6}" y="${(y + 3.2).toFixed(2)}" text-anchor="end">${esc(text)}</text>`);
+	}
+
+	const columnStride = Math.max(1, Math.ceil(58 / cellWidth));
+	const columnLabels = [];
+	for (let column = 0; column < columns; column += columnStride) {
+		const ts = grid.timestamps[column];
+		const x = PAD.left + column * cellWidth;
+		columnLabels.push(`<text class="axis-label" x="${x.toFixed(2)}" y="${(PAD.top + plotHeight + 14).toFixed(2)}">${esc(formatTime(ts, span))}${grid.partial.has(ts) ? "*" : ""}</text>`);
+	}
+
+	host.innerHTML = `<svg width="${PAD.left + plotWidth + PAD.right}" height="${PAD.top + plotHeight + PAD.bottom}">
+		${cells.join("")}
+		<line class="axis-line" x1="${PAD.left}" y1="${PAD.top + plotHeight + 0.5}" x2="${PAD.left + plotWidth}" y2="${PAD.top + plotHeight + 0.5}"/>
+		${rowLabels.join("")}${columnLabels.join("")}
+		<rect class="cursor-cell" hidden/>
+	</svg>`;
+
+	// a heatmap cannot fill gaps, so a chart much shorter than the window asked
+	// for means those columns hold no data at all rather than being hidden
+	const asked = state.window ? state.window.end - state.window.start : span;
+	const covered = `${formatTime(grid.timestamps[0], asked)}–${formatTime(grid.timestamps[columns - 1], asked)}`;
+	const coverage =
+		state.window && span < 0.9 * asked
+			? `${covered}, the only columns with data in the ${formatTime(state.window.start, asked)}–${formatTime(state.window.end, asked)} requested`
+			: covered;
+
+	legend.innerHTML = `<span>0</span>
+		<div class="swatches"><div class="swatch" style="background:${ZERO_FILL};border:1px solid var(--line)"></div>${RAMP.map((color) => `<div class="swatch" style="background:${color}"></div>`).join("")}</div>
+		<span>${formatNumber(max)} per cell</span>
+		<span>· ${grid.rows} buckets × ${columns} columns · ${shown.length} of ${grid.series.length} series · ${esc(coverage)}${grid.partial.size ? " · * partial column" : ""}</span>`;
+
+	attachHover(host.querySelector("svg"), { grid, matrix, shown, columns, cellWidth, cellHeight, plotHeight, span, max });
+}
+
+function attachHover(svg, view) {
+	const tooltip = $("#tooltip");
+	const cursor = svg.querySelector(".cursor-cell");
+
+	svg.addEventListener("mouseleave", () => {
+		tooltip.hidden = true;
+		cursor.setAttribute("hidden", "");
+	});
+
+	svg.addEventListener("mousemove", (event) => {
+		const box = svg.getBoundingClientRect();
+		const column = Math.floor((event.clientX - box.left - PAD.left) / view.cellWidth);
+		const row = view.grid.rows - 1 - Math.floor((event.clientY - box.top - PAD.top) / view.cellHeight);
+		if (column < 0 || column >= view.columns || row < 0 || row >= view.grid.rows) {
+			tooltip.hidden = true;
+			cursor.setAttribute("hidden", "");
+			return;
+		}
+
+		cursor.removeAttribute("hidden");
+		cursor.setAttribute("x", PAD.left + column * view.cellWidth);
+		cursor.setAttribute("y", PAD.top + (view.grid.rows - 1 - row) * view.cellHeight);
+		cursor.setAttribute("width", view.cellWidth);
+		cursor.setAttribute("height", view.cellHeight);
+
+		const ts = view.grid.timestamps[column];
+		const breakdown = view.shown
+			.map((entry) => ({ key: entry.key, count: entry.byTs.get(ts)?.[row] ?? 0 }))
+			.filter((entry) => entry.count > 0)
+			.sort((a, b) => b.count - a.count);
+
+		tooltip.innerHTML = [
+			`<b>${esc(formatNumber(view.matrix[column][row]))}</b> in ${esc(bucketRange(view.grid, row))}`,
+			`${esc(new Date(ts).toTimeString().slice(0, 8))}${view.grid.partial.has(ts) ? "  (partial)" : ""}`,
+			...breakdown.slice(0, 6).map((entry) => `  ${esc(entry.key)}  ${esc(formatNumber(entry.count))}`),
+			breakdown.length > 6 ? `  … ${breakdown.length - 6} more` : "",
+		]
+			.filter(Boolean)
+			.join("\n");
+
+		tooltip.hidden = false;
+		const width = tooltip.offsetWidth;
+		tooltip.style.left = `${Math.min(event.clientX + 14, window.innerWidth - width - 8)}px`;
+		tooltip.style.top = `${Math.min(event.clientY + 14, window.innerHeight - tooltip.offsetHeight - 8)}px`;
+	});
+}
+
+/* ---------- chrome ---------- */
+
+function showBanner(message) {
+	const banner = $("#banner");
+	banner.textContent = message;
+	banner.hidden = !message;
+}
+
+function renderJSON(selector, text) {
+	$(selector).textContent = text.length > MAX_JSON_CHARS ? `${text.slice(0, MAX_JSON_CHARS)}\n… truncated for display, Copy takes the whole thing` : text;
+}
+
+function wire() {
+	$("#modes").addEventListener("click", (event) => {
+		if (event.target.dataset.mode) {
+			setMode(event.target.dataset.mode);
+		}
+	});
+
+	$("#bucket-kinds").addEventListener("click", (event) => {
+		if (event.target.dataset.kind) {
+			setBucketKind(event.target.dataset.kind);
+		}
+	});
+
+	$("#scale").addEventListener("input", () => {
+		$("#scale-hint").textContent = scaleHint(Number($("#scale").value));
+	});
+
+	$("#add-row").addEventListener("click", () => {
+		addRow();
+		refreshPanes();
+	});
+
+	$("#range").addEventListener("change", () => {
+		attributesByMetric.clear();
+		refreshGroupOptions();
+	});
+
+	$("#group-input").addEventListener("change", (event) => {
+		const key = event.target.value.trim();
+		if (key && !state.groupBy.includes(key)) {
+			state.groupBy.push(key);
+			renderGroupChips();
+			refreshGroupOptions();
+		}
+		event.target.value = "";
+	});
+
+	$("#group-chips").addEventListener("click", (event) => {
+		const key = event.target.dataset.key;
+		if (key) {
+			state.groupBy = state.groupBy.filter((candidate) => candidate !== key);
+			renderGroupChips();
+			refreshGroupOptions();
+		}
+	});
+
+	$("#form").addEventListener("submit", (event) => {
+		event.preventDefault();
+		run();
+	});
+
+	$("#color-scale").addEventListener("change", (event) => {
+		state.colorScale = event.target.value;
+		renderChart();
+	});
+
+	$("#groups").addEventListener("change", (event) => {
+		const key = event.target.dataset.key;
+		if (!key) {
+			return;
+		}
+		if (event.target.checked) {
+			state.hidden.delete(key);
+		} else {
+			state.hidden.add(key);
+		}
+		$("#group-count").textContent = `${state.grid.series.length - state.hidden.size} of ${state.grid.series.length} shown`;
+		renderChart();
+	});
+
+	$("#select-all").addEventListener("click", () => {
+		state.hidden.clear();
+		renderGroups();
+		renderChart();
+	});
+
+	$("#select-none").addEventListener("click", () => {
+		state.hidden = new Set(state.grid.series.map((entry) => entry.key));
+		renderGroups();
+		renderChart();
+	});
+
+	for (const button of document.querySelectorAll("[data-copy]")) {
+		button.addEventListener("click", async () => {
+			const text = button.dataset.copy === "request" ? state.requestText : state.responseText;
+			try {
+				await navigator.clipboard.writeText(text);
+				button.textContent = "Copied";
+			} catch {
+				button.textContent = "Copy failed";
+			}
+			setTimeout(() => {
+				button.textContent = "Copy";
+			}, 1200);
+		});
+	}
+
+	let resizeTimer = 0;
+	window.addEventListener("resize", () => {
+		clearTimeout(resizeTimer);
+		resizeTimer = setTimeout(renderChart, 120);
+	});
+}
+
+wire();
+setMode("metric");
+renderGroupChips();
+setBucketKind("default");

heatmap-poc/static/index.html

@@ -0,0 +1,168 @@
+<!doctype html>
+<html lang="en">
+<head>
+	<meta charset="utf-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1">
+	<title>Heatmap POC</title>
+	<link rel="stylesheet" href="style.css">
+</head>
+<body>
+	<header>
+		<h1>Heatmap POC</h1>
+		<p class="sub"><code>POST /api/v5/query_range</code> with <code>requestType: "heatmap"</code></p>
+	</header>
+
+	<div class="layout">
+		<form id="form" class="card form">
+			<div class="segmented" id="modes">
+				<button type="button" data-mode="metric" class="on">Metric</button>
+				<button type="button" data-mode="formula">Formula</button>
+				<button type="button" data-mode="promql">PromQL</button>
+			</div>
+
+			<div class="grid-2">
+				<label>Time range
+					<select id="range">
+						<option value="15">Last 15 minutes</option>
+						<option value="60" selected>Last 1 hour</option>
+						<option value="180">Last 3 hours</option>
+						<option value="360">Last 6 hours</option>
+						<option value="1440">Last 24 hours</option>
+						<option value="10080">Last 7 days</option>
+					</select>
+				</label>
+				<label>Step (seconds)
+					<input id="step" type="number" min="1" step="1" value="60">
+				</label>
+			</div>
+
+			<section id="builder-pane">
+				<div id="rows"></div>
+				<button type="button" id="add-row" class="ghost" hidden>+ Add metric</button>
+
+				<label id="formula-field" hidden>Formula
+					<input id="formula" placeholder="A / B" autocomplete="off">
+				</label>
+
+				<div class="chips-field">
+					<span class="chips-label">Group by</span>
+					<div id="group-chips" class="chips"></div>
+					<input id="group-input" list="group-options" placeholder="Add an attribute…" autocomplete="off">
+					<datalist id="group-options"></datalist>
+					<p class="hint" id="group-hint" hidden></p>
+				</div>
+			</section>
+
+			<section id="promql-pane" hidden>
+				<label>PromQL
+					<textarea id="promql" rows="4" spellcheck="false" placeholder='sum by (le) (increase({__name__="http.server.duration.bucket"}[5m]))'></textarea>
+				</label>
+				<p class="hint">The bucket axis comes from the <code>le</code> labels the query returns, so bucket options are rejected here. A dotted metric name needs the <code>{__name__="…"}</code> form.</p>
+			</section>
+
+			<fieldset id="bucket-pane">
+				<legend>Bucket options</legend>
+				<div class="segmented small" id="bucket-kinds">
+					<button type="button" data-kind="default" class="on">Default</button>
+					<button type="button" data-kind="log">Log</button>
+					<button type="button" data-kind="linear">Linear</button>
+				</div>
+				<p class="hint" id="bucket-default-hint">Omitted from the request. The backend falls back to a log axis at scale 4.</p>
+				<label id="scale-field" hidden>Scale
+					<input id="scale" type="number" min="-4" max="4" step="1" value="4">
+					<span class="hint" id="scale-hint"></span>
+				</label>
+				<div class="grid-2" id="linear-fields" hidden>
+					<label>Max value
+						<input id="max-value" type="number" min="0" step="any" value="1000">
+					</label>
+					<label>Number of buckets
+						<input id="num-buckets" type="number" min="1" max="512" step="1" value="60">
+					</label>
+				</div>
+			</fieldset>
+
+			<div class="run-row">
+				<button type="submit" id="run">Run query</button>
+				<label class="inline"><input type="checkbox" id="no-cache" checked> Bypass cache</label>
+			</div>
+		</form>
+
+		<div class="results">
+			<p id="banner" hidden></p>
+
+			<section class="card">
+				<div class="card-head">
+					<h2>Heatmap</h2>
+					<div class="head-tools">
+						<label class="inline">Colour scale
+							<select id="color-scale">
+								<option value="linear" selected>Linear</option>
+								<option value="log">Log</option>
+							</select>
+						</label>
+					</div>
+				</div>
+				<div id="chart" class="chart"><p class="empty">Run a query to draw the heatmap.</p></div>
+				<div id="legend" class="legend"></div>
+			</section>
+
+			<section class="card" id="groups-card" hidden>
+				<div class="card-head">
+					<h2>Groups <span id="group-count" class="count"></span></h2>
+					<div class="head-tools">
+						<button type="button" class="ghost" id="select-all">All</button>
+						<button type="button" class="ghost" id="select-none">None</button>
+					</div>
+				</div>
+				<div id="groups" class="groups"></div>
+			</section>
+
+			<div class="grid-2 json-grid">
+				<section class="card">
+					<div class="card-head">
+						<h2>Request</h2>
+						<button type="button" class="ghost" data-copy="request">Copy</button>
+					</div>
+					<pre id="request" class="json"></pre>
+				</section>
+				<section class="card">
+					<div class="card-head">
+						<h2>Response</h2>
+						<button type="button" class="ghost" data-copy="response">Copy</button>
+					</div>
+					<pre id="response" class="json"></pre>
+				</section>
+			</div>
+		</div>
+	</div>
+
+	<div id="tooltip" class="tooltip" hidden></div>
+
+	<template id="row-template">
+		<div class="metric-row">
+			<span class="row-name"></span>
+			<div class="row-body">
+				<div class="metric-field">
+					<input class="metric-input" placeholder="Search metrics…" autocomplete="off" spellcheck="false">
+					<datalist></datalist>
+					<span class="badge" hidden></span>
+				</div>
+				<div class="agg-fields" hidden>
+					<label>Time
+						<select class="time-agg"></select>
+					</label>
+					<label>Space
+						<select class="space-agg"></select>
+					</label>
+				</div>
+				<input class="filter-input" placeholder='Filter, e.g. service = "api"' autocomplete="off" spellcheck="false">
+				<p class="row-note hint" hidden></p>
+			</div>
+			<button type="button" class="remove-row" title="Remove this metric">&times;</button>
+		</div>
+	</template>
+
+	<script src="app.js" type="module"></script>
+</body>
+</html>

heatmap-poc/static/style.css

@@ -0,0 +1,552 @@
+:root {
+	color-scheme: dark;
+
+	--surface: #0b0d12;
+	--card: #12151d;
+	--card-head: #171b25;
+	--line: #242a38;
+	--line-soft: #1b2030;
+
+	--ink: #e6e9f0;
+	--ink-soft: #a2abbd;
+	--ink-faint: #6b7488;
+
+	--accent: #4a80c4;
+	--accent-ink: #b9d8f5;
+	--danger: #e0736b;
+	--danger-bg: #2a1618;
+
+	--cell-zero: #0e1016;
+}
+
+* {
+	box-sizing: border-box;
+}
+
+/* the display rules below would otherwise beat the UA's [hidden] rule, and SVG
+   elements never honoured the attribute on their own */
+[hidden] {
+	display: none !important;
+}
+
+body {
+	margin: 0;
+	padding: 20px;
+	background: var(--surface);
+	color: var(--ink);
+	font: 13px/1.5 ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif;
+}
+
+header {
+	margin-bottom: 16px;
+}
+
+h1 {
+	margin: 0;
+	font-size: 17px;
+	font-weight: 600;
+	letter-spacing: -0.01em;
+}
+
+h2 {
+	margin: 0;
+	font-size: 12px;
+	font-weight: 600;
+	text-transform: uppercase;
+	letter-spacing: 0.06em;
+	color: var(--ink-soft);
+}
+
+.sub {
+	margin: 3px 0 0;
+	color: var(--ink-faint);
+}
+
+code {
+	font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+	font-size: 0.92em;
+	color: var(--ink-soft);
+}
+
+.layout {
+	display: grid;
+	grid-template-columns: minmax(320px, 380px) minmax(0, 1fr);
+	gap: 16px;
+	align-items: start;
+}
+
+@media (max-width: 900px) {
+	.layout {
+		grid-template-columns: minmax(0, 1fr);
+	}
+}
+
+.card {
+	background: var(--card);
+	border: 1px solid var(--line);
+	border-radius: 8px;
+	overflow: hidden;
+}
+
+.card-head {
+	display: flex;
+	align-items: center;
+	justify-content: space-between;
+	gap: 10px;
+	padding: 8px 12px;
+	background: var(--card-head);
+	border-bottom: 1px solid var(--line);
+}
+
+.head-tools {
+	display: flex;
+	align-items: center;
+	gap: 8px;
+}
+
+.count {
+	color: var(--ink-faint);
+	font-weight: 400;
+	text-transform: none;
+	letter-spacing: 0;
+}
+
+/* ---------- form ---------- */
+
+.form {
+	padding: 14px;
+	display: grid;
+	gap: 14px;
+	position: sticky;
+	top: 20px;
+}
+
+label {
+	display: grid;
+	gap: 5px;
+	font-size: 12px;
+	color: var(--ink-soft);
+}
+
+label.inline {
+	display: inline-flex;
+	align-items: center;
+	gap: 6px;
+}
+
+.grid-2 {
+	display: grid;
+	grid-template-columns: 1fr 1fr;
+	gap: 10px;
+}
+
+input,
+select,
+textarea {
+	width: 100%;
+	padding: 6px 8px;
+	background: var(--surface);
+	color: var(--ink);
+	border: 1px solid var(--line);
+	border-radius: 5px;
+	font: inherit;
+}
+
+input[type="checkbox"] {
+	width: auto;
+	accent-color: var(--accent);
+}
+
+textarea {
+	font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+	resize: vertical;
+}
+
+input:focus-visible,
+select:focus-visible,
+textarea:focus-visible,
+button:focus-visible {
+	outline: 2px solid var(--accent);
+	outline-offset: 1px;
+}
+
+fieldset {
+	margin: 0;
+	padding: 10px 12px 12px;
+	border: 1px solid var(--line);
+	border-radius: 6px;
+	display: grid;
+	gap: 10px;
+}
+
+legend {
+	padding: 0 5px;
+	font-size: 11px;
+	font-weight: 600;
+	text-transform: uppercase;
+	letter-spacing: 0.06em;
+	color: var(--ink-soft);
+}
+
+button {
+	padding: 6px 12px;
+	background: var(--accent);
+	color: #fff;
+	border: 1px solid transparent;
+	border-radius: 5px;
+	font: inherit;
+	font-weight: 500;
+	cursor: pointer;
+}
+
+button:hover {
+	filter: brightness(1.12);
+}
+
+button.ghost {
+	background: transparent;
+	color: var(--ink-soft);
+	border-color: var(--line);
+	font-weight: 400;
+}
+
+button.ghost:hover {
+	color: var(--ink);
+	border-color: var(--ink-faint);
+	filter: none;
+}
+
+button[disabled] {
+	opacity: 0.5;
+	cursor: default;
+	filter: none;
+}
+
+.segmented {
+	display: flex;
+	gap: 2px;
+	padding: 2px;
+	background: var(--surface);
+	border: 1px solid var(--line);
+	border-radius: 6px;
+}
+
+.segmented button {
+	flex: 1;
+	background: transparent;
+	color: var(--ink-soft);
+	font-weight: 400;
+}
+
+.segmented button.on {
+	background: var(--line);
+	color: var(--ink);
+}
+
+.segmented button:hover {
+	filter: none;
+	color: var(--ink);
+}
+
+.segmented.small button {
+	padding: 4px 8px;
+	font-size: 12px;
+}
+
+.hint {
+	margin: 0;
+	font-size: 11px;
+	color: var(--ink-faint);
+}
+
+.hint.warn {
+	color: var(--danger);
+}
+
+.run-row {
+	display: flex;
+	align-items: center;
+	gap: 12px;
+}
+
+.run-row button {
+	flex: 1;
+}
+
+/* ---------- metric rows ---------- */
+
+#rows {
+	display: grid;
+	gap: 8px;
+}
+
+.metric-row {
+	display: grid;
+	grid-template-columns: auto minmax(0, 1fr) auto;
+	gap: 8px;
+	align-items: start;
+	padding: 8px;
+	background: var(--surface);
+	border: 1px solid var(--line-soft);
+	border-radius: 6px;
+}
+
+.row-name {
+	width: 20px;
+	padding-top: 6px;
+	font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+	font-weight: 600;
+	color: var(--accent-ink);
+	text-align: center;
+}
+
+.row-body {
+	display: grid;
+	gap: 8px;
+	min-width: 0;
+}
+
+.metric-field {
+	display: flex;
+	align-items: center;
+	gap: 6px;
+	min-width: 0;
+}
+
+.badge {
+	flex: none;
+	padding: 2px 6px;
+	background: var(--line);
+	border-radius: 4px;
+	font-size: 10px;
+	font-weight: 600;
+	text-transform: uppercase;
+	letter-spacing: 0.04em;
+	color: var(--accent-ink);
+	white-space: nowrap;
+}
+
+.badge.bad {
+	background: var(--danger-bg);
+	color: var(--danger);
+}
+
+.agg-fields {
+	display: grid;
+	grid-template-columns: 1fr 1fr;
+	gap: 8px;
+}
+
+.remove-row {
+	padding: 2px 7px;
+	background: transparent;
+	color: var(--ink-faint);
+	border-color: transparent;
+	font-size: 15px;
+	line-height: 1.2;
+}
+
+.remove-row:hover {
+	color: var(--danger);
+	filter: none;
+}
+
+/* ---------- group by chips ---------- */
+
+.chips-field {
+	display: grid;
+	gap: 6px;
+}
+
+.chips-label {
+	font-size: 12px;
+	color: var(--ink-soft);
+}
+
+.chips {
+	display: flex;
+	flex-wrap: wrap;
+	gap: 5px;
+}
+
+.chips:empty {
+	display: none;
+}
+
+.chip {
+	display: inline-flex;
+	align-items: center;
+	gap: 5px;
+	padding: 2px 4px 2px 8px;
+	background: var(--line);
+	border-radius: 11px;
+	font-size: 12px;
+}
+
+.chip button {
+	padding: 0 3px;
+	background: transparent;
+	color: var(--ink-faint);
+	border: 0;
+	font-size: 13px;
+	line-height: 1;
+}
+
+.chip button:hover {
+	color: var(--danger);
+	filter: none;
+}
+
+/* ---------- results ---------- */
+
+.results {
+	display: grid;
+	gap: 16px;
+	min-width: 0;
+}
+
+#banner {
+	margin: 0;
+	padding: 10px 12px;
+	background: var(--danger-bg);
+	border: 1px solid #4a2427;
+	border-radius: 8px;
+	color: var(--danger);
+	white-space: pre-wrap;
+}
+
+.chart {
+	padding: 12px;
+	overflow: auto;
+	max-height: 620px;
+}
+
+.chart svg {
+	display: block;
+}
+
+.empty {
+	margin: 0;
+	padding: 28px 0;
+	color: var(--ink-faint);
+	text-align: center;
+}
+
+.axis-label {
+	fill: var(--ink-faint);
+	font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+	font-size: 10px;
+}
+
+.axis-line {
+	stroke: var(--line);
+	stroke-width: 1;
+}
+
+.cursor-cell {
+	fill: none;
+	stroke: var(--accent-ink);
+	stroke-width: 1.5;
+	pointer-events: none;
+}
+
+.legend {
+	display: flex;
+	align-items: center;
+	gap: 10px;
+	padding: 0 12px 12px;
+	color: var(--ink-faint);
+	font-size: 11px;
+}
+
+.legend:empty {
+	display: none;
+}
+
+.legend .swatches {
+	display: flex;
+	gap: 2px;
+}
+
+.legend .swatch {
+	width: 20px;
+	height: 10px;
+	border-radius: 2px;
+}
+
+.groups {
+	display: grid;
+	grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
+	gap: 2px 12px;
+	padding: 10px 12px;
+	max-height: 220px;
+	overflow: auto;
+}
+
+.groups label {
+	display: flex;
+	align-items: center;
+	gap: 7px;
+	min-width: 0;
+	padding: 2px 0;
+	color: var(--ink);
+	font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+	font-size: 11px;
+	cursor: pointer;
+}
+
+.groups .name {
+	overflow: hidden;
+	text-overflow: ellipsis;
+	white-space: nowrap;
+}
+
+.groups .total {
+	margin-left: auto;
+	flex: none;
+	color: var(--ink-faint);
+}
+
+.json-grid {
+	align-items: start;
+}
+
+@media (max-width: 1200px) {
+	.json-grid {
+		grid-template-columns: minmax(0, 1fr);
+	}
+}
+
+.json {
+	margin: 0;
+	padding: 12px;
+	max-height: 420px;
+	overflow: auto;
+	font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+	font-size: 11px;
+	line-height: 1.55;
+	color: var(--ink-soft);
+	white-space: pre;
+	tab-size: 2;
+}
+
+.tooltip {
+	position: fixed;
+	z-index: 10;
+	max-width: 320px;
+	padding: 7px 9px;
+	background: #1c212e;
+	border: 1px solid var(--line);
+	border-radius: 6px;
+	box-shadow: 0 6px 18px rgb(0 0 0 / 45%);
+	font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+	font-size: 11px;
+	line-height: 1.6;
+	pointer-events: none;
+	white-space: pre;
+}
+
+.tooltip b {
+	color: var(--accent-ink);
+	font-weight: 600;
+}

pkg/apiserver/apiserver.go

@@ -1,14 +1,10 @@
 package apiserver
 
 import (
-	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/gorilla/mux"
 )
 
 type APIServer interface {
-	// APIServer is a long running service serving the SigNoz API.
-	factory.ServiceWithHealthy
-
 	// Returns the mux router for the API server. Primarily used for collecting OpenAPI operations.
 	Router() *mux.Router
 

pkg/apiserver/config.go

@@ -3,16 +3,13 @@ package apiserver
 import (
 	"time"
 
-	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
-	httpserver "github.com/SigNoz/signoz/pkg/http/server"
 )
 
 // Config holds the configuration for config.
 type Config struct {
-	httpserver.Config `mapstructure:",squash" yaml:",squash"`
-	Timeout           Timeout `mapstructure:"timeout"`
-	Logging           Logging `mapstructure:"logging"`
+	Timeout Timeout `mapstructure:"timeout"`
+	Logging Logging `mapstructure:"logging"`
 }
 
 type Timeout struct {
@@ -35,10 +32,6 @@ func NewConfigFactory() factory.ConfigFactory {
 
 func newConfig() factory.Config {
 	return &Config{
-		Config: httpserver.Config{
-			Address:     "0.0.0.0:8080",
-			ReadTimeout: 60 * time.Second,
-		},
 		Timeout: Timeout{
 			Default: 60 * time.Second,
 			Max:     600 * time.Second,
@@ -59,13 +52,5 @@ func newConfig() factory.Config {
 }
 
 func (c Config) Validate() error {
-	if err := c.Config.Validate(); err != nil {
-		return err
-	}
-
-	if c.Address == "" {
-		return errors.NewInvalidInputf(errors.CodeInvalidInput, "apiserver.address is required")
-	}
-
 	return nil
 }

pkg/apiserver/config_test.go

@@ -8,18 +8,11 @@ import (
 	"github.com/SigNoz/signoz/pkg/config"
 	"github.com/SigNoz/signoz/pkg/config/envprovider"
 	"github.com/SigNoz/signoz/pkg/factory"
-	httpserver "github.com/SigNoz/signoz/pkg/http/server"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestNewWithEnvProvider(t *testing.T) {
-	t.Setenv("SIGNOZ_APISERVER_ADDRESS", "0.0.0.0:9090")
-	t.Setenv("SIGNOZ_APISERVER_READ__TIMEOUT", "80s")
-	t.Setenv("SIGNOZ_APISERVER_TLS_ENABLED", "true")
-	t.Setenv("SIGNOZ_APISERVER_TLS_CERT__FILE", "/etc/signoz/server.crt")
-	t.Setenv("SIGNOZ_APISERVER_TLS_KEY__FILE", "/etc/signoz/server.key")
-	t.Setenv("SIGNOZ_APISERVER_TLS_MIN__VERSION", "1.3")
 	t.Setenv("SIGNOZ_APISERVER_TIMEOUT_DEFAULT", "70s")
 	t.Setenv("SIGNOZ_APISERVER_TIMEOUT_MAX", "700s")
 	t.Setenv("SIGNOZ_APISERVER_TIMEOUT_EXCLUDED__ROUTES", "/excluded1,/excluded2")
@@ -45,16 +38,6 @@ func TestNewWithEnvProvider(t *testing.T) {
 	require.NoError(t, err)
 
 	expected := &Config{
-		Config: httpserver.Config{
-			Address:     "0.0.0.0:9090",
-			ReadTimeout: 80 * time.Second,
-			TLS: httpserver.TLS{
-				Enabled:    true,
-				CertFile:   "/etc/signoz/server.crt",
-				KeyFile:    "/etc/signoz/server.key",
-				MinVersion: "1.3",
-			},
-		},
 		Timeout: Timeout{
 			Default: 70 * time.Second,
 			Max:     700 * time.Second,

pkg/apiserver/signozapiserver/provider.go

@@ -2,11 +2,9 @@ package signozapiserver
 
 import (
 	"context"
-	"net/http"
 
 	"github.com/SigNoz/signoz/pkg/alertmanager"
 	"github.com/SigNoz/signoz/pkg/apiserver"
-	"github.com/SigNoz/signoz/pkg/auditor"
 	"github.com/SigNoz/signoz/pkg/authz"
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/flagger"
@@ -14,8 +12,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/global"
 	"github.com/SigNoz/signoz/pkg/http/handler"
 	"github.com/SigNoz/signoz/pkg/http/middleware"
-	httpserver "github.com/SigNoz/signoz/pkg/http/server"
-	"github.com/SigNoz/signoz/pkg/identn"
 	"github.com/SigNoz/signoz/pkg/licensing"
 	"github.com/SigNoz/signoz/pkg/modules/aiobservability"
 	"github.com/SigNoz/signoz/pkg/modules/authdomain"
@@ -41,22 +37,18 @@ import (
 	"github.com/SigNoz/signoz/pkg/prometheus"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/ruler"
-	"github.com/SigNoz/signoz/pkg/sharder"
 	"github.com/SigNoz/signoz/pkg/statsreporter"
 	"github.com/SigNoz/signoz/pkg/subscription"
 	"github.com/SigNoz/signoz/pkg/types"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/web"
 	"github.com/SigNoz/signoz/pkg/zeus"
 	"github.com/gorilla/mux"
 )
 
 type provider struct {
-	globalConfig               global.Config
-	web                        web.Web
+	config                     apiserver.Config
+	settings                   factory.ScopedProviderSettings
 	router                     *mux.Router
-	httpServer                 *httpserver.Server
-	healthyC                   chan struct{}
 	authzMiddleware            *middleware.AuthZ
 	authzService               authz.AuthZ
 	orgHandler                 organization.Handler
@@ -140,11 +132,6 @@ func NewFactory(
 	rulerHandler ruler.Handler,
 	statsHandler statsreporter.Handler,
 	savedViewHandler savedview.Handler,
-	globalConfig global.Config,
-	identNResolver identn.IdentNResolver,
-	sharder sharder.Sharder,
-	auditor auditor.Auditor,
-	web web.Web,
 	quickFilterModule quickfilter.Module,
 	quickFilterHandler quickfilter.Handler,
 ) factory.ProviderFactory[apiserver.APIServer, apiserver.Config] {
@@ -192,11 +179,6 @@ func NewFactory(
 			rulerHandler,
 			statsHandler,
 			savedViewHandler,
-			globalConfig,
-			identNResolver,
-			sharder,
-			auditor,
-			web,
 			quickFilterModule,
 			quickFilterHandler,
 		)
@@ -246,11 +228,6 @@ func newProvider(
 	rulerHandler ruler.Handler,
 	statsHandler statsreporter.Handler,
 	savedViewHandler savedview.Handler,
-	globalConfig global.Config,
-	identNResolver identn.IdentNResolver,
-	sharder sharder.Sharder,
-	auditor auditor.Auditor,
-	web web.Web,
 	quickFilterModule quickfilter.Module,
 	quickFilterHandler quickfilter.Handler,
 ) (apiserver.APIServer, error) {
@@ -258,10 +235,9 @@ func newProvider(
 	router := mux.NewRouter().UseEncodedPath()
 
 	provider := &provider{
-		globalConfig:               globalConfig,
-		web:                        web,
+		config:                     config,
+		settings:                   settings,
 		router:                     router,
-		healthyC:                   make(chan struct{}),
 		orgHandler:                 orgHandler,
 		userHandler:                userHandler,
 		authzService:               authzService,
@@ -306,68 +282,13 @@ func newProvider(
 
 	provider.authzMiddleware = middleware.NewAuthZ(settings.Logger(), orgGetter, authzService)
 
-	router.Use(middleware.NewRecovery(settings.Logger()).Wrap)
-	router.Use(middleware.NewOtel("apiserver", providerSettings.MeterProvider, providerSettings.TracerProvider).Wrap)
-	router.Use(middleware.NewIdentN(identNResolver, sharder, settings.Logger()).Wrap)
-	router.Use(middleware.NewTimeout(settings.Logger(),
-		config.Timeout.ExcludedRoutes,
-		config.Timeout.Default,
-		config.Timeout.Max,
-	).Wrap)
-	router.Use(middleware.NewResource(settings.Logger()).Wrap)
-	router.Use(middleware.NewAudit(settings.Logger(), config.Logging.ExcludedRoutes, auditor).Wrap)
-	router.Use(middleware.NewComment().Wrap)
-
 	if err := provider.AddToRouter(router); err != nil {
 		return nil, err
 	}
 
-	httpHandler := middleware.NewCors().Wrap(router)
-	httpHandler = middleware.NewCompress().Wrap(httpHandler)
-
-	routePrefix := globalConfig.ExternalPath()
-	if routePrefix != "" {
-		prefixed := http.StripPrefix(routePrefix, httpHandler)
-		httpHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-			switch req.URL.Path {
-			case "/api/v1/health", "/api/v2/healthz", "/api/v2/readyz", "/api/v2/livez":
-				router.ServeHTTP(w, req)
-				return
-			}
-
-			prefixed.ServeHTTP(w, req)
-		})
-	}
-
-	httpServer, err := httpserver.New(settings.Logger(), config.Config, httpHandler)
-	if err != nil {
-		return nil, err
-	}
-	provider.httpServer = httpServer
-
 	return provider, nil
 }
 
-func (provider *provider) Start(ctx context.Context) error {
-	// Mount the web routes last so the catch-all prefix does not shadow API
-	// routes registered on the router after construction.
-	if err := provider.web.AddToRouter(provider.router); err != nil {
-		return err
-	}
-
-	close(provider.healthyC)
-
-	return provider.httpServer.Start(ctx)
-}
-
-func (provider *provider) Stop(ctx context.Context) error {
-	return provider.httpServer.Stop(ctx)
-}
-
-func (provider *provider) Healthy() <-chan struct{} {
-	return provider.healthyC
-}
-
 func (provider *provider) Router() *mux.Router {
 	return provider.router
 }

pkg/factory/registry.go

@@ -78,40 +78,6 @@ func NewRegistry(ctx context.Context, logger *slog.Logger, services ...NamedServ
 	}, nil
 }
 
-// Add registers additional services into the registry. It must be called before Start.
-func (registry *Registry) Add(ctx context.Context, services ...NamedService) error {
-	added := make([]*serviceWithState, 0, len(services))
-	for _, s := range services {
-		if _, ok := registry.servicesByName[s.Name()]; ok {
-			return errors.Newf(errors.TypeInvalidInput, ErrCodeInvalidRegistry, "cannot add service, duplicate service name %q", s.Name())
-		}
-		added = append(added, newServiceWithState(s))
-	}
-
-	for _, ss := range added {
-		registry.services = append(registry.services, ss)
-		registry.servicesByName[ss.service.Name()] = ss
-	}
-
-	for _, ss := range added {
-		for _, dep := range ss.service.DependsOn() {
-			if dep == ss.service.Name() {
-				registry.logger.ErrorContext(ctx, "ignoring self-dependency", slog.Any("service", ss.service.Name()))
-				continue
-			}
-
-			if _, ok := registry.servicesByName[dep]; !ok {
-				registry.logger.ErrorContext(ctx, "ignoring unknown dependency", slog.Any("service", ss.service.Name()), slog.Any("dependency", dep))
-				continue
-			}
-
-			ss.dependsOn = append(ss.dependsOn, dep)
-		}
-	}
-
-	return detectCyclicDeps(registry.services)
-}
-
 func (registry *Registry) Start(ctx context.Context) {
 	for _, ss := range registry.services {
 		go func(ss *serviceWithState) {

pkg/factory/registry_test.go

@@ -342,61 +342,3 @@ func TestDependsOnCycleReturnsError(t *testing.T) {
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "dependency cycles detected")
 }
-
-func TestRegistryAdd(t *testing.T) {
-	s1 := newTestService(t)
-	s2 := newTestService(t)
-
-	registry, err := NewRegistry(context.Background(), slog.New(slog.DiscardHandler), NewNamedService(MustNewName("s1"), s1))
-	require.NoError(t, err)
-
-	require.NoError(t, registry.Add(context.Background(), NewNamedService(MustNewName("s2"), s2)))
-
-	ctx := context.Background()
-	registry.Start(ctx)
-
-	require.NoError(t, registry.AwaitHealthy(ctx))
-	byState := registry.ServicesByState()
-	assert.Len(t, byState[StateRunning], 2)
-	assert.True(t, registry.IsHealthy())
-
-	assert.NoError(t, registry.Stop(ctx))
-}
-
-func TestRegistryAddDuplicateReturnsError(t *testing.T) {
-	s1 := newTestService(t)
-
-	registry, err := NewRegistry(context.Background(), slog.New(slog.DiscardHandler), NewNamedService(MustNewName("s1"), s1))
-	require.NoError(t, err)
-
-	err = registry.Add(context.Background(), NewNamedService(MustNewName("s1"), newTestService(t)))
-	assert.Error(t, err)
-	assert.Contains(t, err.Error(), "duplicate service name")
-}
-
-func TestRegistryAddWithDependency(t *testing.T) {
-	s1 := newHealthyTestService(t)
-	s2 := newTestService(t)
-
-	registry, err := NewRegistry(context.Background(), slog.New(slog.DiscardHandler), NewNamedService(MustNewName("s1"), s1))
-	require.NoError(t, err)
-
-	// s2 depends on the already registered s1.
-	require.NoError(t, registry.Add(context.Background(), NewNamedService(MustNewName("s2"), s2, MustNewName("s1"))))
-
-	ctx := context.Background()
-	registry.Start(ctx)
-
-	// s2 stays in STARTING until s1 is healthy.
-	require.Eventually(t, func() bool {
-		byState := registry.ServicesByState()
-		return len(byState[StateStarting]) == 2
-	}, time.Second, time.Millisecond)
-
-	close(s1.healthyC)
-
-	require.NoError(t, registry.AwaitHealthy(ctx))
-	assert.True(t, registry.IsHealthy())
-
-	assert.NoError(t, registry.Stop(ctx))
-}

pkg/flagger/registry.go

@@ -3,15 +3,16 @@ package flagger
 import "github.com/SigNoz/signoz/pkg/types/featuretypes"
 
 var (
-	FeatureUseSpanMetrics         = featuretypes.MustNewName("use_span_metrics")
-	FeatureKafkaSpanEval          = featuretypes.MustNewName("kafka_span_eval")
-	FeatureHideRootUser           = featuretypes.MustNewName("hide_root_user")
-	FeaturePutMetersInZeus        = featuretypes.MustNewName("put_meters_in_zeus")
-	FeatureUseMeterReporter       = featuretypes.MustNewName("use_meter_reporter")
-	FeatureUseJSONBody            = featuretypes.MustNewName("use_json_body")
-	FeatureEnableAIObservability  = featuretypes.MustNewName("enable_ai_observability")
-	FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
-	FeatureResolveSemconvFamilies = featuretypes.MustNewName("resolve_semconv_families")
+	FeatureUseSpanMetrics            = featuretypes.MustNewName("use_span_metrics")
+	FeatureKafkaSpanEval             = featuretypes.MustNewName("kafka_span_eval")
+	FeatureHideRootUser              = featuretypes.MustNewName("hide_root_user")
+	FeaturePutMetersInZeus           = featuretypes.MustNewName("put_meters_in_zeus")
+	FeatureUseMeterReporter          = featuretypes.MustNewName("use_meter_reporter")
+	FeatureUseJSONBody               = featuretypes.MustNewName("use_json_body")
+	FeatureEnableAIObservability     = featuretypes.MustNewName("enable_ai_observability")
+	FeatureEnableMetricsReduction    = featuretypes.MustNewName("enable_metrics_reduction")
+	FeatureUsePrometheusClickhouseV2 = featuretypes.MustNewName("use_prometheus_clickhouse_v2")
+	FeatureResolveSemconvFamilies    = featuretypes.MustNewName("resolve_semconv_families")
 )
 
 func MustNewRegistry() featuretypes.Registry {
@@ -80,6 +81,14 @@ func MustNewRegistry() featuretypes.Registry {
 			DefaultVariant: featuretypes.MustNewName("disabled"),
 			Variants:       featuretypes.NewBooleanVariants(),
 		},
+		&featuretypes.Feature{
+			Name:           FeatureUsePrometheusClickhouseV2,
+			Kind:           featuretypes.KindBoolean,
+			Stage:          featuretypes.StageExperimental,
+			Description:    "Runs PromQL queries on the clickhousev2 provider alongside the served engine result and logs any difference; serving is unaffected.",
+			DefaultVariant: featuretypes.MustNewName("disabled"),
+			Variants:       featuretypes.NewBooleanVariants(),
+		},
 		&featuretypes.Feature{
 			Name:           FeatureResolveSemconvFamilies,
 			Kind:           featuretypes.KindBoolean,

pkg/http/middleware/compress.go

@@ -1,17 +0,0 @@
-package middleware
-
-import (
-	"net/http"
-
-	gorillahandlers "github.com/gorilla/handlers"
-)
-
-type Compress struct{}
-
-func NewCompress() *Compress {
-	return &Compress{}
-}
-
-func (middleware *Compress) Wrap(next http.Handler) http.Handler {
-	return gorillahandlers.CompressHandler(next)
-}

pkg/http/middleware/cors.go

@@ -1,25 +0,0 @@
-package middleware
-
-import (
-	"net/http"
-
-	"github.com/rs/cors"
-)
-
-type Cors struct {
-	cors *cors.Cors
-}
-
-func NewCors() *Cors {
-	return &Cors{
-		cors: cors.New(cors.Options{
-			AllowedOrigins: []string{"*"},
-			AllowedMethods: []string{"GET", "DELETE", "POST", "PUT", "PATCH", "OPTIONS"},
-			AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "cache-control"},
-		}),
-	}
-}
-
-func (middleware *Cors) Wrap(next http.Handler) http.Handler {
-	return middleware.cors.Handler(next)
-}

pkg/http/middleware/otel.go

@@ -1,43 +0,0 @@
-package middleware
-
-import (
-	"net/http"
-	"slices"
-
-	"github.com/gorilla/mux"
-	"go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
-	"go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/propagation"
-	"go.opentelemetry.io/otel/trace"
-)
-
-// defaultExcludedRoutes are the health endpoints kept out of tracing/metrics to
-// avoid drowning telemetry in probe traffic.
-var defaultExcludedRoutes = []string{
-	"/api/v1/health",
-	"/api/v2/healthz",
-	"/api/v2/readyz",
-	"/api/v2/livez",
-}
-
-type Otel struct {
-	wrap mux.MiddlewareFunc
-}
-
-func NewOtel(service string, meterProvider metric.MeterProvider, tracerProvider trace.TracerProvider) *Otel {
-	return &Otel{
-		wrap: otelmux.Middleware(
-			service,
-			otelmux.WithMeterProvider(meterProvider),
-			otelmux.WithTracerProvider(tracerProvider),
-			otelmux.WithPropagators(propagation.NewCompositeTextMapPropagator(propagation.Baggage{}, propagation.TraceContext{})),
-			otelmux.WithFilter(func(r *http.Request) bool {
-				return !slices.Contains(defaultExcludedRoutes, r.URL.Path)
-			}),
-		),
-	}
-}
-
-func (middleware *Otel) Wrap(next http.Handler) http.Handler {
-	return middleware.wrap(next)
-}

pkg/http/server/config.go

@@ -1,89 +1,9 @@
 package server
 
-import (
-	"crypto/tls"
-	"time"
-
-	"github.com/SigNoz/signoz/pkg/errors"
-)
-
-var tlsVersions = map[string]uint16{
-	"1.2": tls.VersionTLS12,
-	"1.3": tls.VersionTLS13,
-}
-
+// Config holds the configuration for http.
 type Config struct {
-	// Address specifies the TCP address for the server to listen on, in the form "host:port".
+	//Address specifies the TCP address for the server to listen on, in the form "host:port".
 	// If empty, ":http" (port 80) is used. The service names are defined in RFC 6335 and assigned by IANA.
 	// See net.Dial for details of the address format.
 	Address string `mapstructure:"address"`
-
-	// ReadTimeout bounds reading an entire request, including the body. Zero means no timeout.
-	ReadTimeout time.Duration `mapstructure:"read_timeout"`
-
-	// WriteTimeout bounds writing the response. Zero means no timeout, required for
-	// streaming endpoints that hold the connection open.
-	WriteTimeout time.Duration `mapstructure:"write_timeout"`
-
-	TLS TLS `mapstructure:"tls"`
-}
-
-type TLS struct {
-	Enabled bool `mapstructure:"enabled"`
-
-	// The full path to the certificate file.
-	CertFile string `mapstructure:"cert_file"`
-
-	// The full path to the key file.
-	KeyFile string `mapstructure:"key_file"`
-
-	// MinVersion is the minimum acceptable TLS version, "1.2" or "1.3". Empty uses the Go default.
-	MinVersion string `mapstructure:"min_version"`
-}
-
-func (c Config) Validate() error {
-	if !c.TLS.Enabled {
-		return nil
-	}
-
-	if c.TLS.CertFile == "" || c.TLS.KeyFile == "" {
-		return errors.NewInvalidInputf(errors.CodeInvalidInput, "tls::cert_file and tls::key_file are required when tls is enabled")
-	}
-
-	_, err := tlsVersion(c.TLS.MinVersion)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (tlsConfig TLS) Config() (*tls.Config, error) {
-	cert, err := tls.LoadX509KeyPair(tlsConfig.CertFile, tlsConfig.KeyFile)
-	if err != nil {
-		return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "cannot load tls::cert_file and tls::key_file: %v", err)
-	}
-
-	minVersion, err := tlsVersion(tlsConfig.MinVersion)
-	if err != nil {
-		return nil, err
-	}
-
-	return &tls.Config{
-		Certificates: []tls.Certificate{cert},
-		MinVersion:   minVersion,
-	}, nil
-}
-
-func tlsVersion(name string) (uint16, error) {
-	if name == "" {
-		return 0, nil
-	}
-
-	version, ok := tlsVersions[name]
-	if !ok {
-		return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid tls version %q, must be \"1.2\" or \"1.3\"", name)
-	}
-
-	return version, nil
 }

pkg/http/server/server.go

@@ -28,30 +28,17 @@ func New(logger *slog.Logger, cfg Config, handler http.Handler) (*Server, error)
 		return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "cannot build http server, logger is required")
 	}
 
-	if err := cfg.Validate(); err != nil {
-		return nil, err
-	}
-
 	srv := &http.Server{
 		Addr:           cfg.Address,
 		Handler:        handler,
-		ReadTimeout:    cfg.ReadTimeout,
-		WriteTimeout:   cfg.WriteTimeout,
+		ReadTimeout:    10 * time.Second,
+		WriteTimeout:   10 * time.Second,
 		MaxHeaderBytes: 1 << 20,
 	}
 
-	if cfg.TLS.Enabled {
-		tlsConfig, err := cfg.TLS.Config()
-		if err != nil {
-			return nil, err
-		}
-
-		srv.TLSConfig = tlsConfig
-	}
-
 	return &Server{
 		srv:     srv,
-		logger:  logger.With(slog.String("pkg", "github.com/SigNoz/signoz/pkg/http/server")),
+		logger:  logger.With(slog.String("pkg", "go.signoz.io/pkg/http/server")),
 		handler: handler,
 		cfg:     cfg,
 	}, nil
@@ -59,18 +46,11 @@ func New(logger *slog.Logger, cfg Config, handler http.Handler) (*Server, error)
 
 func (server *Server) Start(ctx context.Context) error {
 	server.logger.InfoContext(ctx, "starting http server", slog.String("address", server.srv.Addr))
-
-	var err error
-	if server.cfg.TLS.Enabled {
-		// The certificate is already loaded in TLSConfig, so ListenAndServeTLS needs no file paths.
-		err = server.srv.ListenAndServeTLS("", "")
-	} else {
-		err = server.srv.ListenAndServe()
-	}
-
-	if err != nil && err != http.ErrServerClosed {
-		server.logger.ErrorContext(ctx, "failed to start server", errors.Attr(err))
-		return err
+	if err := server.srv.ListenAndServe(); err != nil {
+		if err != http.ErrServerClosed {
+			server.logger.ErrorContext(ctx, "failed to start server", errors.Attr(err))
+			return err
+		}
 	}
 	return nil
 }

pkg/http/server/server_test.go

@@ -1,243 +0,0 @@
-package server
-
-import (
-	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
-	"io"
-	"log/slog"
-	"math/big"
-	"net"
-	"net/http"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestNew(t *testing.T) {
-	logger := slog.New(slog.DiscardHandler)
-	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})
-	certFile, keyFile := writeSelfSignedCert(t)
-
-	corruptFile := filepath.Join(t.TempDir(), "corrupt.crt")
-	require.NoError(t, os.WriteFile(corruptFile, []byte("not a pem"), 0o644))
-
-	testCases := []struct {
-		name       string
-		config     Config
-		err        bool
-		minVersion uint16
-	}{
-		{
-			name:   "TLSDisabled",
-			config: Config{},
-		},
-		{
-			name:   "TLSDisabled_WithCertAndKey",
-			config: Config{TLS: TLS{CertFile: "ignored.crt", KeyFile: "ignored.key"}},
-		},
-		{
-			name:   "TLSEnabled_WithoutCertAndKey",
-			config: Config{TLS: TLS{Enabled: true}},
-			err:    true,
-		},
-		{
-			name:   "TLSEnabled_WithoutKey",
-			config: Config{TLS: TLS{Enabled: true, CertFile: "server.crt"}},
-			err:    true,
-		},
-		{
-			name:   "TLSEnabled_WithoutCert",
-			config: Config{TLS: TLS{Enabled: true, KeyFile: "server.key"}},
-			err:    true,
-		},
-		{
-			name:   "TLSEnabled_InvalidMinVersion",
-			config: Config{TLS: TLS{Enabled: true, CertFile: "tls.crt", KeyFile: "tls.key", MinVersion: "1.1"}},
-			err:    true,
-		},
-		{
-			name:   "TLSEnabled_MissingFiles",
-			config: Config{TLS: TLS{Enabled: true, CertFile: "missing.crt", KeyFile: "missing.key"}},
-			err:    true,
-		},
-		{
-			name:   "TLSEnabled_CorruptCertFile",
-			config: Config{TLS: TLS{Enabled: true, CertFile: corruptFile, KeyFile: keyFile}},
-			err:    true,
-		},
-		{
-			name:   "TLSEnabled_DefaultVersions",
-			config: Config{TLS: TLS{Enabled: true, CertFile: certFile, KeyFile: keyFile}},
-		},
-		{
-			name:       "TLSEnabled_WithMin",
-			config:     Config{TLS: TLS{Enabled: true, CertFile: certFile, KeyFile: keyFile, MinVersion: "1.3"}},
-			minVersion: tls.VersionTLS13,
-		},
-	}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			server, err := New(logger, testCase.config, handler)
-			if testCase.err {
-				assert.Error(t, err)
-				return
-			}
-
-			require.NoError(t, err)
-			if !testCase.config.TLS.Enabled {
-				assert.Nil(t, server.srv.TLSConfig)
-				return
-			}
-
-			require.NotNil(t, server.srv.TLSConfig)
-			assert.Len(t, server.srv.TLSConfig.Certificates, 1)
-			assert.Equal(t, testCase.minVersion, server.srv.TLSConfig.MinVersion)
-		})
-	}
-}
-
-func TestStartWithTLS(t *testing.T) {
-	certFile, keyFile := writeSelfSignedCert(t)
-	addr := freeAddr(t)
-
-	server, err := New(
-		slog.New(slog.DiscardHandler),
-		Config{Address: addr, TLS: TLS{Enabled: true, CertFile: certFile, KeyFile: keyFile}},
-		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) }),
-	)
-	require.NoError(t, err)
-
-	errC := make(chan error, 1)
-	go func() { errC <- server.Start(context.Background()) }()
-
-	certPEM, err := os.ReadFile(certFile)
-	require.NoError(t, err)
-	pool := x509.NewCertPool()
-	require.True(t, pool.AppendCertsFromPEM(certPEM))
-	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}}}
-
-	var (
-		statusCode int
-		body       []byte
-		tlsVersion uint16
-	)
-	require.Eventually(t, func() bool {
-		resp, err := client.Get("https://" + addr)
-		if err != nil {
-			return false
-		}
-		defer func() { _ = resp.Body.Close() }()
-
-		body, err = io.ReadAll(resp.Body)
-		if err != nil {
-			return false
-		}
-
-		statusCode = resp.StatusCode
-		if resp.TLS != nil {
-			tlsVersion = resp.TLS.Version
-		}
-		return true
-	}, 5*time.Second, 25*time.Millisecond)
-
-	assert.Equal(t, http.StatusOK, statusCode)
-	assert.Equal(t, "ok", string(body))
-	assert.GreaterOrEqual(t, tlsVersion, uint16(tls.VersionTLS12))
-
-	plainResp, err := http.Get("http://" + addr)
-	require.NoError(t, err)
-	_ = plainResp.Body.Close()
-	assert.Equal(t, http.StatusBadRequest, plainResp.StatusCode)
-
-	require.NoError(t, server.Stop(context.Background()))
-	require.NoError(t, <-errC)
-}
-
-func TestStartWithoutTLS(t *testing.T) {
-	addr := freeAddr(t)
-
-	server, err := New(
-		slog.New(slog.DiscardHandler),
-		Config{Address: addr},
-		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("pong")) }),
-	)
-	require.NoError(t, err)
-
-	errC := make(chan error, 1)
-	go func() { errC <- server.Start(context.Background()) }()
-
-	var (
-		statusCode    int
-		tlsNegotiated bool
-	)
-	require.Eventually(t, func() bool {
-		resp, err := http.Get("http://" + addr)
-		if err != nil {
-			return false
-		}
-		defer func() { _ = resp.Body.Close() }()
-
-		statusCode = resp.StatusCode
-		tlsNegotiated = resp.TLS != nil
-		return true
-	}, 5*time.Second, 25*time.Millisecond)
-
-	assert.Equal(t, http.StatusOK, statusCode)
-	assert.False(t, tlsNegotiated)
-
-	require.NoError(t, server.Stop(context.Background()))
-	require.NoError(t, <-errC)
-}
-
-func freeAddr(t *testing.T) string {
-	t.Helper()
-
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	require.NoError(t, err)
-	defer func() { _ = listener.Close() }()
-
-	return listener.Addr().String()
-}
-
-func writeSelfSignedCert(t *testing.T) (string, string) {
-	t.Helper()
-
-	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	require.NoError(t, err)
-
-	template := &x509.Certificate{
-		SerialNumber: big.NewInt(1),
-		Subject:      pkix.Name{CommonName: "localhost"},
-		NotBefore:    time.Now().Add(-time.Hour),
-		NotAfter:     time.Now().Add(time.Hour),
-		KeyUsage:     x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
-	}
-
-	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
-	require.NoError(t, err)
-
-	keyDER, err := x509.MarshalECPrivateKey(key)
-	require.NoError(t, err)
-
-	dir := t.TempDir()
-	certFile := filepath.Join(dir, "server.crt")
-	keyFile := filepath.Join(dir, "server.key")
-
-	require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644))
-	require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600))
-
-	return certFile, keyFile
-}

pkg/pprof/httppprof/provider.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	nethttppprof "net/http/pprof"
 	runtimepprof "runtime/pprof"
-	"time"
 
 	"github.com/SigNoz/signoz/pkg/factory"
 	httpserver "github.com/SigNoz/signoz/pkg/http/server"
@@ -24,7 +23,7 @@ func NewFactory() factory.ProviderFactory[pprof.PProf, pprof.Config] {
 func New(_ context.Context, settings factory.ProviderSettings, config pprof.Config) (pprof.PProf, error) {
 	server, err := httpserver.New(
 		settings.Logger.With(slog.String("pkg", "github.com/SigNoz/signoz/pkg/pprof/httppprof")),
-		httpserver.Config{Address: config.Address, ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second},
+		httpserver.Config{Address: config.Address},
 		newHandler(),
 	)
 	if err != nil {

pkg/prometheus/clickhouseprometheus/capture.go

@@ -0,0 +1,90 @@
+package clickhouseprometheus
+
+import (
+	"context"
+	"sync"
+
+	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/storage"
+)
+
+// statementRecorder collects the statements a PromQL evaluation would run.
+// Safe for concurrent use: the engine may Select selectors concurrently.
+type statementRecorder struct {
+	mu         sync.Mutex
+	statements []prometheus.CapturedStatement
+}
+
+func (r *statementRecorder) record(query string, args []any) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.statements = append(r.statements, prometheus.CapturedStatement{Query: query, Args: args})
+}
+
+func (r *statementRecorder) Statements() []prometheus.CapturedStatement {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	out := make([]prometheus.CapturedStatement, len(r.statements))
+	copy(out, r.statements)
+	return out
+}
+
+// captureClient builds the same SQL as the real client but records it and
+// returns an empty result instead of executing.
+type captureClient struct {
+	*client
+	recorder *statementRecorder
+}
+
+func (c *captureClient) Read(ctx context.Context, query *prompb.Query, _ bool) (storage.SeriesSet, error) {
+	// Raw-SQL passthrough ({job="rawsql", query="..."}): record the raw query.
+	if len(query.Matchers) == 2 {
+		var hasJob bool
+		var queryString string
+		for _, m := range query.Matchers {
+			if m.Type == prompb.LabelMatcher_EQ && m.Name == "job" && m.Value == "rawsql" {
+				hasJob = true
+			}
+			if m.Type == prompb.LabelMatcher_EQ && m.Name == "query" {
+				queryString = m.Value
+			}
+		}
+		if hasJob && queryString != "" {
+			c.recorder.record(queryString, nil)
+			return storage.EmptySeriesSet(), nil
+		}
+	}
+
+	// Without executing the series lookup, only an exact-name selector's
+	// metric name is known.
+	var metricNames []string
+	for _, matcher := range query.Matchers {
+		if matcher.Name == "__name__" && matcher.Type == prompb.LabelMatcher_EQ {
+			metricNames = []string{matcher.Value}
+		}
+	}
+
+	// Build the executing path's queries, but only record them.
+	sub, err := seriesLookupQuery(query, true)
+	if err != nil {
+		return nil, err
+	}
+	samplesQuery, samplesArgs := buildSamplesQuery(int64(query.StartTimestampMs), int64(query.EndTimestampMs), metricNames, sub)
+	c.recorder.record(samplesQuery, samplesArgs)
+
+	return storage.EmptySeriesSet(), nil
+}
+
+// captureQueryable adapts the capturing read client to storage.Queryable.
+type captureQueryable struct {
+	inner storage.SampleAndChunkQueryable
+}
+
+func (c captureQueryable) Querier(mint, maxt int64) (storage.Querier, error) {
+	querier, err := c.inner.Querier(mint, maxt)
+	if err != nil {
+		return nil, err
+	}
+	return storage.NewMergeQuerier(nil, []storage.Querier{querier}, storage.ChainedSeriesMerge), nil
+}

pkg/prometheus/clickhouseprometheus/client.go

@@ -0,0 +1,517 @@
+package clickhouseprometheus
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"sort"
+	"sync"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
+	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/cespare/xxhash/v2"
+	"github.com/huandu/go-sqlbuilder"
+	promValue "github.com/prometheus/prometheus/model/value"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/remote"
+)
+
+type client struct {
+	settings       factory.ScopedProviderSettings
+	telemetryStore telemetrystore.TelemetryStore
+}
+
+func NewReadClient(settings factory.ScopedProviderSettings, telemetryStore telemetrystore.TelemetryStore) remote.ReadClient {
+	return &client{
+		settings:       settings,
+		telemetryStore: telemetryStore,
+	}
+}
+
+func (client *client) Read(ctx context.Context, query *prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
+	if len(query.Matchers) == 2 {
+		var hasJob bool
+		var queryString string
+		for _, m := range query.Matchers {
+			if m.Type == prompb.LabelMatcher_EQ && m.Name == "job" && m.Value == "rawsql" {
+				hasJob = true
+			}
+			if m.Type == prompb.LabelMatcher_EQ && m.Name == "query" {
+				queryString = m.Value
+			}
+		}
+
+		if hasJob && queryString != "" {
+			res, err := client.queryRaw(ctx, queryString, int64(query.EndTimestampMs))
+			if err != nil {
+				return nil, err
+			}
+
+			return remote.FromQueryResult(sortSeries, res), nil
+		}
+	}
+
+	lookup, err := seriesLookupQuery(query, false)
+	if err != nil {
+		return nil, err
+	}
+	lookupSQL, lookupArgs := lookup.BuildWithFlavor(sqlbuilder.ClickHouse)
+
+	fingerprints, metricNames, err := client.getFingerprintsFromClickhouseQuery(ctx, lookupSQL, lookupArgs)
+	if err != nil {
+		return nil, err
+	}
+	if len(fingerprints) == 0 {
+		return remote.FromQueryResult(sortSeries, new(prompb.QueryResult)), nil
+	}
+
+	sub, err := seriesLookupQuery(query, true)
+	if err != nil {
+		return nil, err
+	}
+	samplesSQL, samplesArgs := buildSamplesQuery(int64(query.StartTimestampMs), int64(query.EndTimestampMs), metricNames, sub)
+
+	res := new(prompb.QueryResult)
+	timeseries, err := client.querySamples(ctx, samplesSQL, samplesArgs, fingerprints)
+	if err != nil {
+		return nil, err
+	}
+	res.Timeseries = timeseries
+
+	return remote.FromQueryResult(sortSeries, res), nil
+}
+
+func (c *client) ReadMultiple(ctx context.Context, queries []*prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
+	if len(queries) == 0 {
+		return storage.EmptySeriesSet(), nil
+	}
+	if len(queries) == 1 {
+		return c.Read(ctx, queries[0], sortSeries)
+	}
+
+	type result struct {
+		ss  storage.SeriesSet
+		err error
+	}
+
+	results := make([]result, len(queries))
+	var wg sync.WaitGroup
+	wg.Add(len(queries))
+	for i, q := range queries {
+		go func(i int, q *prompb.Query) {
+			defer wg.Done()
+			ss, err := c.Read(ctx, q, sortSeries)
+			results[i] = result{ss, err}
+		}(i, q)
+	}
+	wg.Wait()
+
+	sets := make([]storage.SeriesSet, 0, len(queries))
+	for _, r := range results {
+		if r.err != nil {
+			return nil, r.err
+		}
+		sets = append(sets, r.ss)
+	}
+	return storage.NewMergeSeriesSet(sets, 0, storage.ChainedSeriesMerge), nil
+}
+
+// anchorRegex makes a pattern fully anchored, the way Prometheus compiles
+// matcher regexes; ClickHouse's match() would otherwise substring-match.
+func anchorRegex(pattern string) string {
+	return "^(?:" + pattern + ")$"
+}
+
+// seriesLookupQuery builds the time-series lookup. It returns a builder so
+// the samples query can embed it as a subquery with the args merged in
+// render order by the builder instead of hand-numbered placeholders.
+func seriesLookupQuery(query *prompb.Query, subQuery bool) (*sqlbuilder.SelectBuilder, error) {
+	sb := sqlbuilder.NewSelectBuilder()
+	if subQuery {
+		sb.Select("fingerprint")
+	} else {
+		sb.Select("fingerprint", "any(labels)")
+	}
+
+	start, end, tableName := getStartAndEndAndTableName(query.StartTimestampMs, query.EndTimestampMs)
+	sb.From(databaseName + "." + tableName)
+
+	sb.Where("temporality IN ['Cumulative', 'Unspecified']")
+	// Inclusive upper bound: registration rows are hour-floored by the
+	// exporter, so a series first registered in the hour starting exactly at
+	// `end` would otherwise be invisible while its samples (<= end) are in
+	// range.
+	sb.Where(fmt.Sprintf("unix_milli >= %d AND unix_milli <= %d", start, end))
+
+	for _, m := range query.Matchers {
+		if m.Name == "__name__" {
+			// __name__ maps onto the metric_name column per matcher type;
+			// reducing regex/negated/absent name matchers to one equality
+			// made such selectors silently return empty.
+			switch m.Type {
+			case prompb.LabelMatcher_EQ:
+				sb.Where(sb.E("metric_name", m.Value))
+			case prompb.LabelMatcher_NEQ:
+				sb.Where(sb.NE("metric_name", m.Value))
+			case prompb.LabelMatcher_RE:
+				sb.Where(fmt.Sprintf("match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
+			case prompb.LabelMatcher_NRE:
+				sb.Where(fmt.Sprintf("not match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
+			default:
+				return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported or invalid matcher type: %s", m.Type.String())
+			}
+			continue
+		}
+		switch m.Type {
+		case prompb.LabelMatcher_EQ:
+			sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) = %s", sb.Var(m.Name), sb.Var(m.Value)))
+		case prompb.LabelMatcher_NEQ:
+			sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) != %s", sb.Var(m.Name), sb.Var(m.Value)))
+		case prompb.LabelMatcher_RE:
+			sb.Where(fmt.Sprintf("match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
+		case prompb.LabelMatcher_NRE:
+			sb.Where(fmt.Sprintf("not match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
+		default:
+			return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported or invalid matcher type: %s", m.Type.String())
+		}
+	}
+
+	sb.GroupBy("fingerprint")
+	return sb, nil
+}
+
+func (client *client) getFingerprintsFromClickhouseQuery(ctx context.Context, query string, args []any) (map[uint64][]prompb.Label, []string, error) {
+	ctx = client.withClickhousePrometheusContext(ctx, "getFingerprintsFromClickhouseQuery")
+	rows, err := client.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+
+	fingerprints := make(map[uint64][]prompb.Label)
+	nameSet := make(map[string]struct{})
+
+	var fingerprint uint64
+	var labelString string
+	for rows.Next() {
+		if err = rows.Scan(&fingerprint, &labelString); err != nil {
+			return nil, nil, err
+		}
+
+		labels, metricName, err := unmarshalLabels(labelString)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		fingerprints[fingerprint] = labels
+		if metricName != "" {
+			nameSet[metricName] = struct{}{}
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, nil, err
+	}
+
+	metricNames := make([]string, 0, len(nameSet))
+	for name := range nameSet {
+		metricNames = append(metricNames, name)
+	}
+	sort.Strings(metricNames)
+
+	return fingerprints, metricNames, nil
+}
+
+// buildSamplesQuery renders the samples SQL for the series selected by
+// subQuery. The metric_name condition exists only for primary-key pruning;
+// the fingerprint filter already selects the right rows.
+//
+// Time bounds are inclusive on both ends because that is Prometheus's
+// storage contract: Select(mint, maxt) returns [start, end] and the engine
+// itself trims each evaluation window to left-open (T-window, T], so the
+// sample at exactly `end` belongs to the last point. This deliberately
+// differs from the query builder's `unix_milli < end`, which is correct for
+// its own model — toStartOfInterval buckets covering [t, t+step), where a
+// sample at `end` falls in an unrendered bucket and end-exclusive ranges
+// tile exactly across cached time slices.
+func buildSamplesQuery(start int64, end int64, metricNames []string, sub *sqlbuilder.SelectBuilder) (string, []any) {
+	sb := sqlbuilder.NewSelectBuilder()
+	sb.Select("metric_name", "fingerprint", "unix_milli", "value", "flags")
+	sb.From(databaseName + "." + distributedSamplesV4)
+
+	if len(metricNames) > 0 {
+		names := make([]any, len(metricNames))
+		for i, name := range metricNames {
+			names[i] = name
+		}
+		sb.Where(sb.In("metric_name", names...))
+	}
+	sb.Where(fmt.Sprintf("fingerprint GLOBAL IN (%s)", sb.Var(sub)))
+	sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
+	sb.OrderBy("fingerprint", "unix_milli")
+
+	return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+}
+
+func (client *client) querySamples(ctx context.Context, query string, args []any, fingerprints map[uint64][]prompb.Label) ([]*prompb.TimeSeries, error) {
+	ctx = client.withClickhousePrometheusContext(ctx, "querySamples")
+
+	rows, err := client.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var res []*prompb.TimeSeries
+	var ts *prompb.TimeSeries
+	var metricName string
+	var fingerprint, prevFingerprint uint64
+	var timestampMs, prevTimestamp int64
+	var value float64
+	var flags uint32
+
+	prevTimestamp = math.MinInt64
+
+	for rows.Next() {
+		if err := rows.Scan(&metricName, &fingerprint, &timestampMs, &value, &flags); err != nil {
+			return nil, err
+		}
+
+		// collect samples in time series
+		if fingerprint != prevFingerprint {
+			// add collected time series to result
+			prevFingerprint = fingerprint
+			if ts != nil {
+				res = append(res, ts)
+			}
+
+			labels := fingerprints[fingerprint]
+			ts = &prompb.TimeSeries{
+				Labels: labels,
+			}
+			prevTimestamp = math.MinInt64
+		}
+
+		if flags&1 == 1 {
+			value = math.Float64frombits(promValue.StaleNaN)
+		}
+
+		if timestampMs == prevTimestamp {
+			continue
+		}
+		prevTimestamp = timestampMs
+
+		// add samples to current time series
+		ts.Samples = append(ts.Samples, prompb.Sample{
+			Timestamp: timestampMs,
+			Value:     value,
+		})
+	}
+
+	// add last time series
+	if ts != nil {
+		res = append(res, ts)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return mergeSeriesWithIdenticalLabels(res), nil
+}
+
+// mergeSeriesWithIdenticalLabels collapses series sharing one labelset into
+// one series each. Distinct fingerprints can map to one labelset: a label
+// value goes empty over a series' lifetime (#8563), the fingerprint
+// algorithm changes across exporter versions, or the env changes. The
+// engine treats the labelset as series identity — duplicates raise
+// "duplicate series", and #8563's workaround of injecting a synthetic
+// fingerprint label silently broke without() and vector matching. Merging
+// at the last point before hand-off keeps any future input-side label
+// normalization collision-safe. Grouping is by an order-insensitive 64-bit
+// hash so the common no-collision case costs one hash and one map insert
+// per series; hash-equal groups are confirmed by exact labelset equality
+// before any merge. Regression:
+// TestClient_QuerySamplesMergesIdenticalLabelSets and
+// tests/integration/tests/promqlconformance/02_fingerprint_probe.py.
+func mergeSeriesWithIdenticalLabels(series []*prompb.TimeSeries) []*prompb.TimeSeries {
+	if len(series) < 2 {
+		return series
+	}
+
+	groups := make(map[uint64][]*prompb.TimeSeries, len(series))
+	order := make([]uint64, 0, len(series))
+	for _, ts := range series {
+		key := labelsHash(ts.Labels)
+		if _, ok := groups[key]; !ok {
+			order = append(order, key)
+		}
+		groups[key] = append(groups[key], ts)
+	}
+	if len(order) == len(series) {
+		return series
+	}
+
+	res := make([]*prompb.TimeSeries, 0, len(order))
+	for _, key := range order {
+		group := groups[key]
+		if len(group) == 1 {
+			res = append(res, group[0])
+			continue
+		}
+		for _, sub := range splitByLabelSet(group) {
+			if len(sub) == 1 {
+				res = append(res, sub[0])
+				continue
+			}
+			res = append(res, mergeSamples(sub))
+		}
+	}
+	return res
+}
+
+var labelHashSep = []byte{0xff}
+
+// labelsHash combines per-label hashes commutatively, so the stored JSON's
+// key order (not canonical across fingerprints) needs no sort.
+func labelsHash(lbls []prompb.Label) uint64 {
+	var h uint64
+	var d xxhash.Digest
+	for _, l := range lbls {
+		d.Reset()
+		_, _ = d.WriteString(l.Name)
+		_, _ = d.Write(labelHashSep)
+		_, _ = d.WriteString(l.Value)
+		h += d.Sum64()
+	}
+	return h
+}
+
+// splitByLabelSet partitions a hash-equal group into sub-groups of exactly
+// equal labelsets, preserving input order; series that merely collide on the
+// 64-bit hash must not be merged.
+func splitByLabelSet(group []*prompb.TimeSeries) [][]*prompb.TimeSeries {
+	var out [][]*prompb.TimeSeries
+outer:
+	for _, ts := range group {
+		for i, sub := range out {
+			if labelSetsEqual(sub[0].Labels, ts.Labels) {
+				out[i] = append(out[i], ts)
+				continue outer
+			}
+		}
+		out = append(out, []*prompb.TimeSeries{ts})
+	}
+	return out
+}
+
+func labelSetsEqual(a, b []prompb.Label) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for _, la := range a {
+		found := false
+		for _, lb := range b {
+			if la.Name == lb.Name {
+				found = la.Value == lb.Value
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+	return true
+}
+
+// mergeSamples k-way merges sample streams that share one labelset. On
+// equal timestamps the highest fingerprint wins: the input is in ascending
+// fingerprint order (samples SQL), keeping the choice deterministic.
+func mergeSamples(group []*prompb.TimeSeries) *prompb.TimeSeries {
+	merged := &prompb.TimeSeries{Labels: group[0].Labels}
+	idx := make([]int, len(group))
+	for {
+		minTs := int64(math.MaxInt64)
+		for i, ts := range group {
+			if idx[i] < len(ts.Samples) && ts.Samples[idx[i]].Timestamp < minTs {
+				minTs = ts.Samples[idx[i]].Timestamp
+			}
+		}
+		if minTs == math.MaxInt64 {
+			return merged
+		}
+		var chosen prompb.Sample
+		for i, ts := range group {
+			if idx[i] < len(ts.Samples) && ts.Samples[idx[i]].Timestamp == minTs {
+				chosen = ts.Samples[idx[i]]
+				idx[i]++
+			}
+		}
+		merged.Samples = append(merged.Samples, chosen)
+	}
+}
+
+func (client *client) queryRaw(ctx context.Context, query string, ts int64) (*prompb.QueryResult, error) {
+	ctx = client.withClickhousePrometheusContext(ctx, "queryRaw")
+
+	rows, err := client.telemetryStore.ClickhouseDB().Query(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	columns := rows.Columns()
+	var res prompb.QueryResult
+	targets := make([]any, len(columns))
+	for i := range targets {
+		targets[i] = new(scanner)
+	}
+
+	for rows.Next() {
+		if err = rows.Scan(targets...); err != nil {
+			return nil, err
+		}
+
+		labels := make([]prompb.Label, 0, len(columns))
+		var value float64
+		for i, c := range columns {
+			v := targets[i].(*scanner)
+			switch c {
+			case "value":
+				value = v.f
+			default:
+				labels = append(labels, prompb.Label{
+					Name:  c,
+					Value: v.s,
+				})
+			}
+		}
+
+		res.Timeseries = append(res.Timeseries, &prompb.TimeSeries{
+			Labels: labels,
+			Samples: []prompb.Sample{{
+				Value:     value,
+				Timestamp: ts,
+			}},
+		})
+	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return &res, nil
+}
+
+func (client *client) withClickhousePrometheusContext(ctx context.Context, functionName string) context.Context {
+	comments := map[string]string{
+		instrumentationtypes.TelemetrySignal:  telemetrytypes.SignalMetrics.StringValue(),
+		instrumentationtypes.CodeNamespace:    "clickhouse-prometheus",
+		instrumentationtypes.CodeFunctionName: functionName,
+	}
+	return ctxtypes.NewContextWithCommentVals(ctx, comments)
+}

pkg/prometheus/clickhouseprometheus/client_query_test.go

@@ -0,0 +1,382 @@
+package clickhouseprometheus
+
+import (
+	"context"
+	"sort"
+	"testing"
+
+	cmock "github.com/SigNoz/clickhouse-go-mock"
+	"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
+	"github.com/stretchr/testify/require"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/huandu/go-sqlbuilder"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/stretchr/testify/assert"
+)
+
+// Test for querySamples method.
+func TestClient_QuerySamples(t *testing.T) {
+	ctx := context.Background()
+	cols := make([]cmock.ColumnType, 0)
+	cols = append(cols, cmock.ColumnType{Name: "metric_name", Type: "String"})
+	cols = append(cols, cmock.ColumnType{Name: "fingerprint", Type: "UInt64"})
+	cols = append(cols, cmock.ColumnType{Name: "unix_milli", Type: "Int64"})
+	cols = append(cols, cmock.ColumnType{Name: "value", Type: "Float64"})
+	cols = append(cols, cmock.ColumnType{Name: "flags", Type: "UInt32"})
+	tests := []struct {
+		name               string
+		start              int64
+		end                int64
+		fingerprints       map[uint64][]prompb.Label
+		metricNames        []string
+		subQuery           string
+		args               []any
+		setupMock          func(mock cmock.ClickConnMockCommon, args ...any)
+		expectedTimeSeries int
+		expectError        bool
+		description        string
+		result             []*prompb.TimeSeries
+	}{
+		{
+			name:  "successful samples retrieval",
+			start: int64(1000),
+			end:   int64(2000),
+			fingerprints: map[uint64][]prompb.Label{
+				123: {
+					{Name: "__name__", Value: "cpu_usage"},
+					{Name: "instance", Value: "localhost:9090"},
+				},
+				456: {
+					{Name: "__name__", Value: "cpu_usage"},
+					{Name: "instance", Value: "localhost:9091"},
+				},
+			},
+			metricNames:        []string{"cpu_usage"},
+			subQuery:           "SELECT metric_name, fingerprint, unix_milli, value, flags",
+			expectedTimeSeries: 2,
+			expectError:        false,
+			description:        "Should successfully retrieve samples for multiple time series",
+			setupMock: func(mock cmock.ClickConnMockCommon, args ...any) {
+				values := [][]interface{}{
+					{"cpu_usage", uint64(123), int64(1001), float64(1.1), uint32(0)},
+					{"cpu_usage", uint64(123), int64(1001), float64(1.1), uint32(0)},
+					{"cpu_usage", uint64(456), int64(1001), float64(1.2), uint32(0)},
+					{"cpu_usage", uint64(456), int64(1001), float64(1.2), uint32(0)},
+					{"cpu_usage", uint64(456), int64(1001), float64(1.2), uint32(0)},
+				}
+				mock.ExpectQuery("SELECT metric_name, fingerprint, unix_milli, value, flags").WithArgs(args...).WillReturnRows(
+					cmock.NewRows(cols, values),
+				)
+			},
+			result: []*prompb.TimeSeries{
+				{
+					Labels: []prompb.Label{
+						{Name: "__name__", Value: "cpu_usage"},
+						{Name: "instance", Value: "localhost:9090"},
+					},
+					Samples: []prompb.Sample{
+						{Timestamp: 1001, Value: 1.1},
+					},
+				},
+				{
+					Labels: []prompb.Label{
+						{Name: "__name__", Value: "cpu_usage"},
+						{Name: "instance", Value: "localhost:9091"},
+					},
+					Samples: []prompb.Sample{
+						{Timestamp: 1001, Value: 1.2},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			telemetryStore := telemetrystoretest.New(telemetrystore.Config{Provider: "clickhouse"}, sqlmock.QueryMatcherRegexp)
+			readClient := client{telemetryStore: telemetryStore}
+			if tt.setupMock != nil {
+				tt.setupMock(telemetryStore.Mock(), "cpu_usage", tt.start, tt.end)
+
+			}
+			result, err := readClient.querySamples(ctx, tt.subQuery, []any{"cpu_usage", tt.start, tt.end}, tt.fingerprints)
+
+			if tt.expectError {
+				assert.Error(t, err)
+				assert.Nil(t, result)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expectedTimeSeries, len(result))
+				assert.Equal(t, result, tt.result)
+			}
+
+		})
+	}
+}
+
+// Regression for the duplicate-series class behind #8563: fingerprints
+// sharing one labelset must come back as one merged series, the higher
+// fingerprint winning equal timestamps.
+func TestClient_QuerySamplesMergesIdenticalLabelSets(t *testing.T) {
+	ctx := context.Background()
+	cols := []cmock.ColumnType{
+		{Name: "metric_name", Type: "String"},
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "unix_milli", Type: "Int64"},
+		{Name: "value", Type: "Float64"},
+		{Name: "flags", Type: "UInt32"},
+	}
+
+	canary := []prompb.Label{
+		{Name: "__name__", Value: "requests"},
+		{Name: "group", Value: "canary"},
+	}
+	production := []prompb.Label{
+		{Name: "__name__", Value: "requests"},
+		{Name: "group", Value: "production"},
+	}
+	fingerprints := map[uint64][]prompb.Label{
+		100: canary,
+		200: canary,
+		300: production,
+	}
+
+	telemetryStore := telemetrystoretest.New(telemetrystore.Config{Provider: "clickhouse"}, sqlmock.QueryMatcherRegexp)
+	// Rows arrive ordered by (fingerprint, unix_milli), matching the SQL.
+	values := [][]any{
+		{"requests", uint64(100), int64(1000), float64(1.0), uint32(0)},
+		{"requests", uint64(100), int64(2000), float64(2.0), uint32(0)},
+		{"requests", uint64(200), int64(2000), float64(20.0), uint32(0)},
+		{"requests", uint64(200), int64(3000), float64(30.0), uint32(0)},
+		{"requests", uint64(300), int64(1500), float64(5.0), uint32(0)},
+	}
+	telemetryStore.Mock().ExpectQuery("SELECT metric_name, fingerprint, unix_milli, value, flags").
+		WithArgs("requests", int64(1000), int64(3000)).
+		WillReturnRows(cmock.NewRows(cols, values))
+
+	readClient := client{telemetryStore: telemetryStore}
+	result, err := readClient.querySamples(ctx, "SELECT metric_name, fingerprint, unix_milli, value, flags", []any{"requests", int64(1000), int64(3000)}, fingerprints)
+	require.NoError(t, err)
+
+	assert.Equal(t, []*prompb.TimeSeries{
+		{
+			Labels: canary,
+			Samples: []prompb.Sample{
+				{Timestamp: 1000, Value: 1.0},
+				{Timestamp: 2000, Value: 20.0},
+				{Timestamp: 3000, Value: 30.0},
+			},
+		},
+		{
+			Labels: production,
+			Samples: []prompb.Sample{
+				{Timestamp: 1500, Value: 5.0},
+			},
+		},
+	}, result)
+}
+
+func TestClient_getFingerprintsFromClickhouseQuery(t *testing.T) {
+	cols := []cmock.ColumnType{
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "labels", Type: "String"},
+	}
+
+	sortLabels := func(ls []prompb.Label) {
+		sort.Slice(ls, func(i, j int) bool {
+			if ls[i].Name == ls[j].Name {
+				return ls[i].Value < ls[j].Value
+			}
+			return ls[i].Name < ls[j].Name
+		})
+	}
+
+	tests := []struct {
+		name       string
+		start, end int64
+		metricName string
+		subQuery   string
+		args       []any
+		setupMock  func(m cmock.ClickConnMockCommon, args ...any)
+		want       map[uint64][]prompb.Label
+		wantNames  []string
+		wantErr    bool
+	}{
+		{
+			name:       "happy-path - two fingerprints",
+			start:      1000,
+			end:        2000,
+			metricName: "cpu_usage",
+			subQuery:   `SELECT fingerprint,labels`,
+			// args slice is empty here, but test‑case still owns it
+			args: []any{},
+
+			setupMock: func(m cmock.ClickConnMockCommon, args ...any) {
+				rows := [][]any{
+					{uint64(123), `{"__name__":"cpu_usage","t1":"s1","t2":"s2"}`},
+					{uint64(234), `{"__name__":"cpu_usage","t1":"s1","t2":"s2","empty":""}`},
+				}
+				m.ExpectQuery(`SELECT fingerprint,labels`).WithArgs(args...).WillReturnRows(
+					cmock.NewRows(cols, rows),
+				)
+			},
+
+			// No synthetic fingerprint label (#8563), empty-valued labels
+			// dropped: both fingerprints present one labelset for
+			// querySamples to merge.
+			want: map[uint64][]prompb.Label{
+				123: {
+					{Name: "__name__", Value: "cpu_usage"},
+					{Name: "t1", Value: "s1"},
+					{Name: "t2", Value: "s2"},
+				},
+				234: {
+					{Name: "__name__", Value: "cpu_usage"},
+					{Name: "t1", Value: "s1"},
+					{Name: "t2", Value: "s2"},
+				},
+			},
+			wantNames: []string{"cpu_usage"},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := context.Background()
+			store := telemetrystoretest.New(
+				telemetrystore.Config{Provider: "clickhouse"},
+				sqlmock.QueryMatcherRegexp,
+			)
+
+			if tc.setupMock != nil {
+				tc.setupMock(store.Mock(), tc.args...)
+			}
+
+			c := client{telemetryStore: store}
+
+			got, gotNames, err := c.getFingerprintsFromClickhouseQuery(ctx, tc.subQuery, tc.args)
+			if tc.wantErr {
+				require.Error(t, err)
+				require.Nil(t, got)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, tc.wantNames, gotNames, "discovered metric names mismatch")
+			require.Equal(t, len(tc.want), len(got), "fingerprint map length mismatch")
+			for fp, expLabels := range tc.want {
+				gotLabels, ok := got[fp]
+				require.Truef(t, ok, "missing fingerprint %d", fp)
+
+				sortLabels(expLabels)
+				sortLabels(gotLabels)
+
+				assert.Equalf(t, expLabels, gotLabels, "labels mismatch for fingerprint %d", fp)
+			}
+		})
+	}
+}
+
+// Regression for nameless/regex-name selectors silently returning empty:
+// the old code reduced every __name__ matcher to `metric_name = <value>`
+// (empty string when absent). Regexes must come out anchored — Prometheus
+// matcher semantics, while ClickHouse match() substring-matches.
+func TestQueryToClickhouseQueryNameMatchers(t *testing.T) {
+	query := func(matchers ...*prompb.LabelMatcher) *prompb.Query {
+		return &prompb.Query{StartTimestampMs: 0, EndTimestampMs: 1000, Matchers: matchers}
+	}
+
+	tests := []struct {
+		name     string
+		query    *prompb.Query
+		contains []string
+		absent   []string
+		args     []any
+	}{
+		{
+			name:     "exact name",
+			query:    query(&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: "__name__", Value: "cpu_usage"}),
+			contains: []string{"metric_name = ?"},
+			args:     []any{"cpu_usage"},
+		},
+		{
+			name:     "regex name is anchored",
+			query:    query(&prompb.LabelMatcher{Type: prompb.LabelMatcher_RE, Name: "__name__", Value: ".+"}),
+			contains: []string{"match(metric_name, ?)"},
+			args:     []any{"^(?:.+)$"},
+		},
+		{
+			name: "nameless selector has no metric_name condition",
+			query: query(
+				&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: "job", Value: "api"},
+				&prompb.LabelMatcher{Type: prompb.LabelMatcher_NRE, Name: "group", Value: "can.*"},
+			),
+			contains: []string{
+				"JSONExtractString(labels, ?) = ?",
+				"not match(JSONExtractString(labels, ?), ?)",
+			},
+			absent: []string{"metric_name"},
+			args:   []any{"job", "api", "group", "^(?:can.*)$"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			lookup, err := seriesLookupQuery(tt.query, false)
+			require.NoError(t, err)
+			sql, args := lookup.BuildWithFlavor(sqlbuilder.ClickHouse)
+			for _, want := range tt.contains {
+				assert.Contains(t, sql, want)
+			}
+			for _, notWant := range tt.absent {
+				assert.NotContains(t, sql, notWant)
+			}
+			assert.Equal(t, tt.args, args)
+		})
+	}
+}
+
+// The samples query narrows by the metric names the lookup discovered and
+// embeds the series lookup as a subquery, the builder merging its args in
+// render order.
+func TestBuildSamplesQueryMetricNames(t *testing.T) {
+	sub := sqlbuilder.NewSelectBuilder()
+	sub.Select("fingerprint")
+	sub.From("t")
+	sub.Where(sub.E("k", "v"))
+
+	sql, args := buildSamplesQuery(5, 9, []string{"a_total", "b_total"}, sub)
+	assert.Contains(t, sql, "metric_name IN (?, ?)")
+	assert.Contains(t, sql, "fingerprint GLOBAL IN (SELECT fingerprint FROM t WHERE k = ?)")
+	assert.Contains(t, sql, "unix_milli >= ? AND unix_milli <= ?")
+	assert.Equal(t, []any{"a_total", "b_total", "v", int64(5), int64(9)}, args)
+
+	sub2 := sqlbuilder.NewSelectBuilder()
+	sub2.Select("fingerprint")
+	sub2.From("t")
+	sql, args = buildSamplesQuery(5, 9, nil, sub2)
+	assert.NotContains(t, sql, "metric_name IN")
+	assert.Equal(t, []any{int64(5), int64(9)}, args)
+}
+
+// Hash grouping must stay order-insensitive (stored JSON key order is not
+// canonical across fingerprints), and a 64-bit hash collision between
+// distinct labelsets must not merge them — splitByLabelSet is that guard.
+func TestLabelsHashAndCollisionSplit(t *testing.T) {
+	lbls := []prompb.Label{
+		{Name: "__name__", Value: "requests"},
+		{Name: "job", Value: "api"},
+		{Name: "instance", Value: "0"},
+	}
+	reversed := []prompb.Label{lbls[2], lbls[1], lbls[0]}
+	assert.Equal(t, labelsHash(lbls), labelsHash(reversed))
+
+	a := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "x"}}}
+	b := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "y"}}}
+	c := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "x"}}}
+	got := splitByLabelSet([]*prompb.TimeSeries{a, b, c})
+	require.Len(t, got, 2)
+	assert.Equal(t, []*prompb.TimeSeries{a, c}, got[0])
+	assert.Equal(t, []*prompb.TimeSeries{b}, got[1])
+}

pkg/prometheus/clickhouseprometheus/json.go

@@ -0,0 +1,34 @@
+package clickhouseprometheus
+
+import (
+	"encoding/json"
+
+	"github.com/prometheus/prometheus/prompb"
+)
+
+// Unmarshals JSON into Prometheus labels. It does not preserve order.
+// Empty-valued labels are dropped: Prometheus treats them as absent, and
+// keeping them lets two fingerprints present duplicate labelsets to the
+// engine (the incident behind #8563).
+func unmarshalLabels(s string) ([]prompb.Label, string, error) {
+	var metricName string
+	m := make(map[string]string)
+	if err := json.Unmarshal([]byte(s), &m); err != nil {
+		return nil, metricName, err
+	}
+	res := make([]prompb.Label, 0, len(m))
+	for n, v := range m {
+		if v == "" {
+			continue
+		}
+		if n == "__name__" {
+			metricName = v
+		}
+
+		res = append(res, prompb.Label{
+			Name:  n,
+			Value: v,
+		})
+	}
+	return res, metricName, nil
+}

pkg/prometheus/clickhouseprometheus/merge_test.go

@@ -0,0 +1,82 @@
+package clickhouseprometheus
+
+import (
+	"testing"
+
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/stretchr/testify/assert"
+)
+
+func mkSeries(value string, samples ...prompb.Sample) *prompb.TimeSeries {
+	return &prompb.TimeSeries{
+		Labels:  []prompb.Label{{Name: "job", Value: value}},
+		Samples: samples,
+	}
+}
+
+func TestMergeSeriesWithIdenticalLabels(t *testing.T) {
+	s := func(ts int64, v float64) prompb.Sample { return prompb.Sample{Timestamp: ts, Value: v} }
+
+	t.Run("empty and single series pass through untouched", func(t *testing.T) {
+		assert.Nil(t, mergeSeriesWithIdenticalLabels(nil))
+		one := []*prompb.TimeSeries{mkSeries("a", s(1, 1))}
+		got := mergeSeriesWithIdenticalLabels(one)
+		assert.Equal(t, one, got)
+	})
+
+	t.Run("no collisions returns the input slice as is", func(t *testing.T) {
+		in := []*prompb.TimeSeries{mkSeries("a", s(1, 1)), mkSeries("b", s(1, 2))}
+		got := mergeSeriesWithIdenticalLabels(in)
+		// same backing slice: the fast path must not rebuild anything
+		assert.Equal(t, &in[0], &got[0])
+	})
+
+	t.Run("disjoint streams concatenate in timestamp order", func(t *testing.T) {
+		// the #8563 shape: the series transitioned fingerprints at a point
+		// in time, so the streams do not overlap at all
+		got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{
+			mkSeries("a", s(1, 1), s(2, 2)),
+			mkSeries("a", s(3, 3), s(4, 4)),
+		})
+		assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 2), s(3, 3), s(4, 4)}, got[0].Samples)
+	})
+
+	t.Run("three fingerprints one labelset", func(t *testing.T) {
+		got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{
+			mkSeries("a", s(1, 1), s(4, 4)),
+			mkSeries("a", s(2, 2)),
+			mkSeries("a", s(3, 3)),
+		})
+		assert.Len(t, got, 1)
+		assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 2), s(3, 3), s(4, 4)}, got[0].Samples)
+	})
+
+	t.Run("equal timestamps everywhere keep the last stream's value", func(t *testing.T) {
+		// input is in ascending fingerprint order; the highest wins each tie
+		got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{
+			mkSeries("a", s(1, 1), s(2, 1)),
+			mkSeries("a", s(1, 9), s(2, 9)),
+		})
+		assert.Equal(t, []prompb.Sample{s(1, 9), s(2, 9)}, got[0].Samples)
+	})
+
+	t.Run("zero-sample series in a group is harmless", func(t *testing.T) {
+		got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{
+			mkSeries("a"),
+			mkSeries("a", s(1, 1)),
+		})
+		assert.Len(t, got, 1)
+		assert.Equal(t, []prompb.Sample{s(1, 1)}, got[0].Samples)
+	})
+
+	t.Run("colliding and distinct series interleave without cross-talk", func(t *testing.T) {
+		got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{
+			mkSeries("a", s(1, 1)),
+			mkSeries("b", s(1, 2)),
+			mkSeries("a", s(2, 3)),
+		})
+		assert.Len(t, got, 2)
+		assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 3)}, got[0].Samples)
+		assert.Equal(t, []prompb.Sample{s(1, 2)}, got[1].Samples)
+	})
+}

pkg/prometheus/clickhouseprometheus/provider.go

@@ -0,0 +1,78 @@
+package clickhouseprometheus
+
+import (
+	"context"
+
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/remote"
+)
+
+var stCallback = func() (int64, error) {
+	return int64(model.Latest), nil
+}
+
+type provider struct {
+	settings       factory.ScopedProviderSettings
+	telemetryStore telemetrystore.TelemetryStore
+	engine         *prometheus.Engine
+	parser         prometheus.Parser
+	queryable      storage.SampleAndChunkQueryable
+}
+
+func NewFactory(telemetryStore telemetrystore.TelemetryStore) factory.ProviderFactory[prometheus.Prometheus, prometheus.Config] {
+	return factory.NewProviderFactory(factory.MustNewName("clickhouse"), func(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config) (prometheus.Prometheus, error) {
+		return New(ctx, providerSettings, config, telemetryStore)
+	})
+}
+
+func New(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config, telemetryStore telemetrystore.TelemetryStore) (prometheus.Prometheus, error) {
+	settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheus")
+
+	readClient := NewReadClient(settings, telemetryStore)
+
+	return &provider{
+		settings:       settings,
+		telemetryStore: telemetryStore,
+		engine:         prometheus.NewEngine(settings.Logger(), config),
+		parser:         prometheus.NewParser(),
+		queryable:      remote.NewSampleAndChunkQueryableClient(readClient, labels.EmptyLabels(), []*labels.Matcher{}, false, stCallback),
+	}, nil
+}
+
+func (provider *provider) Engine() *prometheus.Engine {
+	return provider.engine
+}
+
+func (provider *provider) Parser() prometheus.Parser {
+	return provider.parser
+}
+
+func (provider *provider) Storage() storage.Queryable {
+	return provider
+}
+
+func (provider *provider) Querier(mint, maxt int64) (storage.Querier, error) {
+	querier, err := provider.queryable.Querier(mint, maxt)
+	if err != nil {
+		return nil, err
+	}
+
+	return storage.NewMergeQuerier(nil, []storage.Querier{querier}, storage.ChainedSeriesMerge), nil
+}
+
+// CapturingStorage implements prometheus.StatementCapturer. Uses a fresh
+// recorder per call so concurrent dry-runs don't share state.
+func (provider *provider) CapturingStorage() (storage.Queryable, prometheus.StatementRecorder) {
+	recorder := &statementRecorder{}
+	capture := &captureClient{
+		client:   &client{settings: provider.settings, telemetryStore: provider.telemetryStore},
+		recorder: recorder,
+	}
+	queryable := remote.NewSampleAndChunkQueryableClient(capture, labels.EmptyLabels(), []*labels.Matcher{}, false, stCallback)
+	return captureQueryable{inner: queryable}, recorder
+}

pkg/prometheus/clickhouseprometheus/scanner.go

@@ -0,0 +1,31 @@
+package clickhouseprometheus
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+var _ sql.Scanner = (*scanner)(nil)
+
+type scanner struct {
+	f float64
+	s string
+}
+
+func (s *scanner) Scan(val any) error {
+	s.f = 0
+	s.s = ""
+
+	s.s = fmt.Sprintf("%v", val)
+	switch val := val.(type) {
+	case int64:
+		s.f = float64(val)
+	case uint64:
+		s.f = float64(val)
+	case float64:
+		s.f = val
+	case []byte:
+		s.s = string(val)
+	}
+	return nil
+}

pkg/prometheus/clickhouseprometheus/table.go

@@ -0,0 +1,41 @@
+package clickhouseprometheus
+
+import "time"
+
+const (
+	databaseName                string = "signoz_metrics"
+	distributedTimeSeriesV4     string = "distributed_time_series_v4"
+	distributedTimeSeriesV46hrs string = "distributed_time_series_v4_6hrs"
+	distributedTimeSeriesV41day string = "distributed_time_series_v4_1day"
+	distributedSamplesV4        string = "distributed_samples_v4"
+)
+
+var (
+	sixHoursInMilliseconds = time.Hour.Milliseconds() * 6
+	oneDayInMilliseconds   = time.Hour.Milliseconds() * 24
+)
+
+// Returns the start time, end time and the table name to use for the query.
+//
+//	If time range is less than 6 hours, we need to use the `time_series_v4` table
+//	else if time range is less than 1 day and greater than 6 hours, we need to use the `time_series_v4_6hrs` table
+//	else we need to use the `time_series_v4_1day` table
+func getStartAndEndAndTableName(start, end int64) (int64, int64, string) {
+	var tableName string
+
+	if end-start <= sixHoursInMilliseconds {
+		// adjust the start time to nearest 1 hour
+		start = start - (start % (time.Hour.Milliseconds() * 1))
+		tableName = distributedTimeSeriesV4
+	} else if end-start <= oneDayInMilliseconds {
+		// adjust the start time to nearest 6 hours
+		start = start - (start % (time.Hour.Milliseconds() * 6))
+		tableName = distributedTimeSeriesV46hrs
+	} else {
+		// adjust the start time to nearest 1 day
+		start = start - (start % (time.Hour.Milliseconds() * 24))
+		tableName = distributedTimeSeriesV41day
+	}
+
+	return start, end, tableName
+}

pkg/prometheus/clickhouseprometheusv2/provider.go

@@ -19,7 +19,11 @@ type provider struct {
 	executor *executor
 }
 
-var _ prometheus.Prometheus = (*provider)(nil)
+var (
+	_ prometheus.Prometheus        = (*provider)(nil)
+	_ prometheus.StatementCapturer = (*provider)(nil)
+	_ prometheus.RangeExecutor     = (*provider)(nil)
+)
 
 func NewFactory(telemetryStore telemetrystore.TelemetryStore) factory.ProviderFactory[prometheus.Prometheus, prometheus.Config] {
 	return factory.NewProviderFactory(factory.MustNewName("clickhousev2"), func(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config) (prometheus.Prometheus, error) {
@@ -43,69 +47,30 @@ func New(_ context.Context, providerSettings factory.ProviderSettings, config pr
 	}, nil
 }
 
-func (p *provider) QueryRange(ctx context.Context, query string, start, end time.Time, step time.Duration) (*prometheus.Result, error) {
-	matrix, served, err := p.executor.TryExecuteRange(ctx, query, start, end, step)
-	if err != nil {
-		return nil, err
-	}
-	if served {
-		return &prometheus.Result{Value: matrix}, nil
-	}
-
-	qry, err := p.engine.NewRangeQuery(p.traitsContext(ctx, query), p, nil, query, start, end, step)
-	if err != nil {
-		return nil, err
-	}
-	return finishQuery(ctx, qry)
+func (p *provider) TryExecuteRange(ctx context.Context, query string, start, end time.Time, step time.Duration) (promql.Matrix, bool, error) {
+	return p.executor.TryExecuteRange(ctx, query, start, end, step)
 }
 
-func (p *provider) Query(ctx context.Context, query string, ts time.Time) (*prometheus.Result, error) {
-	qry, err := p.engine.NewInstantQuery(p.traitsContext(ctx, query), p, nil, query, ts)
-	if err != nil {
-		return nil, err
-	}
-	return finishQuery(ctx, qry)
+func (p *provider) Engine() *prometheus.Engine {
+	return p.engine
 }
 
-// A fresh recorder per call keeps concurrent dry-runs isolated. Exec drives
-// a Select per selector (recording SQL) but reads no data.
-func (p *provider) Statements(ctx context.Context, query string, start, end time.Time, step time.Duration) ([]prometheus.CapturedStatement, error) {
-	recorder := &statementRecorder{}
-	capture := &captureQueryable{client: p.client, recorder: recorder}
-	qry, err := p.engine.NewRangeQuery(p.traitsContext(ctx, query), capture, nil, query, start, end, step)
-	if err != nil {
-		return nil, err
-	}
-	defer qry.Close()
-	if res := qry.Exec(ctx); res.Err != nil {
-		return nil, res.Err
-	}
-	return recorder.Statements(), nil
+func (p *provider) Parser() prometheus.Parser {
+	return p.parser
 }
 
-// traitsContext attaches the query's traits so the storage can prove
-// step-aligned optimizations safe (see prometheus.QueryTraits). A parse
-// failure surfaces from the engine with its own error.
-func (p *provider) traitsContext(ctx context.Context, query string) context.Context {
-	expr, err := p.parser.ParseExpr(query)
-	if err != nil {
-		return ctx
-	}
-	return prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
-}
-
-// finishQuery packages an engine evaluation. The query is closed only on
-// error: Close returns the result's sample slices to the engine's pool, and
-// the returned Value must stay valid for the caller.
-func finishQuery(ctx context.Context, qry promql.Query) (*prometheus.Result, error) {
-	res := qry.Exec(ctx)
-	if res.Err != nil {
-		qry.Close()
-		return nil, res.Err
-	}
-	return &prometheus.Result{Value: res.Value, Warnings: res.Warnings, Stats: qry.Stats()}, nil
+func (p *provider) Storage() storage.Queryable {
+	return p
 }
 
 func (p *provider) Querier(mint, maxt int64) (storage.Querier, error) {
 	return &querier{mint: mint, maxt: maxt, client: p.client}, nil
 }
+
+// CapturingStorage implements prometheus.StatementCapturer: a storage that
+// records each selector's SQL without executing it, for the preview path.
+// A fresh recorder per call keeps concurrent dry-runs isolated.
+func (p *provider) CapturingStorage() (storage.Queryable, prometheus.StatementRecorder) {
+	recorder := &statementRecorder{}
+	return &captureQueryable{client: p.client, recorder: recorder}, recorder
+}

pkg/prometheus/config.go

@@ -24,6 +24,10 @@ type Config struct {
 
 	// Timeout is the maximum time a query is allowed to run before being aborted.
 	Timeout time.Duration `mapstructure:"timeout"`
+
+	// ProviderName selects the storage provider: "clickhouse" (default) or
+	// "clickhousev2".
+	ProviderName string `mapstructure:"provider"`
 }
 
 func NewConfigFactory() factory.ConfigFactory {
@@ -37,7 +41,8 @@ func newConfig() factory.Config {
 			Path:          "",
 			MaxConcurrent: 20,
 		},
-		Timeout: 2 * time.Minute,
+		Timeout:      2 * time.Minute,
+		ProviderName: "clickhouse",
 	}
 }
 
@@ -45,9 +50,15 @@ func (c Config) Validate() error {
 	if c.Timeout <= 0 {
 		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::timeout must be greater than 0")
 	}
+	if c.ProviderName != "" && c.ProviderName != "clickhouse" && c.ProviderName != "clickhousev2" {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::provider must be one of [clickhouse, clickhousev2], got %q", c.ProviderName)
+	}
 	return nil
 }
 
 func (c Config) Provider() string {
-	return "clickhousev2"
+	if c.ProviderName == "" {
+		return "clickhouse"
+	}
+	return c.ProviderName
 }

pkg/prometheus/handler.go

@@ -10,7 +10,6 @@ import (
 
 	promModel "github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/promql"
-	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/util/stats"
 
 	"github.com/SigNoz/signoz/pkg/errors"
@@ -82,8 +81,36 @@ func (h *handler) QueryRange(w http.ResponseWriter, r *http.Request) {
 	}
 	defer cancel()
 
-	res, err := h.prom.QueryRange(ctx, r.FormValue("query"), start, end, step)
-	h.respondResult(ctx, w, r, res, err)
+	if h.tryRangeExecutor(ctx, w, r, start, end, step) {
+		return
+	}
+
+	qry, err := h.prom.Engine().NewRangeQuery(ctx, h.prom.Storage(), nil, r.FormValue("query"), start, end, step)
+	if err != nil {
+		h.respondError(r.Context(), w, errBadData, err)
+		return
+	}
+	h.exec(ctx, w, r, qry)
+}
+
+// tryRangeExecutor serves the query the way a RangeExecutor provider is
+// designed to serve: evaluated inside the datastore when the shape allows.
+// It reports whether the response was written.
+func (h *handler) tryRangeExecutor(ctx context.Context, w http.ResponseWriter, r *http.Request, start, end time.Time, step time.Duration) bool {
+	re, ok := h.prom.(RangeExecutor)
+	if !ok {
+		return false
+	}
+	matrix, served, err := re.TryExecuteRange(ctx, r.FormValue("query"), start, end, step)
+	if err != nil {
+		h.respondError(ctx, w, errExec, err)
+		return true
+	}
+	if !served {
+		return false
+	}
+	h.respond(ctx, w, &queryData{ResultType: matrix.Type(), Result: matrix}, nil, nil)
+	return true
 }
 
 // Query evaluates an expression at a single instant: query and optional
@@ -107,34 +134,35 @@ func (h *handler) Query(w http.ResponseWriter, r *http.Request) {
 	}
 	defer cancel()
 
-	res, err := h.prom.Query(ctx, r.FormValue("query"), ts)
-	h.respondResult(ctx, w, r, res, err)
+	qry, err := h.prom.Engine().NewInstantQuery(ctx, h.prom.Storage(), nil, r.FormValue("query"), ts)
+	if err != nil {
+		h.respondError(r.Context(), w, errBadData, err)
+		return
+	}
+	h.exec(ctx, w, r, qry)
 }
 
-func (h *handler) respondResult(ctx context.Context, w http.ResponseWriter, r *http.Request, res *Result, err error) {
-	if err != nil {
-		h.logger.ErrorContext(ctx, "error evaluating promql query", errors.Attr(err))
-		var parseErrs parser.ParseErrors
-		if errors.As(err, &parseErrs) {
-			h.respondError(ctx, w, errBadData, err)
-			return
-		}
-		switch err.(type) {
+func (h *handler) exec(ctx context.Context, w http.ResponseWriter, r *http.Request, qry promql.Query) {
+	defer qry.Close()
+	res := qry.Exec(ctx)
+	if res.Err != nil {
+		h.logger.ErrorContext(ctx, "error evaluating promql query", errors.Attr(res.Err))
+		switch res.Err.(type) {
 		case promql.ErrQueryCanceled:
-			h.respondError(ctx, w, errCanceled, err)
+			h.respondError(ctx, w, errCanceled, res.Err)
 		case promql.ErrQueryTimeout:
-			h.respondError(ctx, w, errTimeout, err)
+			h.respondError(ctx, w, errTimeout, res.Err)
 		case promql.ErrStorage:
-			h.respondError(ctx, w, errInternal, err)
+			h.respondError(ctx, w, errInternal, res.Err)
 		default:
-			h.respondError(ctx, w, errExec, err)
+			h.respondError(ctx, w, errExec, res.Err)
 		}
 		return
 	}
 
 	data := &queryData{ResultType: res.Value.Type(), Result: res.Value}
-	if r.FormValue("stats") != "" && res.Stats != nil {
-		data.Stats = stats.NewQueryStats(res.Stats)
+	if r.FormValue("stats") != "" {
+		data.Stats = stats.NewQueryStats(qry.Stats())
 	}
 	warnings, infos := res.Warnings.AsStrings(r.FormValue("query"), 10, 10)
 	h.respond(ctx, w, data, warnings, infos)

pkg/prometheus/prometheus.go

@@ -6,8 +6,7 @@ import (
 
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
-	"github.com/prometheus/prometheus/util/annotations"
-	"github.com/prometheus/prometheus/util/stats"
+	"github.com/prometheus/prometheus/storage"
 )
 
 type Engine = promql.Engine
@@ -15,25 +14,9 @@ type Engine = promql.Engine
 type Parser = parser.Parser
 
 type Prometheus interface {
-	// QueryRange evaluates a range query: inside the datastore when the
-	// query's shape allows it, else in the engine over the provider's
-	// storage, which is always exact.
-	QueryRange(ctx context.Context, query string, start, end time.Time, step time.Duration) (*Result, error)
-
-	// Query evaluates an instant query in the engine.
-	Query(ctx context.Context, query string, ts time.Time) (*Result, error)
-
-	// Statements returns the datastore statements the engine path of a
-	// range query would run, captured without executing them.
-	Statements(ctx context.Context, query string, start, end time.Time, step time.Duration) ([]CapturedStatement, error)
-}
-
-// Result is one evaluation's outcome. The caller owns Value: the provider
-// never returns its sample slices to the engine's pools.
-type Result struct {
-	Value    parser.Value
-	Warnings annotations.Annotations
-	Stats    *stats.Statistics
+	Engine() *Engine
+	Storage() storage.Queryable
+	Parser() Parser
 }
 
 // CapturedStatement is one datastore statement a PromQL query would run,
@@ -42,3 +25,33 @@ type CapturedStatement struct {
 	Query string
 	Args  []any
 }
+
+// StatementRecorder reads back the statements captured against a capturing
+// Storage (see StatementCapturer).
+type StatementRecorder interface {
+	Statements() []CapturedStatement
+}
+
+// StatementCapturer is an optional Prometheus-provider capability, discovered
+// via type assertion: it returns a Storage that records each Select's statement
+// without executing it, plus a recorder to read them back.
+type StatementCapturer interface {
+	CapturingStorage() (storage.Queryable, StatementRecorder)
+}
+
+// ProviderClickhouseV2 is the clickhousev2 provider name: the factory
+// registration, the prometheus::provider config value and the
+// X-SigNoz-PromQL-Provider request header all use it, so they cannot drift
+// apart.
+const ProviderClickhouseV2 = "clickhousev2"
+
+// RangeExecutor is the optional capability of a provider that can evaluate
+// some range queries entirely inside the datastore. ok=false means the
+// query is not evaluable that way. The caller then runs the engine over the
+// provider's Storage, which is always exact. Only the clickhousev2 provider
+// implements this capability. When that provider is the only one, the
+// capability folds into Prometheus itself, and the engine-vs-datastore
+// decision becomes internal.
+type RangeExecutor interface {
+	TryExecuteRange(ctx context.Context, query string, start, end time.Time, step time.Duration) (promql.Matrix, bool, error)
+}

pkg/prometheus/prometheustest/grid.go

@@ -1,31 +0,0 @@
-package prometheustest
-
-import (
-	cmock "github.com/SigNoz/clickhouse-go-mock"
-)
-
-// GridCols is the result shape of a transpiled unit statement.
-var GridCols = []cmock.ColumnType{
-	{Name: "gkey", Type: "String"},
-	{Name: "grid", Type: "Array(Nullable(Float64))"},
-}
-
-// LastSampleGrid builds the grid a transpiled instant selector returns: per
-// slot t, the latest sample in the left-open lookback window (t-lookback, t].
-func LastSampleGrid(tsMs []int64, values []float64, startMs, endMs, stepMs, lookbackMs int64) []*float64 {
-	grid := make([]*float64, (endMs-startMs)/stepMs+1)
-	for i := range grid {
-		slot := startMs + int64(i)*stepMs
-		best := -1
-		for j, ts := range tsMs {
-			if ts > slot-lookbackMs && ts <= slot && (best == -1 || ts >= tsMs[best]) {
-				best = j
-			}
-		}
-		if best >= 0 {
-			v := values[best]
-			grid[i] = &v
-		}
-	}
-	return grid
-}

pkg/prometheus/prometheustest/provider.go

@@ -5,16 +5,58 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/prometheus"
-	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheus"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/storage/remote"
 )
 
-// New returns the clickhousev2 provider over the given telemetry store, so
-// tests exercise the production read path against a mock store.
-func New(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config, telemetryStore telemetrystore.TelemetryStore) prometheus.Prometheus {
-	provider, err := clickhouseprometheusv2.New(ctx, providerSettings, config, telemetryStore)
-	if err != nil {
-		panic(err)
-	}
-	return provider
+var _ prometheus.Prometheus = (*Provider)(nil)
+
+type Provider struct {
+	queryable storage.SampleAndChunkQueryable
+	engine    *prometheus.Engine
+	parser    prometheus.Parser
+}
+
+var stCallback = func() (int64, error) {
+	return int64(model.Latest), nil
+}
+
+func New(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config, telemetryStore telemetrystore.TelemetryStore) *Provider {
+
+	settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/prometheus/prometheustest")
+
+	engine := prometheus.NewEngine(settings.Logger(), config)
+
+	readClient := clickhouseprometheus.NewReadClient(settings, telemetryStore)
+
+	queryable := remote.NewSampleAndChunkQueryableClient(readClient, labels.EmptyLabels(), []*labels.Matcher{}, false, stCallback)
+
+	return &Provider{
+		engine:    engine,
+		parser:    prometheus.NewParser(),
+		queryable: queryable,
+	}
+}
+
+func (provider *Provider) Engine() *prometheus.Engine {
+	return provider.engine
+}
+
+func (provider *Provider) Storage() storage.Queryable {
+	return provider.queryable
+}
+
+func (provider *Provider) Parser() prometheus.Parser {
+	return provider.parser
+}
+
+func (provider *Provider) Close() error {
+	if provider.engine != nil {
+		provider.engine.Close()
+	}
+	return nil
 }

pkg/querier/api.go

@@ -57,6 +57,7 @@ func (handler *handler) QueryRange(rw http.ResponseWriter, req *http.Request) {
 		render.Error(rw, err)
 		return
 	}
+	queryRangeRequest.PromQLProvider = req.Header.Get("X-SigNoz-PromQL-Provider")
 
 	// Validate the query request
 	if err := queryRangeRequest.Validate(); err != nil {

pkg/querier/bucket_cache.go

@@ -451,7 +451,7 @@ func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.Cach
 	// Merge values based on type
 	var mergedValue any
 	switch resultType {
-	case qbtypes.RequestTypeTimeSeries:
+	case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 		mergedValue = bc.mergeTimeSeriesValues(ctx, buckets)
 		// Raw and Scalar types are not cached, so no merge needed
 	}
@@ -476,14 +476,36 @@ func (bc *bucketCache) mergeTimeSeriesValues(ctx context.Context, buckets []*qbt
 	}
 	seriesMap := make(map[seriesKey]*qbtypes.TimeSeries, estimatedSeries)
 
+	decodedTimeSeriesData := make([]*qbtypes.TimeSeriesData, 0, len(buckets))
+
+	// Alias and Meta are taken from whichever cached bucket covers the latest
+	// range, and the buckets do not arrive in StartMs order, so keep the winner
+	// per AggregationBucket.Index alongside the StartMs that won it.
+	aggregationIndexToLatest := map[int]*qbtypes.AggregationBucket{}
+	aggregationIndexToLatestStartMs := map[int]uint64{}
+
 	for _, bucket := range buckets {
 		var tsData *qbtypes.TimeSeriesData
 		if err := json.Unmarshal(bucket.Value, &tsData); err != nil {
 			bc.logger.ErrorContext(ctx, "failed to unmarshal time series data", errors.Attr(err))
 			continue
 		}
+		decodedTimeSeriesData = append(decodedTimeSeriesData, tsData)
 
 		for _, aggBucket := range tsData.Aggregations {
+			if _, seen := aggregationIndexToLatest[aggBucket.Index]; !seen || bucket.StartMs >= aggregationIndexToLatestStartMs[aggBucket.Index] {
+				aggregationIndexToLatest[aggBucket.Index] = aggBucket
+				aggregationIndexToLatestStartMs[aggBucket.Index] = bucket.StartMs
+			}
+		}
+	}
+
+	mergedUpperBounds := qbtypes.MergeBucketUpperBounds(decodedTimeSeriesData...)
+
+	for _, tsData := range decodedTimeSeriesData {
+		for _, aggBucket := range tsData.Aggregations {
+			aggBucket.ReindexValuesToNewUpperBounds(mergedUpperBounds[aggBucket.Index])
+
 			for _, series := range aggBucket.Series {
 				// Create series key from labels
 				key := seriesKey{
@@ -556,10 +578,15 @@ func (bc *bucketCache) mergeTimeSeriesValues(ctx context.Context, buckets []*qbt
 			}
 		}
 
-		result.Aggregations = append(result.Aggregations, &qbtypes.AggregationBucket{
+		aggBucket := &qbtypes.AggregationBucket{
 			Index:  index,
 			Series: seriesList,
-		})
+		}
+		if latest, ok := aggregationIndexToLatest[index]; ok {
+			aggBucket.Alias = latest.Alias
+			aggBucket.Meta = latest.Meta
+		}
+		result.Aggregations = append(result.Aggregations, aggBucket)
 	}
 
 	return result
@@ -572,7 +599,7 @@ func (bc *bucketCache) isEmptyResult(result *qbtypes.Result) (isEmpty bool, isFi
 	}
 
 	switch result.Type {
-	case qbtypes.RequestTypeTimeSeries:
+	case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 		if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
 			// No aggregations at all means truly empty
 			if len(tsData.Aggregations) == 0 {
@@ -699,14 +726,19 @@ func (bc *bucketCache) trimResultToFluxBoundary(result *qbtypes.Result, fluxBoun
 	}
 
 	switch result.Type {
-	case qbtypes.RequestTypeTimeSeries:
+	case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 		// Trim time series data
 		if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok && tsData != nil {
 			trimmedData := &qbtypes.TimeSeriesData{}
 
 			for _, aggBucket := range tsData.Aggregations {
+				// Meta has to survive the trim: a heatmap's counts are
+				// positional against Meta.Buckets, so a cached bucket that
+				// lost its axis cannot be read back against anything.
 				trimmedBucket := &qbtypes.AggregationBucket{
 					Index: aggBucket.Index,
+					Alias: aggBucket.Alias,
+					Meta:  aggBucket.Meta,
 				}
 
 				for _, series := range aggBucket.Series {
@@ -766,7 +798,7 @@ func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, startMs,
 	}
 
 	switch result.Type {
-	case qbtypes.RequestTypeTimeSeries:
+	case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 		if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
 			filteredData := &qbtypes.TimeSeriesData{
 				Aggregations: make([]*qbtypes.AggregationBucket, 0, len(tsData.Aggregations)),

pkg/querier/builder_query.go

@@ -92,6 +92,10 @@ func (q *builderQuery[T]) Fingerprint() string {
 	// This needs to include all fields that affect the query results
 	parts := []string{q.queryType.StringValue()}
 
+	// A heatmap and a time series query can share every spec field and still
+	// return different rows, so the request type has to separate their entries
+	parts = append(parts, fmt.Sprintf("requestType=%s", q.kind.StringValue()))
+
 	// Add signal type
 	parts = append(parts, fmt.Sprintf("signal=%s", q.spec.Signal.StringValue()))
 
@@ -130,6 +134,9 @@ func (q *builderQuery[T]) Fingerprint() string {
 					}
 					part += ":" + route
 				}
+				if a.HeatmapBucketing != nil {
+					part += ":" + fingerprintHeatmapBucketing(*a.HeatmapBucketing)
+				}
 				aggParts = append(aggParts, part)
 			}
 		}
@@ -185,6 +192,16 @@ func (q *builderQuery[T]) Fingerprint() string {
 	return strings.Join(parts, "&")
 }
 
+// fingerprintHeatmapBucketing captures only what changes the rows ClickHouse
+// returns, which is why LogBucketsSpec.Scale is absent: coarsening it happens in
+// postprocessing, so every scale reads one cache entry.
+func fingerprintHeatmapBucketing(b qbtypes.HeatmapBucketing) string {
+	if b.Kind == qbtypes.BucketsKindLinear {
+		return fmt.Sprintf("%s:%v:%d", b.Kind.StringValue(), b.MaxValue, b.NumBuckets)
+	}
+	return b.Kind.StringValue()
+}
+
 func fingerprintGroupByKey(gb qbtypes.GroupByKey) string {
 	return fingerprintFieldKey(gb.TelemetryFieldKey)
 }
@@ -412,7 +429,7 @@ func (q *builderQuery[T]) narrowWindowByTraceID(ctx context.Context, fromMS, toM
 func emptyResultFor(kind qbtypes.RequestType, queryName string) *qbtypes.Result {
 	var value any
 	switch kind {
-	case qbtypes.RequestTypeTimeSeries:
+	case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 		value = &qbtypes.TimeSeriesData{QueryName: queryName}
 	case qbtypes.RequestTypeScalar:
 		value = &qbtypes.ScalarData{QueryName: queryName}
@@ -465,8 +482,9 @@ func (q *builderQuery[T]) executeWithContext(ctx context.Context, query string,
 	queryWindow := &qbtypes.TimeRange{From: q.fromMS, To: q.toMS}
 
 	kind := q.kind
-	// all metric queries are time series then reduced if required
-	if q.spec.Signal == telemetrytypes.SignalMetrics {
+	// all metric queries are time series then reduced if required, except
+	// heatmaps, whose statement returns a row per bucket rather than per point
+	if q.spec.Signal == telemetrytypes.SignalMetrics && kind != qbtypes.RequestTypeHeatmap {
 		kind = qbtypes.RequestTypeTimeSeries
 	}
 

pkg/querier/builder_query_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/SigNoz/signoz/pkg/querybuilder"
+	"github.com/SigNoz/signoz/pkg/types/metrictypes"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/stretchr/testify/assert"
@@ -120,6 +121,170 @@ func TestBuilderQueryFingerprintQueryType(t *testing.T) {
 	assert.Empty(t, ai.Fingerprint())
 }
 
+func TestBuilderQueryFingerprintHeatmapBucketing(t *testing.T) {
+	coarseLogScale := 1
+
+	testCases := []struct {
+		description   string
+		left          *builderQuery[qbtypes.MetricAggregation]
+		right         *builderQuery[qbtypes.MetricAggregation]
+		expectedEqual bool
+	}{
+		{
+			// fingerprintHeatmapBucketing leaves LogScale out, so the two are
+			// indistinguishable here by design
+			description: "a coarser logScale reads the same cache entry",
+			left: &builderQuery[qbtypes.MetricAggregation]{
+				queryType: qbtypes.QueryTypeBuilder,
+				kind:      qbtypes.RequestTypeHeatmap,
+				spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+					Signal: telemetrytypes.SignalMetrics,
+					Aggregations: []qbtypes.MetricAggregation{{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLog, LogScale: qbtypes.MaxLogScale, NumBuckets: qbtypes.DefaultNumBuckets},
+					}},
+				},
+			},
+			right: &builderQuery[qbtypes.MetricAggregation]{
+				queryType: qbtypes.QueryTypeBuilder,
+				kind:      qbtypes.RequestTypeHeatmap,
+				spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+					Signal: telemetrytypes.SignalMetrics,
+					Aggregations: []qbtypes.MetricAggregation{{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLog, LogScale: qbtypes.MaxLogScale, NumBuckets: qbtypes.DefaultNumBuckets},
+					}},
+				},
+			},
+			expectedEqual: true,
+		},
+		{
+			description: "linear separates on maxValue",
+			left: &builderQuery[qbtypes.MetricAggregation]{
+				queryType: qbtypes.QueryTypeBuilder,
+				kind:      qbtypes.RequestTypeHeatmap,
+				spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+					Signal: telemetrytypes.SignalMetrics,
+					Aggregations: []qbtypes.MetricAggregation{{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 500, NumBuckets: 25},
+					}},
+				},
+			},
+			right: &builderQuery[qbtypes.MetricAggregation]{
+				queryType: qbtypes.QueryTypeBuilder,
+				kind:      qbtypes.RequestTypeHeatmap,
+				spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+					Signal: telemetrytypes.SignalMetrics,
+					Aggregations: []qbtypes.MetricAggregation{{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 800, NumBuckets: 25},
+					}},
+				},
+			},
+			expectedEqual: false,
+		},
+		{
+			description: "linear separates on numBuckets",
+			left: &builderQuery[qbtypes.MetricAggregation]{
+				queryType: qbtypes.QueryTypeBuilder,
+				kind:      qbtypes.RequestTypeHeatmap,
+				spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+					Signal: telemetrytypes.SignalMetrics,
+					Aggregations: []qbtypes.MetricAggregation{{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 500, NumBuckets: 25},
+					}},
+				},
+			},
+			right: &builderQuery[qbtypes.MetricAggregation]{
+				queryType: qbtypes.QueryTypeBuilder,
+				kind:      qbtypes.RequestTypeHeatmap,
+				spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+					Signal: telemetrytypes.SignalMetrics,
+					Aggregations: []qbtypes.MetricAggregation{{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 500, NumBuckets: 40},
+					}},
+				},
+			},
+			expectedEqual: false,
+		},
+		{
+			description: "linear and log are separate entries",
+			left: &builderQuery[qbtypes.MetricAggregation]{
+				queryType: qbtypes.QueryTypeBuilder,
+				kind:      qbtypes.RequestTypeHeatmap,
+				spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+					Signal: telemetrytypes.SignalMetrics,
+					Aggregations: []qbtypes.MetricAggregation{{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLinear, MaxValue: 500, NumBuckets: 25},
+					}},
+				},
+			},
+			right: &builderQuery[qbtypes.MetricAggregation]{
+				queryType: qbtypes.QueryTypeBuilder,
+				kind:      qbtypes.RequestTypeHeatmap,
+				spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+					Signal: telemetrytypes.SignalMetrics,
+					Aggregations: []qbtypes.MetricAggregation{{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{Kind: qbtypes.BucketsKindLog, LogScale: qbtypes.MaxLogScale, NumBuckets: qbtypes.DefaultNumBuckets},
+					}},
+				},
+			},
+			expectedEqual: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.description, func(t *testing.T) {
+			if testCase.expectedEqual {
+				assert.Equal(t, testCase.left.Fingerprint(), testCase.right.Fingerprint())
+				return
+			}
+			assert.NotEqual(t, testCase.left.Fingerprint(), testCase.right.Fingerprint())
+		})
+	}
+
+	t.Run("a coarser scale is carried but reads the same cache entry", func(t *testing.T) {
+		finest := (&qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLog, Spec: qbtypes.LogBucketsSpec{}}).ToHeatmapBucketing()
+		coarse := (&qbtypes.BucketOptions{Kind: qbtypes.BucketsKindLog, Spec: qbtypes.LogBucketsSpec{Scale: &coarseLogScale}}).ToHeatmapBucketing()
+
+		assert.NotEqual(t, finest.LogScale, coarse.LogScale)
+		assert.Equal(t, fingerprintHeatmapBucketing(finest), fingerprintHeatmapBucketing(coarse))
+	})
+
+	t.Run("a histogram folds in no bucket options at all", func(t *testing.T) {
+		// resolveHeatmapBucketing leaves histograms nil, so bucketOptions sent
+		// alongside one must not fragment its cache
+		histogram := &builderQuery[qbtypes.MetricAggregation]{
+			queryType: qbtypes.QueryTypeBuilder,
+			kind:      qbtypes.RequestTypeHeatmap,
+			spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+				Signal: telemetrytypes.SignalMetrics,
+				Aggregations: []qbtypes.MetricAggregation{{
+					MetricName: "signoz_latency",
+					Type:       metrictypes.HistogramType,
+				}},
+			},
+		}
+
+		fingerprint := histogram.Fingerprint()
+		assert.NotContains(t, fingerprint, qbtypes.BucketsKindLog.StringValue())
+		assert.NotContains(t, fingerprint, qbtypes.BucketsKindLinear.StringValue())
+	})
+}
+
 func TestMakeBucketsOrder(t *testing.T) {
 	// Test that makeBuckets returns buckets in reverse chronological order by default
 	// Using milliseconds as input - need > 1 hour range to get multiple buckets

pkg/querier/consume.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/ClickHouse/clickhouse-go/v2/lib/chcol"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"github.com/SigNoz/signoz/pkg/errors"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/spantypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrystoretypes"
@@ -83,6 +84,8 @@ func consume(rows driver.Rows, kind qbtypes.RequestType, queryWindow *qbtypes.Ti
 		payload, err = readAsTimeSeries(rows, queryWindow, step, queryName)
 	case qbtypes.RequestTypeScalar:
 		payload, err = readAsScalar(rows, queryName)
+	case qbtypes.RequestTypeHeatmap:
+		payload, err = readAsHeatmap(rows, queryWindow, step, queryName)
 	case qbtypes.RequestTypeRaw, qbtypes.RequestTypeTrace, qbtypes.RequestTypeRawStream:
 		payload, err = readAsRaw(rows, queryName)
 		// TODO: add support for other request types
@@ -112,35 +115,6 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt
 
 	stepMs := uint64(step.Milliseconds())
 
-	// Helper function to check if a timestamp represents a partial value
-	isPartialValue := func(timestamp int64) bool {
-		if stepMs == 0 || queryWindow == nil {
-			return false
-		}
-
-		timestampMs := uint64(timestamp)
-
-		// For the first interval, check if query start is misaligned
-		// The first complete interval starts at the first timestamp >= queryWindow.From that is aligned to step
-		firstCompleteInterval := queryWindow.From
-		if queryWindow.From%stepMs != 0 {
-			// Round up to next step boundary
-			firstCompleteInterval = ((queryWindow.From / stepMs) + 1) * stepMs
-		}
-
-		// If timestamp is before the first complete interval, it's partial
-		if timestampMs < firstCompleteInterval {
-			return true
-		}
-
-		// For the last interval, check if it would extend beyond query end
-		if timestampMs+stepMs > queryWindow.To {
-			return queryWindow.To%stepMs != 0
-		}
-
-		return false
-	}
-
 	// Pre-allocate for labels based on column count
 	lblValsCapacity := len(colNames) - 1 // -1 for timestamp
 	if lblValsCapacity < 0 {
@@ -271,7 +245,7 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt
 			series.Values = append(series.Values, &qbtypes.TimeSeriesValue{
 				Timestamp: ts,
 				Value:     val,
-				Partial:   isPartialValue(ts),
+				Partial:   isPartialValue(ts, queryWindow, stepMs),
 			})
 		}
 	}
@@ -315,6 +289,132 @@ func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbt
 	}, nil
 }
 
+func hasHeatmapBucketBounds(colNames []string) bool {
+	var hasMin, hasMax bool
+	for _, colName := range colNames {
+		switch stripKeyAlias(colName) {
+		case qbtypes.HeatmapBucketMinColumn:
+			hasMin = true
+		case qbtypes.HeatmapBucketMaxColumn:
+			hasMax = true
+		}
+	}
+	return hasMin && hasMax
+}
+
+// readAsHeatmap folds one row per cell — (timestamp, group labels, bucket bounds, count) — into one series per group.
+func readAsHeatmap(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbtypes.Step, queryName string) (*qbtypes.TimeSeriesData, error) {
+	colTypes := rows.ColumnTypes()
+	colNames := rows.Columns()
+
+	if !hasHeatmapBucketBounds(colNames) {
+		return nil, errors.NewInvalidInputf(errors.CodeInvalidInput,
+			"a heatmap needs a %q and a %q column to know the extent of each bucket",
+			qbtypes.HeatmapBucketMinColumn, qbtypes.HeatmapBucketMaxColumn)
+	}
+
+	slots := make([]any, len(colTypes))
+	for i, ct := range colTypes {
+		slots[i] = reflect.New(ct.ScanType()).Interface()
+	}
+
+	stepMs := uint64(step.Milliseconds())
+
+	accumulator := newHeatmapAccumulator()
+
+	for rows.Next() {
+		if err := rows.Scan(slots...); err != nil {
+			return nil, err
+		}
+
+		var (
+			ts      int64
+			bounds  bucketBounds
+			count   float64
+			lblVals []string
+			lblObjs []*qbtypes.Label
+		)
+
+		for idx, ptr := range slots {
+			name := stripKeyAlias(colNames[idx])
+			value := derefValue(ptr)
+
+			if t, ok := value.(time.Time); ok {
+				ts = t.UnixMilli()
+				continue
+			}
+
+			switch name {
+			case qbtypes.HeatmapBucketMinColumn:
+				bounds.Lower = numericAsFloat(value)
+			case qbtypes.HeatmapBucketMaxColumn:
+				bounds.Upper = numericAsFloat(value)
+			default:
+				if aggRe.MatchString(name) || slices.Contains(legacyReservedColumnTargetAliases, name) {
+					count = numericAsFloat(value)
+					continue
+				}
+				// a nullable label column comes back as a nil any, which would
+				// otherwise key the series on the literal "<nil>"
+				if value == nil {
+					value = ""
+				}
+				lblVals = append(lblVals, fmt.Sprint(value))
+				lblObjs = append(lblObjs, &qbtypes.Label{
+					Key:   telemetrytypes.TelemetryFieldKey{Name: name},
+					Value: value,
+				})
+			}
+		}
+
+		if ts == 0 || !isValidBucketBounds(bounds) || math.IsNaN(count) || math.IsInf(count, 0) {
+			continue
+		}
+		sort.Strings(lblVals)
+		labelsKey := strings.Join(lblVals, ",")
+
+		if err := accumulator.addCell(labelsKey, lblObjs, ts, bounds, count); err != nil {
+			return nil, err
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return accumulator.foldSeries(queryWindow, stepMs, queryName)
+}
+
+// isPartialValue reports whether the step interval starting at timestamp is only
+// partly covered by the query window, which happens when the window boundaries
+// are not step-aligned.
+func isPartialValue(timestamp int64, queryWindow *qbtypes.TimeRange, stepMs uint64) bool {
+	if stepMs == 0 || queryWindow == nil {
+		return false
+	}
+
+	timestampMs := uint64(timestamp)
+
+	// For the first interval, check if query start is misaligned
+	// The first complete interval starts at the first timestamp >= queryWindow.From that is aligned to step
+	firstCompleteInterval := queryWindow.From
+	if queryWindow.From%stepMs != 0 {
+		// Round up to next step boundary
+		firstCompleteInterval = ((queryWindow.From / stepMs) + 1) * stepMs
+	}
+
+	// If timestamp is before the first complete interval, it's partial
+	if timestampMs < firstCompleteInterval {
+		return true
+	}
+
+	// For the last interval, check if it would extend beyond query end
+	if timestampMs+stepMs > queryWindow.To {
+		return queryWindow.To%stepMs != 0
+	}
+
+	return false
+}
+
 func isNumericKind(t reflect.Type) bool {
 	if t == nil {
 		return false

pkg/querier/heatmap_accumulator.go

@@ -0,0 +1,163 @@
+package querier
+
+import (
+	"math"
+	"slices"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+)
+
+// bucketBounds holds the values in (Lower, Upper]. Only Upper reaches the
+// response; Lower is what says whether two rows bucketed the same way.
+type bucketBounds struct {
+	Lower float64
+	Upper float64
+}
+
+// heatmapColumn maps a bucket to the count in it, holding one timestamp's cells.
+// Keyed rather than indexed by bucket because the axis is only known once every
+// cell has been seen.
+type heatmapColumn map[bucketBounds]float64
+
+func isValidBucketBounds(bounds bucketBounds) bool {
+	return !math.IsNaN(bounds.Lower) && !math.IsNaN(bounds.Upper) &&
+		!math.IsInf(bounds.Upper, -1) && bounds.Lower < bounds.Upper
+}
+
+// heatmapSeries accumulates one group's columns while the rows are read.
+type heatmapSeries struct {
+	labels            []*qbtypes.Label
+	timestampToColumn map[int64]heatmapColumn
+}
+
+// heatmapAccumulator collects cells from either reader and folds them into one
+// series per group.
+type heatmapAccumulator struct {
+	keyToSeries  map[string]*heatmapSeries
+	seriesOrder  []string
+	upperToLower map[float64]float64
+}
+
+func newHeatmapAccumulator() *heatmapAccumulator {
+	return &heatmapAccumulator{
+		keyToSeries:  map[string]*heatmapSeries{},
+		upperToLower: map[float64]float64{},
+	}
+}
+
+// addCell files one cell under the group labelsKey identifies, keeping the
+// labels from the first cell seen for it. A response carrying upper bounds
+// alone cannot express two rows cutting the same bucket differently, so the
+// second of them is rejected here.
+func (a *heatmapAccumulator) addCell(labelsKey string, lbls []*qbtypes.Label, ts int64, bounds bucketBounds, count float64) error {
+	if seenLowerBound, isUpperBoundSeenBefore := a.upperToLower[bounds.Upper]; isUpperBoundSeenBefore && seenLowerBound != bounds.Lower {
+		return errors.NewInvalidInputf(errors.CodeInvalidInput,
+			"the bucket ending at %v is reported as starting at both %v and %v", bounds.Upper, seenLowerBound, bounds.Lower).
+			WithAdditional("Every row of a heatmap has to cut its buckets the same way")
+	}
+	a.upperToLower[bounds.Upper] = bounds.Lower
+
+	series, found := a.keyToSeries[labelsKey]
+	if !found {
+		series = &heatmapSeries{labels: lbls, timestampToColumn: map[int64]heatmapColumn{}}
+		a.keyToSeries[labelsKey] = series
+		a.seriesOrder = append(a.seriesOrder, labelsKey)
+	}
+	if series.timestampToColumn[ts] == nil {
+		series.timestampToColumn[ts] = heatmapColumn{}
+	}
+	series.timestampToColumn[ts][bounds] += count
+
+	return nil
+}
+
+// resolveBucketAxis lists the upper bounds the counts are indexed against. A
+// gap goes on the axis as its own empty bucket, or the bucket above it would
+// widen to cover a range nothing bucketed.
+func (a *heatmapAccumulator) resolveBucketAxis() ([]float64, error) {
+	upperBounds := make([]float64, 0, len(a.upperToLower))
+	for upperBound := range a.upperToLower {
+		if !math.IsInf(upperBound, 1) {
+			upperBounds = append(upperBounds, upperBound)
+		}
+	}
+	slices.Sort(upperBounds)
+
+	axis := make([]float64, 0, 2*len(upperBounds))
+	for index, upperBound := range upperBounds {
+		if index > 0 {
+			previousUpperBound := upperBounds[index-1]
+			switch lowerBoundForCurrUpperBound := a.upperToLower[upperBound]; {
+			case lowerBoundForCurrUpperBound < previousUpperBound:
+				return nil, errors.NewInvalidInputf(errors.CodeInvalidInput,
+					"the buckets ending at %v and %v overlap", previousUpperBound, upperBound).
+					WithAdditional("Every row of a heatmap has to cut its buckets the same way")
+			case lowerBoundForCurrUpperBound > previousUpperBound:
+				axis = append(axis, lowerBoundForCurrUpperBound)
+			}
+		}
+		axis = append(axis, upperBound)
+	}
+
+	return axis, nil
+}
+
+// foldSeries turns the collected cells into one series per group, in the order
+// the groups first appeared.
+func (a *heatmapAccumulator) foldSeries(queryWindow *qbtypes.TimeRange, stepMs uint64, queryName string) (*qbtypes.TimeSeriesData, error) {
+	if len(a.seriesOrder) == 0 {
+		return &qbtypes.TimeSeriesData{QueryName: queryName}, nil
+	}
+
+	upperBounds, err := a.resolveBucketAxis()
+	if err != nil {
+		return nil, err
+	}
+
+	upperBoundToIndex := make(map[float64]int, len(upperBounds)+1)
+	for index, upperBound := range upperBounds {
+		upperBoundToIndex[upperBound] = index
+	}
+	// the index past the last upper bound is where the +Inf overflow lands
+	upperBoundToIndex[math.Inf(1)] = len(upperBounds)
+
+	bucket := &qbtypes.AggregationBucket{
+		Index:  0,
+		Alias:  "__result_0",
+		Meta:   qbtypes.AggregationMeta{Buckets: upperBounds},
+		Series: make([]*qbtypes.TimeSeries, 0, len(a.seriesOrder)),
+	}
+
+	for _, labelsKey := range a.seriesOrder {
+		accumulated := a.keyToSeries[labelsKey]
+
+		timestamps := make([]int64, 0, len(accumulated.timestampToColumn))
+		for ts := range accumulated.timestampToColumn {
+			timestamps = append(timestamps, ts)
+		}
+		slices.Sort(timestamps)
+
+		series := &qbtypes.TimeSeries{
+			Labels: accumulated.labels,
+			Values: make([]*qbtypes.TimeSeriesValue, 0, len(timestamps)),
+		}
+		for _, ts := range timestamps {
+			values := make([]float64, len(upperBounds)+1)
+			for bounds, count := range accumulated.timestampToColumn[ts] {
+				values[upperBoundToIndex[bounds.Upper]] += count
+			}
+			series.Values = append(series.Values, &qbtypes.TimeSeriesValue{
+				Timestamp: ts,
+				Values:    values,
+				Partial:   isPartialValue(ts, queryWindow, stepMs),
+			})
+		}
+		bucket.Series = append(bucket.Series, series)
+	}
+
+	return &qbtypes.TimeSeriesData{
+		QueryName:    queryName,
+		Aggregations: []*qbtypes.AggregationBucket{bucket},
+	}, nil
+}

pkg/querier/heatmap_test.go

@@ -0,0 +1,101 @@
+package querier
+
+import (
+	"testing"
+
+	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMergeTimeSeriesResultsUnionsHeatmapAxes(t *testing.T) {
+	// a log axis holds whichever bands the data reached, so a wide cached range
+	// and a narrow fresh one routinely disagree on which bands exist
+	cached := &qbtypes.TimeSeriesData{
+		QueryName: "A",
+		Aggregations: []*qbtypes.AggregationBucket{{
+			Index: 0,
+			Meta:  qbtypes.AggregationMeta{Buckets: []float64{1, 4, 16}},
+			Series: []*qbtypes.TimeSeries{{
+				Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host.name"}, Value: "node-1"}},
+				Values: []*qbtypes.TimeSeriesValue{{Timestamp: 1710000000000, Values: []float64{1, 2, 3, 4}}},
+			}},
+		}},
+	}
+	fresh := []*qbtypes.Result{{
+		Value: &qbtypes.TimeSeriesData{
+			QueryName: "A",
+			Aggregations: []*qbtypes.AggregationBucket{{
+				Index: 0,
+				Meta:  qbtypes.AggregationMeta{Buckets: []float64{2, 4}},
+				Series: []*qbtypes.TimeSeries{{
+					Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host.name"}, Value: "node-1"}},
+					Values: []*qbtypes.TimeSeriesValue{{Timestamp: 1710000060000, Values: []float64{5, 6, 7}}},
+				}},
+			}},
+		},
+	}}
+
+	merged := (&querier{}).mergeTimeSeriesResults(cached, fresh)
+
+	require.Len(t, merged.Aggregations, 1)
+	aggBucket := merged.Aggregations[0]
+	assert.Equal(t, []float64{1, 2, 4, 16}, aggBucket.Meta.Buckets)
+
+	require.Len(t, aggBucket.Series, 1)
+	require.Len(t, aggBucket.Series[0].Values, 2)
+	// the cached 16 band survives even though the fresh range never reached it
+	assert.Equal(t, []float64{1, 0, 2, 3, 4}, aggBucket.Series[0].Values[0].Values)
+	// and the fresh 2 band survives even though the cached range never had it
+	assert.Equal(t, []float64{0, 5, 6, 0, 7}, aggBucket.Series[0].Values[1].Values)
+}
+
+func TestTrimResultToFluxBoundaryKeepsTheHeatmapAxis(t *testing.T) {
+	cache := &bucketCache{logger: instrumentationtest.New().Logger()}
+
+	result := &qbtypes.Result{
+		Type: qbtypes.RequestTypeHeatmap,
+		Value: &qbtypes.TimeSeriesData{
+			Aggregations: []*qbtypes.AggregationBucket{{
+				Index: 0,
+				Alias: "__result_0",
+				Meta:  qbtypes.AggregationMeta{Unit: "By", Buckets: []float64{1, 2, 4}},
+				Series: []*qbtypes.TimeSeries{{
+					Values: []*qbtypes.TimeSeriesValue{
+						{Timestamp: 1710000000000, Values: []float64{1, 2, 3, 4}},
+					},
+				}},
+			}},
+		},
+	}
+
+	trimmed := cache.trimResultToFluxBoundary(result, 1710000060000)
+
+	tsData, ok := trimmed.Value.(*qbtypes.TimeSeriesData)
+	require.True(t, ok)
+	require.Len(t, tsData.Aggregations, 1)
+
+	// the counts are positional against the axis, so a cached bucket that lost
+	// Meta.Buckets would be realigned from an empty axis and collapse into the
+	// overflow slot on the way back out
+	aggBucket := tsData.Aggregations[0]
+	assert.Equal(t, []float64{1, 2, 4}, aggBucket.Meta.Buckets)
+	assert.Equal(t, "By", aggBucket.Meta.Unit)
+	assert.Equal(t, "__result_0", aggBucket.Alias)
+}
+
+func TestRealignFromAnEmptyAxisCollapsesIntoTheOverflow(t *testing.T) {
+	// pins the behaviour the trim bug exposed: with no axis to read the counts
+	// against, everything lands in the overflow slot
+	aggBucket := &qbtypes.AggregationBucket{
+		Series: []*qbtypes.TimeSeries{{
+			Values: []*qbtypes.TimeSeriesValue{{Timestamp: 1710000000000, Values: []float64{7, 8, 9, 10}}},
+		}},
+	}
+
+	aggBucket.ReindexValuesToNewUpperBounds([]float64{1, 2, 4})
+
+	assert.Equal(t, []float64{0, 0, 0, 7}, aggBucket.Series[0].Values[0].Values)
+}

pkg/querier/postprocess.go

@@ -195,6 +195,16 @@ func postProcessBuilderQuery[T any](
 	return result
 }
 
+// resolveHeatmapBucketAxis brings the bucket axis to the resolution the caller
+// asked for. Downscaling runs first so AddHeatmapBucketsWithNoCounts adds them
+// at that resolution rather than the finer one ClickHouse bucketed at.
+func resolveHeatmapBucketAxis(tsData *qbtypes.TimeSeriesData, bucketing qbtypes.HeatmapBucketing) {
+	if bucketing.Kind == qbtypes.BucketsKindLog {
+		qbtypes.DownscaleHeatmapResolution(tsData, bucketing.LogScale)
+	}
+	qbtypes.AddHeatmapBucketsWithNoCounts(tsData, bucketing)
+}
+
 // postProcessMetricQuery applies postprocessing to a metric query result.
 func postProcessMetricQuery(
 	q *querier,
@@ -216,6 +226,12 @@ func postProcessMetricQuery(
 		}
 	}
 
+	if req.RequestType == qbtypes.RequestTypeHeatmap && config.HeatmapBucketing != nil {
+		if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
+			resolveHeatmapBucketAxis(tsData, *config.HeatmapBucketing)
+		}
+	}
+
 	result = q.applySeriesLimit(result, query.Limit, query.Order)
 
 	if len(query.Functions) > 0 {
@@ -342,6 +358,19 @@ func (q *querier) applyFormulas(ctx context.Context, results map[string]*qbtypes
 				result = q.applySeriesLimit(result, formula.Limit, formula.Order)
 				results[name] = result
 			}
+		case qbtypes.RequestTypeHeatmap:
+			// The queries a formula reads were run as time series, so what
+			// arrives here is one value per group per timestamp.
+			result := q.processTimeSeriesFormula(ctx, results, formula, req)
+			if result != nil {
+				if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {
+					bucketing := formula.BucketOptions.ToHeatmapBucketing()
+					bucketFormulaOutputAsHeatmap(tsData, bucketing)
+					resolveHeatmapBucketAxis(tsData, bucketing)
+				}
+				result = q.applySeriesLimit(result, formula.Limit, formula.Order)
+				results[name] = result
+			}
 		case qbtypes.RequestTypeScalar:
 			result := q.processScalarFormula(ctx, results, formula, req)
 			// For scalar results, apply limit by processScalarFormula itself since it needs to be applied before converting back to scalar format
@@ -410,6 +439,89 @@ func (q *querier) processTimeSeriesFormula(
 	return result
 }
 
+func bucketFormulaOutputAsHeatmap(tsData *qbtypes.TimeSeriesData, bucketing qbtypes.HeatmapBucketing) {
+	// A formula is one expression, so processTimeSeriesFormula gives it one
+	// aggregation.
+	if tsData == nil || len(tsData.Aggregations) == 0 || tsData.Aggregations[0] == nil {
+		return
+	}
+	aggBucket := tsData.Aggregations[0]
+
+	calculateUpperBound := calculateLogValueUpperBound
+	if bucketing.Kind == qbtypes.BucketsKindLinear {
+		calculateUpperBound = func(value float64) float64 {
+			return calculateLinearValueUpperBound(bucketing, value)
+		}
+	}
+
+	// +Inf is the open-above overflow rather than an upper bound of its own, and
+	// a NaN value has no bucket at all, so neither goes on the axis.
+	upperBounds := []float64{}
+	for _, series := range aggBucket.Series {
+		for _, point := range series.Values {
+			upperBound := calculateUpperBound(point.Value)
+			if !math.IsNaN(upperBound) && !math.IsInf(upperBound, 0) {
+				upperBounds = append(upperBounds, upperBound)
+			}
+		}
+	}
+	slices.Sort(upperBounds)
+	upperBounds = slices.Compact(upperBounds)
+
+	upperBoundToIndex := make(map[float64]int, len(upperBounds))
+	for index, upperBound := range upperBounds {
+		upperBoundToIndex[upperBound] = index
+	}
+
+	overflowIndex := len(upperBounds)
+	for _, series := range aggBucket.Series {
+		for _, point := range series.Values {
+			upperBound := calculateUpperBound(point.Value)
+			point.Values = make([]float64, overflowIndex+1)
+			point.Value = 0
+			switch {
+			case math.IsNaN(upperBound):
+			case math.IsInf(upperBound, 1):
+				point.Values[overflowIndex] = 1
+			default:
+				point.Values[upperBoundToIndex[upperBound]] = 1
+			}
+		}
+	}
+
+	aggBucket.Meta.Buckets = upperBounds
+}
+
+// calculateLinearValueUpperBound and calculateLogValueUpperBound are the Go side
+// of what renderLinearUpperBoundExpr and renderLogUpperBoundExpr emit, and have
+// to stay identical to them: a formula heatmap and a metric heatmap that
+// disagreed here would put their counts in different buckets.
+func calculateLinearValueUpperBound(bucketing qbtypes.HeatmapBucketing, value float64) float64 {
+	if value > bucketing.MaxValue {
+		return math.Inf(1)
+	}
+	numBuckets := float64(bucketing.NumBuckets)
+	index := math.Min(math.Max(math.Ceil(value*numBuckets/bucketing.MaxValue), 1), numBuckets)
+	return index * bucketing.MaxValue / numBuckets
+}
+
+// Like renderLogUpperBoundExpr, this reads MaxLogScale rather than the requested
+// scale: ClickHouse buckets at the finest resolution and resolveHeatmapBucketAxis
+// folds the axis down afterwards.
+func calculateLogValueUpperBound(value float64) float64 {
+	if value <= 0 {
+		return 0
+	}
+	if value <= qbtypes.MinLogUpperBound {
+		return qbtypes.MinLogUpperBound
+	}
+	if value > qbtypes.MaxLogUpperBound {
+		return math.Inf(1)
+	}
+	bucketsPerDoubling := math.Exp2(qbtypes.MaxLogScale)
+	return math.Exp2(math.Ceil(math.Log2(value)*bucketsPerDoubling) / bucketsPerDoubling)
+}
+
 func (q *querier) processScalarFormula(
 	ctx context.Context,
 	results map[string]*qbtypes.Result,
@@ -494,7 +606,7 @@ func (q *querier) processScalarFormula(
 				bucket := &qbtypes.AggregationBucket{
 					Index:  aggIdx,
 					Alias:  scalarData.Columns[colIdx].Name,
-					Meta:   scalarData.Columns[colIdx].Meta,
+					Meta:   qbtypes.AggregationMeta{Unit: scalarData.Columns[colIdx].Meta.Unit},
 					Series: make([]*qbtypes.TimeSeries, 0),
 				}
 
@@ -667,13 +779,14 @@ func convertTimeSeriesDataToScalar(tsData *qbtypes.TimeSeriesData, queryName str
 		if name == "" {
 			name = fmt.Sprintf("__result_%d", agg.Index)
 		}
-		columns = append(columns, &qbtypes.ColumnDescriptor{
+		column := &qbtypes.ColumnDescriptor{
 			TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: name},
 			QueryName:         queryName,
 			AggregationIndex:  int64(agg.Index),
-			Meta:              agg.Meta,
 			Type:              qbtypes.ColumnTypeAggregation,
-		})
+		}
+		column.Meta.Unit = agg.Meta.Unit
+		columns = append(columns, column)
 	}
 
 	// Build rows.

pkg/querier/preview.go

@@ -50,7 +50,7 @@ func (q *querier) QueryRangePreview(
 		env := []qbtypes.QueryEnvelope{req.CompositeQuery.Queries[idx]}
 		ps.Warnings = append(ps.Warnings, q.adjustStepInterval(env, req.Start, req.End)...)
 
-		missingMetricQueries, metricWarnings, mErr := q.resolveMetricMetadata(ctx, orgID, env, req.Start, req.End)
+		missingMetricQueries, metricWarnings, mErr := q.resolveMetricMetadata(ctx, orgID, env, req.Start, req.End, req.RequestType)
 		if mErr != nil {
 			// Report this query's error but keep previewing the rest.
 			ps.Error = mErr
@@ -231,7 +231,7 @@ func (q *querier) buildPreviewProviders(
 			sub.CompositeQuery = qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{query}}
 		}
 
-		built, _, bErr := q.buildQueries(orgID, &sub, deps, missingMetricQuerySet, event)
+		built, _, bErr := q.buildQueries(orgID, &sub, deps, missingMetricQuerySet, event, promqlOptions{})
 		if bErr != nil {
 			errs[name] = bErr
 			continue

pkg/querier/promql_heatmap.go

@@ -0,0 +1,149 @@
+package querier
+
+import (
+	"fmt"
+	"math"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/prometheus/prometheus/model/labels"
+
+	"github.com/prometheus/prometheus/promql"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// promHistogramBucketLabel is the label a classic histogram carries its
+// cumulative upper bound on.
+const promHistogramBucketLabel = "le"
+
+// cumulativeColumn maps a bucket's upper bound to the cumulative count at it.
+// Differencing turns it into the per-band counts a heatmapColumn holds.
+type cumulativeColumn map[float64]float64
+
+// promHeatmapGroup assembles one group across the several matrix series its `le`
+// values arrive as, since differencing needs all of them.
+type promHeatmapGroup struct {
+	labels     []*qbv5.Label
+	labelsKey  string
+	cumulative map[int64]cumulativeColumn
+}
+
+// foldMatrixAsHeatmap folds a matrix of one cumulative series per (group, `le`)
+// into one series per group whose points hold a count per band.
+func foldMatrixAsHeatmap(matrix promql.Matrix, queryWindow *qbv5.TimeRange, stepMs uint64, queryName string) (*qbv5.TimeSeriesData, error) {
+	groups, groupOrder := collectCumulativeGroups(matrix)
+
+	// An empty matrix is only ever the window having no data, but series that
+	// all lack `le` say the expression itself cannot draw a heatmap.
+	if len(matrix) > 0 && len(groups) == 0 {
+		return nil, errors.NewInvalidInputf(errors.CodeInvalidInput,
+			"the query returned no `le` labels to build a bucket axis from").
+			WithAdditional("Keep `le` through the aggregation, e.g. sum by (le) (rate(metric_bucket[5m]))")
+	}
+
+	accumulator := newHeatmapAccumulator()
+	for _, labelsKey := range groupOrder {
+		if err := groups[labelsKey].addDifferencedCells(accumulator); err != nil {
+			return nil, err
+		}
+	}
+
+	return accumulator.foldSeries(queryWindow, stepMs, queryName)
+}
+
+// collectCumulativeGroups reads the matrix into one group per label set. A series
+// without `le` has no band to sit in, so an expression that dropped the label
+// draws nothing.
+func collectCumulativeGroups(matrix promql.Matrix) (groups map[string]*promHeatmapGroup, groupOrder []string) {
+	groups = map[string]*promHeatmapGroup{}
+
+	for _, promSeries := range matrix {
+		upperBound, ok := extractBucketUpperBound(promSeries.Metric)
+		if !ok {
+			continue
+		}
+
+		lbls, labelsKey := extractHeatmapGroup(promSeries.Metric)
+		group, ok := groups[labelsKey]
+		if !ok {
+			group = &promHeatmapGroup{labels: lbls, labelsKey: labelsKey, cumulative: map[int64]cumulativeColumn{}}
+			groups[labelsKey] = group
+			groupOrder = append(groupOrder, labelsKey)
+		}
+
+		for _, point := range promSeries.Floats {
+			if math.IsNaN(point.F) || math.IsInf(point.F, 0) {
+				continue
+			}
+			if group.cumulative[point.T] == nil {
+				group.cumulative[point.T] = cumulativeColumn{}
+			}
+			group.cumulative[point.T][upperBound] = point.F
+		}
+	}
+
+	return groups, groupOrder
+}
+
+func extractBucketUpperBound(metric labels.Labels) (float64, bool) {
+	raw := metric.Get(promHistogramBucketLabel)
+	if raw == "" {
+		return 0, false
+	}
+	upperBound, err := strconv.ParseFloat(raw, 64)
+	if err != nil || math.IsNaN(upperBound) || math.IsInf(upperBound, -1) {
+		return 0, false
+	}
+	return upperBound, true
+}
+
+// extractHeatmapGroup returns a series' group labels — everything but `le`.
+func extractHeatmapGroup(metric labels.Labels) ([]*qbv5.Label, string) {
+	lbls := make([]*qbv5.Label, 0, metric.Len())
+	pairs := make([]string, 0, metric.Len())
+
+	metric.Range(func(l labels.Label) {
+		if l.Name == promHistogramBucketLabel || excludePromLabel(l.Name) {
+			return
+		}
+		lbls = append(lbls, &qbv5.Label{
+			Key:   telemetrytypes.TelemetryFieldKey{Name: l.Name},
+			Value: l.Value,
+		})
+		pairs = append(pairs, fmt.Sprintf("%s=%s", l.Name, l.Value))
+	})
+
+	sort.Strings(pairs)
+	return lbls, strings.Join(pairs, ",")
+}
+
+// each cell is its upper bound's cumulative count minus the one below it, and
+// runs from that lower `le` up to its own. Nothing bounds the lowest one below:
+// a classic histogram counts negative observations in it too.
+func (g *promHeatmapGroup) addDifferencedCells(accumulator *heatmapAccumulator) error {
+	for ts, cumulative := range g.cumulative {
+		upperBounds := make([]float64, 0, len(cumulative))
+		for upperBound := range cumulative {
+			upperBounds = append(upperBounds, upperBound)
+		}
+		slices.Sort(upperBounds)
+
+		previousCount := float64(0)
+		previousBound := math.Inf(-1)
+		for _, upperBound := range upperBounds {
+			bounds := bucketBounds{Lower: previousBound, Upper: upperBound}
+			if err := accumulator.addCell(g.labelsKey, g.labels, ts, bounds, math.Max(cumulative[upperBound]-previousCount, 0)); err != nil {
+				return err
+			}
+			previousCount = cumulative[upperBound]
+			previousBound = upperBound
+		}
+	}
+
+	return nil
+}

pkg/querier/promql_heatmap_test.go

@@ -0,0 +1,88 @@
+package querier
+
+import (
+	"log/slog"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+
+	qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The cache key is the fingerprint alone, so two request types over one
+// expression must not produce the same one — a time series payload served to a
+// heatmap request has no axis and reads back as a single collapsed band.
+func TestFingerprintSeparatesHeatmapFromTimeSeries(t *testing.T) {
+	fingerprintFor := func(requestType qbv5.RequestType) string {
+		q := &promqlQuery{
+			logger:      slog.New(slog.DiscardHandler),
+			query:       qbv5.PromQuery{Name: "A", Query: "sum by (le) (increase(signoz_latency_bucket[5m]))", Step: qbv5.Step{Duration: time.Minute}},
+			tr:          qbv5.TimeRange{From: 1710000000000, To: 1710003600000},
+			requestType: requestType,
+		}
+		return q.Fingerprint()
+	}
+
+	heatmap := fingerprintFor(qbv5.RequestTypeHeatmap)
+	timeSeries := fingerprintFor(qbv5.RequestTypeTimeSeries)
+
+	assert.NotEmpty(t, heatmap, "a heatmap decomposes into time buckets like a time series")
+	assert.NotEqual(t, timeSeries, heatmap)
+	assert.Empty(t, fingerprintFor(qbv5.RequestTypeScalar), "a scalar result is its window's last point")
+}
+
+func TestFoldMatrixAsHeatmapClampsADecreasingCumulativeCount(t *testing.T) {
+	at := int64(1710000000000)
+
+	matrix := promql.Matrix{
+		{
+			Metric: labels.FromStrings("le", "5"),
+			Floats: []promql.FPoint{{T: at, F: 10}},
+		},
+		{
+			Metric: labels.FromStrings("le", "10"),
+			Floats: []promql.FPoint{{T: at, F: 4}},
+		},
+	}
+
+	data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")
+	require.NoError(t, err)
+	require.Len(t, data.Aggregations, 1)
+
+	// a cumulative count that went backwards would difference to -6
+	assert.Equal(t, []float64{10, 0, 0}, data.Aggregations[0].Series[0].Values[0].Values)
+}
+
+func TestFoldMatrixAsHeatmapWidensTheBandOverAMissingUpperBound(t *testing.T) {
+	at := int64(1710000000000)
+
+	matrix := promql.Matrix{
+		{
+			Metric: labels.FromStrings("le", "5"),
+			Floats: []promql.FPoint{{T: at, F: 3}},
+		},
+		{
+			Metric: labels.FromStrings("le", "10"),
+			Floats: []promql.FPoint{{T: at, F: math.NaN()}},
+		},
+		{
+			Metric: labels.FromStrings("le", "20"),
+			Floats: []promql.FPoint{{T: at, F: 30}},
+		},
+	}
+
+	data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")
+	require.NoError(t, err)
+	require.Len(t, data.Aggregations, 1)
+
+	aggregation := data.Aggregations[0]
+	// 10 carried nothing to difference against, so it is not on the axis at all
+	// and 20 differences against 5, holding what (5,10] and (10,20] would split
+	assert.Equal(t, []float64{5, 20}, aggregation.Meta.Buckets)
+	assert.Equal(t, []float64{3, 27, 0}, aggregation.Series[0].Values[0].Values)
+}

pkg/querier/promql_query.go

@@ -102,6 +102,24 @@ type promqlQuery struct {
 	tr          qbv5.TimeRange
 	requestType qbv5.RequestType
 	vars        map[string]qbv5.VariableItem
+	opts        promqlOptions
+}
+
+// promqlOptions is how a PromQL query relates to the clickhousev2 provider
+// (see querier.promqlOptions for where the fields come from and why they are
+// flag-gated). Both providers are nil for a plain request, so a plain
+// request costs nothing extra.
+type promqlOptions struct {
+	// shadow, when set, runs the query on this provider after serving and
+	// logs any result difference; the response is never affected.
+	shadow prometheus.Prometheus
+	// shadowSlots is the querier-wide admission for shadow runs, shared by
+	// every query so the bound holds per process.
+	shadowSlots chan struct{}
+	// serve, when set, serves the response from this provider instead of the
+	// default path. Comparison callers fetch the default and the pinned
+	// result as two API calls and diff them.
+	serve prometheus.Prometheus
 }
 
 var _ qbv5.Query = (*promqlQuery)(nil)
@@ -114,20 +132,33 @@ func newPromqlQuery(
 	tr qbv5.TimeRange,
 	requestType qbv5.RequestType,
 	variables map[string]qbv5.VariableItem,
+	opts promqlOptions,
 ) *promqlQuery {
 	return &promqlQuery{
 		logger:      logger,
 		promEngine:  promEngine,
-		parser:      prometheus.NewParser(),
+		parser:      promEngine.Parser(),
 		query:       query,
 		tr:          tr,
 		requestType: requestType,
 		vars:        variables,
+		opts:        opts,
 	}
 }
 
 func (q *promqlQuery) Fingerprint() string {
-	if q.requestType != qbv5.RequestTypeTimeSeries {
+	// A pinned request must not share cache entries with default serving: a
+	// cached default result would satisfy the pin without running the pinned
+	// provider, and a pinned result would poison normal serving. No
+	// fingerprint means no caching at all — the pin exists to observe a
+	// provider, so a cache in front of it defeats the point.
+	if q.opts.serve != nil {
+		return ""
+	}
+
+	switch q.requestType {
+	case qbv5.RequestTypeTimeSeries, qbv5.RequestTypeHeatmap:
+	default:
 		return ""
 	}
 
@@ -138,6 +169,8 @@ func (q *promqlQuery) Fingerprint() string {
 	}
 	parts := []string{
 		"promql",
+		// one expression returns a different shape per request type
+		fmt.Sprintf("requestType=%s", q.requestType.StringValue()),
 		query,
 		q.query.Step.String(),
 	}
@@ -239,9 +272,15 @@ func (q *promqlQuery) Statement(_ context.Context) (*qbv5.Statement, error) {
 	return &qbv5.Statement{Query: rendered}, nil
 }
 
-// PreviewStatements returns the ClickHouse statement(s) this PromQL query
-// would run on the engine path, captured without executing them.
+// PreviewStatements returns the ClickHouse statement(s) this PromQL query would
+// run, captured by driving the engine with a Storage that records each selector's
+// SQL and returns no data. Returns nil if capture is unsupported.
 func (q *promqlQuery) PreviewStatements(ctx context.Context) ([]prometheus.CapturedStatement, error) {
+	storer, ok := q.promEngine.(prometheus.StatementCapturer)
+	if !ok {
+		return nil, nil
+	}
+
 	rendered, err := q.renderVars(q.query.Query, q.vars, q.tr.From, q.tr.To)
 	if err != nil {
 		return nil, err
@@ -250,11 +289,42 @@ func (q *promqlQuery) PreviewStatements(ctx context.Context) ([]prometheus.Captu
 	start := int64(querybuilder.ToNanoSecs(q.tr.From))
 	end := int64(querybuilder.ToNanoSecs(q.tr.To))
 
-	statements, err := q.promEngine.Statements(ctx, rendered, time.Unix(0, start), time.Unix(0, end), q.query.Step.Duration)
-	if err != nil {
-		return nil, q.evalError(rendered, err)
+	// Attach the same query traits as Execute so the captured statements
+	// match what the live path would run.
+	if expr, parseErr := q.parser.ParseExpr(rendered); parseErr == nil {
+		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
 	}
-	return statements, nil
+
+	capStorage, recorder := storer.CapturingStorage()
+	if capStorage == nil {
+		return nil, nil
+	}
+	qry, err := q.promEngine.Engine().NewRangeQuery(
+		ctx,
+		capStorage,
+		nil,
+		rendered,
+		time.Unix(0, start),
+		time.Unix(0, end),
+		q.query.Step.Duration,
+	)
+	if err != nil {
+		if e := tryEnhancePromQLExecError(err); e != nil {
+			return nil, e
+		}
+		return nil, enhancePromQLError(rendered, err)
+	}
+	defer qry.Close()
+
+	// Exec drives a Select per selector (recording SQL) but reads no data.
+	if res := qry.Exec(ctx); res.Err != nil {
+		if e := tryEnhancePromQLExecError(res.Err); e != nil {
+			return nil, e
+		}
+		return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", res.Err)
+	}
+
+	return recorder.Statements(), nil
 }
 
 func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
@@ -272,6 +342,13 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		return nil, err
 	}
 
+	// Attach query traits so the storage can prove step-aligned optimizations
+	// safe (see prometheus.QueryTraits). A parse failure surfaces below via
+	// the engine with the enhanced error message.
+	if expr, parseErr := q.parser.ParseExpr(query); parseErr == nil {
+		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
+	}
+
 	// Accumulate ClickHouse-side scan stats across every storage query this
 	// evaluation issues (engine selectors or the compiled executor): progress
 	// options propagate to each ClickHouse query through the context.
@@ -286,56 +363,148 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 
 	began := time.Now()
 
-	res, err := q.promEngine.QueryRange(ctx, query, time.Unix(0, start), time.Unix(0, end), q.query.Step.Duration)
-	if err != nil {
-		return nil, q.evalError(query, err)
+	// A pinned provider serves directly from it: comparison callers fetch
+	// the default result and the pinned result as two API calls and diff
+	// them.
+	if q.opts.serve != nil {
+		matrix, err := q.serveFromProvider(ctx, query, start, end)
+		if err != nil {
+			if enhanced := tryEnhancePromQLExecError(err); enhanced != nil {
+				return nil, enhanced
+			}
+			return nil, err
+		}
+		return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned)
 	}
 
-	matrix, ok := res.Value.(promql.Matrix)
-	if !ok {
-		return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "promql query %q returned %T, expected a matrix", query, res.Value)
+	// When the serving provider has the RangeExecutor capability
+	// (prometheus::provider: clickhousev2), serve the way the provider is
+	// designed to serve: transpiled when the shape allows. Without this the
+	// override would silently run the engine path only.
+	if re, ok := q.promEngine.(prometheus.RangeExecutor); ok {
+		matrix, served, err := re.TryExecuteRange(ctx, query, time.Unix(0, start), time.Unix(0, end), q.query.Step.Duration)
+		if err != nil {
+			if enhanced := tryEnhancePromQLExecError(err); enhanced != nil {
+				return nil, enhanced
+			}
+			return nil, err
+		}
+		if served {
+			return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned)
+		}
+	}
+
+	qry, err := q.promEngine.Engine().NewRangeQuery(
+		ctx,
+		q.promEngine.Storage(),
+		nil,
+		query,
+		time.Unix(0, start),
+		time.Unix(0, end),
+		q.query.Step.Duration,
+	)
+	if err != nil {
+		// NewRangeQuery can fail with execution errors (e.g. context deadline exceeded)
+		// during the query queue/scheduling stage, not just parse errors.
+		if err := tryEnhancePromQLExecError(err); err != nil {
+			return nil, err
+		}
+
+		return nil, enhancePromQLError(query, err)
+	}
+
+	res := qry.Exec(ctx)
+	if res.Err != nil {
+		if err := tryEnhancePromQLExecError(res.Err); err != nil {
+			return nil, err
+		}
+
+		return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", res.Err)
+	}
+
+	defer qry.Close()
+
+	matrix, promErr := res.Matrix()
+	if promErr != nil {
+		return nil, errors.WrapInternalf(promErr, errors.CodeInternal, "error getting matrix from promql query %q", query)
+	}
+
+	if q.opts.shadow != nil {
+		// Shadows detach from the request, so without admission a dashboard
+		// burst would stack unbounded ClickHouse work for up to the shadow
+		// timeout — the concurrency pattern behind the original outages.
+		// Non-blocking: at the cap the comparison is skipped, not queued;
+		// a sampled shadow stream is exactly as useful for rollout evidence.
+		select {
+		case q.opts.shadowSlots <- struct{}{}:
+			// The engine pools the result's sample slices on Close; the
+			// shadow comparison needs a stable copy of what was served.
+			served := copyMatrix(matrix)
+			servedIn := time.Since(began)
+			go func() {
+				defer func() { <-q.opts.shadowSlots }()
+				q.runShadowCompare(context.WithoutCancel(ctx), query, start, end, served, servedIn)
+			}()
+		default:
+			q.logger.DebugContext(ctx, "promql shadow skipped: at concurrency cap", slog.String("query", query))
+		}
 	}
 
 	warnings, _ := res.Warnings.AsStrings(query, 10, 0)
-	return q.toResult(matrix, warnings, began, &statsMu, &rowsScanned, &bytesScanned), nil
+	return q.toResult(matrix, warnings, began, &statsMu, &rowsScanned, &bytesScanned)
 }
 
-// evalError types an evaluation error: engine execution classes first, then
-// parse errors with the migration hints, everything else internal.
-func (q *promqlQuery) evalError(query string, err error) error {
-	if enhanced := tryEnhancePromQLExecError(err); enhanced != nil {
-		return enhanced
-	}
-	var parseErrs parser.ParseErrors
-	if errors.As(err, &parseErrs) {
-		return enhancePromQLError(query, err)
-	}
-	// The transpiled path raises typed user errors of its own; keep them.
-	if errors.Ast(err, errors.TypeInvalidInput) {
-		return err
-	}
-	return errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", err)
+// excludePromLabel hides only known SigNoz storage keys: label names are user
+// data and may legitimately start with "__" (e.g. __address__), so a blanket
+// dunder strip mangles user labelsets. The __scope./__resource. prefixes cover
+// every exporter version's keys.
+func excludePromLabel(labelName string) bool {
+	return labelName == "__temporality__" ||
+		strings.HasPrefix(labelName, "__scope.") ||
+		strings.HasPrefix(labelName, "__resource.")
 }
 
-// toResult converts an evaluated matrix into the v5 result shape, attaching
-// the ClickHouse scan stats accumulated during evaluation.
-func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) *qbv5.Result {
-	// Hide only known SigNoz storage keys: label names are user data and may
-	// legitimately start with "__" (e.g. __address__), so a blanket dunder
-	// strip mangles user labelsets. The __scope./__resource. prefixes cover
-	// every exporter version's keys.
-	excludeLabel := func(labelName string) bool {
-		return labelName == "__temporality__" ||
-			strings.HasPrefix(labelName, "__scope.") ||
-			strings.HasPrefix(labelName, "__resource.")
+// collectExecStats snapshots the scan counters a query accumulated. Callers take
+// it at the point they are done with the matrix, so the duration covers the
+// shaping they did.
+func collectExecStats(began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) qbv5.ExecStats {
+	statsMu.Lock()
+	defer statsMu.Unlock()
+	return qbv5.ExecStats{
+		RowsScanned:  *rowsScanned,
+		BytesScanned: *bytesScanned,
+		DurationMS:   uint64(time.Since(began).Milliseconds()),
+	}
+}
+
+func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) (*qbv5.Result, error) {
+	if q.requestType == qbv5.RequestTypeHeatmap {
+		return q.toResultForHeatmap(matrix, warnings, began, statsMu, rowsScanned, bytesScanned)
+	}
+	return q.toResultForTimeSeriesAndScalar(matrix, warnings, began, statsMu, rowsScanned, bytesScanned), nil
+}
+
+func (q *promqlQuery) toResultForHeatmap(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) (*qbv5.Result, error) {
+	tsData, err := foldMatrixAsHeatmap(matrix, &q.tr, uint64(q.query.Step.Milliseconds()), q.query.Name)
+	if err != nil {
+		return nil, err
 	}
 
+	return &qbv5.Result{
+		Type:     q.requestType,
+		Value:    tsData,
+		Warnings: warnings,
+		Stats:    collectExecStats(began, statsMu, rowsScanned, bytesScanned),
+	}, nil
+}
+
+func (q *promqlQuery) toResultForTimeSeriesAndScalar(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) *qbv5.Result {
 	var series []*qbv5.TimeSeries
 	for _, v := range matrix {
 		var s qbv5.TimeSeries
 		lbls := make([]*qbv5.Label, 0, v.Metric.Len())
 		v.Metric.Range(func(l labels.Label) {
-			if excludeLabel(l.Name) {
+			if excludePromLabel(l.Name) {
 				return
 			}
 			lbls = append(lbls, &qbv5.Label{
@@ -363,13 +532,7 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti
 		series = append(series, &s)
 	}
 
-	statsMu.Lock()
-	stats := qbv5.ExecStats{
-		RowsScanned:  *rowsScanned,
-		BytesScanned: *bytesScanned,
-		DurationMS:   uint64(time.Since(began).Milliseconds()),
-	}
-	statsMu.Unlock()
+	stats := collectExecStats(began, statsMu, rowsScanned, bytesScanned)
 
 	tsData := &qbv5.TimeSeriesData{QueryName: q.query.Name}
 	// No bucket at all when nothing survived: a bucket holding no series reads

pkg/querier/promql_query_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/prometheustest"
 	qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -448,6 +449,18 @@ func TestQuotedMetricOutsideBracesPattern(t *testing.T) {
 	}
 }
 
+// A pinned request must not share cache entries with default serving: a
+// cached default result would satisfy the pin without running the pinned
+// provider.
+func TestFingerprint_PinnedProviderBypassesCache(t *testing.T) {
+	q := &promqlQuery{
+		logger: slog.Default(),
+		query:  qbv5.PromQuery{Query: "up"},
+		opts:   promqlOptions{serve: &prometheustest.Provider{}},
+	}
+	assert.Empty(t, q.Fingerprint())
+}
+
 func TestToResultDropsNonFiniteValues(t *testing.T) {
 	tests := []struct {
 		description        string
@@ -482,7 +495,8 @@ func TestToResultDropsNonFiniteValues(t *testing.T) {
 
 			var mu sync.Mutex
 			var rows, bytes uint64
-			result := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes)
+			result, err := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes)
+			require.NoError(t, err)
 
 			tsData, ok := result.Value.(*qbv5.TimeSeriesData)
 			require.True(t, ok)
@@ -513,7 +527,9 @@ func TestToResultDropsSeriesAndBucketLeftEmpty(t *testing.T) {
 
 	var mu sync.Mutex
 	var rows, bytes uint64
-	tsData, ok := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes).Value.(*qbv5.TimeSeriesData)
+	result, err := q.toResult(matrix, nil, time.Now(), &mu, &rows, &bytes)
+	require.NoError(t, err)
+	tsData, ok := result.Value.(*qbv5.TimeSeriesData)
 	require.True(t, ok)
 	require.Len(t, tsData.Aggregations, 1)
 	require.Len(t, tsData.Aggregations[0].Series, 1, "the all-NaN series is gone")
@@ -522,7 +538,9 @@ func TestToResultDropsSeriesAndBucketLeftEmpty(t *testing.T) {
 	allNaN := promql.Matrix{
 		{Metric: labels.FromStrings("job_name", "idleJob"), Floats: []promql.FPoint{{T: 1000, F: math.NaN()}}},
 	}
-	tsData, ok = q.toResult(allNaN, nil, time.Now(), &mu, &rows, &bytes).Value.(*qbv5.TimeSeriesData)
+	result, err = q.toResult(allNaN, nil, time.Now(), &mu, &rows, &bytes)
+	require.NoError(t, err)
+	tsData, ok = result.Value.(*qbv5.TimeSeriesData)
 	require.True(t, ok)
 	assert.Empty(t, tsData.Aggregations)
 }

pkg/querier/promql_shadow.go

@@ -0,0 +1,183 @@
+package querier
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+)
+
+// shadowTimeout bounds a shadow evaluation; a shadow run must never outlive
+// the request by much or pile up.
+const shadowTimeout = 2 * time.Minute
+
+// runShadowCompare executes the query on the clickhousev2 provider exactly
+// as it would serve (transpiled when the shape allows, engine over the v2
+// querier otherwise), compares against the served result and logs the
+// outcome. Serving is never affected: this runs after the response, off the
+// request context, and only logs. The mismatch and failure logs are the
+// rollout evidence — serving cuts over to v2 only after they stay clean.
+func (q *promqlQuery) runShadowCompare(ctx context.Context, query string, startNs, endNs int64, served promql.Matrix, servedIn time.Duration) {
+	defer func() {
+		if r := recover(); r != nil {
+			q.logger.ErrorContext(ctx, "promql shadow comparison panicked", slog.Any("panic", r), slog.String("query", query))
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(ctx, shadowTimeout)
+	defer cancel()
+
+	// The request context carries the served response's scan-stats progress
+	// callback; without replacing it the shadow's ClickHouse progress would
+	// race into the served stats. The response itself was already sent.
+	ctx = clickhouse.Context(ctx, clickhouse.WithProgress(func(*clickhouse.Progress) {}))
+
+	if expr, parseErr := q.parser.ParseExpr(query); parseErr == nil {
+		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
+	}
+
+	start, end := time.Unix(0, startNs), time.Unix(0, endNs)
+	began := time.Now()
+	shadow, transpiled, err := executeOnProvider(ctx, q.opts.shadow, query, start, end, q.query.Step.Duration)
+	shadowIn := time.Since(began)
+
+	logAttrs := []any{
+		slog.String("query", query),
+		slog.Int64("start_ms", startNs/int64(time.Millisecond)),
+		slog.Int64("end_ms", endNs/int64(time.Millisecond)),
+		slog.Duration("step", q.query.Step.Duration),
+		slog.Bool("transpiled", transpiled),
+		slog.Duration("served_in", servedIn),
+		slog.Duration("shadow_in", shadowIn),
+	}
+
+	if err != nil {
+		// A shadow failure would be a serving failure after rollout; surface
+		// it at the same level as a result mismatch.
+		q.logger.WarnContext(ctx, "promql shadow execution failed", append(logAttrs, slog.Any("error", err))...)
+		return
+	}
+
+	servedNorm := normalizeShadowMatrix(served)
+	shadowNorm := normalizeShadowMatrix(shadow)
+	if diff := diffShadowMatrices(servedNorm, shadowNorm); diff != "" {
+		q.logger.WarnContext(ctx, "promql shadow comparison mismatch", append(logAttrs,
+			slog.String("diff", diff),
+			slog.Int("served_series", len(servedNorm)),
+			slog.Int("shadow_series", len(shadowNorm)),
+		)...)
+		return
+	}
+	// Matches log the timings: served_in vs shadow_in across the fleet is
+	// the perf evidence for the cutover, gathered for free.
+	q.logger.DebugContext(ctx, "promql shadow comparison matched", logAttrs...)
+}
+
+func (q *promqlQuery) serveFromProvider(ctx context.Context, query string, startNs, endNs int64) (promql.Matrix, error) {
+	matrix, _, err := executeOnProvider(ctx, q.opts.serve, query, time.Unix(0, startNs), time.Unix(0, endNs), q.query.Step.Duration)
+	return matrix, err
+}
+
+// The returned matrix is an owned copy.
+func executeOnProvider(ctx context.Context, prov prometheus.Prometheus, query string, start, end time.Time, step time.Duration) (promql.Matrix, bool, error) {
+	if re, ok := prov.(prometheus.RangeExecutor); ok {
+		matrix, served, err := re.TryExecuteRange(ctx, query, start, end, step)
+		if err != nil {
+			return nil, true, err
+		}
+		if served {
+			return matrix, true, nil
+		}
+	}
+
+	qry, err := prov.Engine().NewRangeQuery(ctx, prov.Storage(), nil, query, start, end, step)
+	if err != nil {
+		return nil, false, err
+	}
+	defer qry.Close()
+
+	res := qry.Exec(ctx)
+	if res.Err != nil {
+		return nil, false, res.Err
+	}
+	matrix, err := res.Matrix()
+	if err != nil {
+		return nil, false, err
+	}
+	// Close returns the result's sample slices to the engine pool.
+	return copyMatrix(matrix), false, nil
+}
+
+func copyMatrix(matrix promql.Matrix) promql.Matrix {
+	out := make(promql.Matrix, 0, len(matrix))
+	for _, s := range matrix {
+		floats := make([]promql.FPoint, len(s.Floats))
+		copy(floats, s.Floats)
+		out = append(out, promql.Series{Metric: s.Metric.Copy(), Floats: floats})
+	}
+	return out
+}
+
+// normalizeShadowMatrix sorts by label set for order-independent
+// comparison. Both providers now resolve series identity the same way
+// (empty-valued labels dropped at read, no synthetic fingerprint label
+// since the v1 series-identity fix), so labels need no normalization.
+func normalizeShadowMatrix(matrix promql.Matrix) promql.Matrix {
+	out := make(promql.Matrix, 0, len(matrix))
+	out = append(out, matrix...)
+	sort.Slice(out, func(i, j int) bool { return labels.Compare(out[i].Metric, out[j].Metric) < 0 })
+	return out
+}
+
+// diffShadowMatrices returns a description of the first difference, or "".
+// Values compare with relative tolerance: spatial aggregations accumulate
+// floats in storage order, which differs between the providers in the last
+// ULP.
+func diffShadowMatrices(served, shadow promql.Matrix) string {
+	const relTol = 1e-9
+	if len(served) != len(shadow) {
+		return fmt.Sprintf("series count: served=%d shadow=%d", len(served), len(shadow))
+	}
+	for i := range served {
+		if labels.Compare(served[i].Metric, shadow[i].Metric) != 0 {
+			return fmt.Sprintf("series %d labels: served=%s shadow=%s", i, served[i].Metric, shadow[i].Metric)
+		}
+		if len(served[i].Floats) != len(shadow[i].Floats) {
+			return fmt.Sprintf("series %s points: served=%d shadow=%d", served[i].Metric, len(served[i].Floats), len(shadow[i].Floats))
+		}
+		for j := range served[i].Floats {
+			a, b := served[i].Floats[j], shadow[i].Floats[j]
+			if a.T != b.T {
+				return fmt.Sprintf("series %s point %d ts: served=%d shadow=%d", served[i].Metric, j, a.T, b.T)
+			}
+			// NaN and infinities first: NaN != NaN and Inf-Inf arithmetic
+			// would otherwise make one-sided NaN and Inf-vs-finite compare
+			// as equal (NaN > x and Inf > Inf are both false).
+			if math.IsNaN(a.F) || math.IsNaN(b.F) {
+				if math.IsNaN(a.F) != math.IsNaN(b.F) {
+					return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
+				}
+				continue
+			}
+			if math.IsInf(a.F, 0) || math.IsInf(b.F, 0) {
+				if a.F != b.F {
+					return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
+				}
+				continue
+			}
+			diff := math.Abs(a.F - b.F)
+			scale := math.Max(math.Abs(a.F), math.Abs(b.F))
+			if diff > relTol*math.Max(scale, 1e-300) && diff > 1e-12 {
+				return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
+			}
+		}
+	}
+	return ""
+}

pkg/querier/promql_shadow_test.go

@@ -0,0 +1,67 @@
+package querier
+
+import (
+	"math"
+	"testing"
+
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizeShadowMatrix(t *testing.T) {
+	matrix := promql.Matrix{
+		{
+			Metric: labels.FromStrings("__name__", "up", "job", "api"),
+			Floats: []promql.FPoint{{T: 1000, F: 1}},
+		},
+		{
+			Metric: labels.FromStrings("a", "1"),
+			Floats: []promql.FPoint{{T: 1000, F: 2}},
+		},
+	}
+	norm := normalizeShadowMatrix(matrix)
+	// sorted by label set; labels pass through untouched — both providers
+	// resolve series identity identically since the v1 series-identity fix
+	assert.Equal(t, labels.FromStrings("__name__", "up", "job", "api"), norm[0].Metric)
+	assert.Equal(t, labels.FromStrings("a", "1"), norm[1].Metric)
+}
+
+func TestDiffShadowMatrices(t *testing.T) {
+	series := func(v float64) promql.Matrix {
+		return promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 1000, F: v}}}}
+	}
+
+	assert.Empty(t, diffShadowMatrices(series(1.5), series(1.5)))
+	// last-ULP differences from storage-order float accumulation are expected
+	assert.Empty(t, diffShadowMatrices(series(0.08888888888888889), series(0.08888888888888888)))
+	assert.Empty(t, diffShadowMatrices(series(math.NaN()), series(math.NaN())))
+
+	assert.Contains(t, diffShadowMatrices(series(1.5), series(1.6)), "value")
+	assert.Contains(t, diffShadowMatrices(series(1.5), promql.Matrix{}), "series count")
+	assert.Contains(t, diffShadowMatrices(
+		series(1.5),
+		promql.Matrix{{Metric: labels.FromStrings("a", "2"), Floats: []promql.FPoint{{T: 1000, F: 1.5}}}},
+	), "labels")
+	assert.Contains(t, diffShadowMatrices(
+		series(1.5),
+		promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 2000, F: 1.5}}}},
+	), "ts")
+}
+
+// One-sided NaN makes every float comparison false, and Inf-Inf arithmetic
+// yields Inf > Inf == false; without explicit handling both divergences log
+// as matched — a shadow comparator that cannot see them would green-light a
+// broken rollout.
+func TestDiffShadowMatrices_SpecialFloats(t *testing.T) {
+	point := func(v float64) promql.Matrix {
+		return promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 1000, F: v}}}}
+	}
+
+	assert.NotEmpty(t, diffShadowMatrices(point(math.NaN()), point(1.5)), "one-sided NaN must diff")
+	assert.NotEmpty(t, diffShadowMatrices(point(1.5), point(math.NaN())), "one-sided NaN must diff either way")
+	assert.NotEmpty(t, diffShadowMatrices(point(math.Inf(1)), point(1.5)), "Inf vs finite must diff")
+	assert.NotEmpty(t, diffShadowMatrices(point(math.Inf(1)), point(math.Inf(-1))), "opposite infinities must diff")
+	assert.Empty(t, diffShadowMatrices(point(math.Inf(1)), point(math.Inf(1))), "equal infinities match")
+	assert.Empty(t, diffShadowMatrices(point(math.NaN()), point(math.NaN())), "both NaN match")
+}

pkg/querier/querier.go

@@ -24,6 +24,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/statsreporter"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
 	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
+	"github.com/SigNoz/signoz/pkg/types/featuretypes"
 	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
 	"github.com/SigNoz/signoz/pkg/types/metrictypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -46,11 +47,19 @@ type Querier interface {
 }
 
 type querier struct {
-	logger                   *slog.Logger
-	fl                       flagger.Flagger
-	telemetryStore           telemetrystore.TelemetryStore
-	metadataStore            telemetrytypes.MetadataStore
-	promEngine               prometheus.Prometheus
+	logger         *slog.Logger
+	fl             flagger.Flagger
+	telemetryStore telemetrystore.TelemetryStore
+	metadataStore  telemetrytypes.MetadataStore
+	promEngine     prometheus.Prometheus
+	// promV2 is the clickhousev2 prometheus provider, wired only when the
+	// serving provider is the default one (nil otherwise). It reads the same
+	// ClickHouse data through a different implementation; PromQL queries
+	// shadow-compare against it behind the use_prometheus_clickhouse_v2 flag
+	// and can be pinned to it for a response (see promqlOptions). It never
+	// serves by default — that cutover happens only after the shadow logs
+	// stay clean.
+	promV2                   prometheus.Prometheus
 	traceStmtBuilder         qbtypes.StatementBuilder[qbtypes.TraceAggregation]
 	aiTraceStmtBuilder       qbtypes.StatementBuilder[qbtypes.TraceAggregation]
 	logStmtBuilder           qbtypes.StatementBuilder[qbtypes.LogAggregation]
@@ -62,8 +71,16 @@ type querier struct {
 	liveDataRefresh          time.Duration
 	builderConfig            builderConfig
 	maxConcurrentQueries     int
+	// shadowSlots bounds concurrent shadow comparisons per process; shadows
+	// detach from their requests, so nothing else limits how many pile up.
+	shadowSlots chan struct{}
 }
 
+// maxConcurrentShadows is deliberately small: a shadow is a full extra
+// ClickHouse evaluation, and a sampled stream of comparisons is exactly as
+// useful for rollout evidence as an exhaustive one under load.
+const maxConcurrentShadows = 8
+
 var _ Querier = (*querier)(nil)
 
 func New(
@@ -71,6 +88,7 @@ func New(
 	telemetryStore telemetrystore.TelemetryStore,
 	metadataStore telemetrytypes.MetadataStore,
 	promEngine prometheus.Prometheus,
+	promV2 prometheus.Prometheus,
 	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
@@ -93,6 +111,7 @@ func New(
 		telemetryStore:           telemetryStore,
 		metadataStore:            metadataStore,
 		promEngine:               promEngine,
+		promV2:                   promV2,
 		traceStmtBuilder:         traceStmtBuilder,
 		aiTraceStmtBuilder:       aiTraceStmtBuilder,
 		logStmtBuilder:           logStmtBuilder,
@@ -106,6 +125,7 @@ func New(
 			logTraceIDWindowPaddingMS: uint64(logTraceIDWindowPadding.Milliseconds()),
 		},
 		maxConcurrentQueries: maxConcurrentQueries,
+		shadowSlots:          make(chan struct{}, maxConcurrentShadows),
 	}
 }
 
@@ -136,7 +156,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 	// We need to set if it is unspecified or adjust it if value is not within recommended range
 	intervalWarnings := q.adjustStepInterval(req.CompositeQuery.Queries, req.Start, req.End)
 
-	missingMetricQueries, metricWarnings, err := q.resolveMetricMetadata(ctx, orgID, req.CompositeQuery.Queries, req.Start, req.End)
+	missingMetricQueries, metricWarnings, err := q.resolveMetricMetadata(ctx, orgID, req.CompositeQuery.Queries, req.Start, req.End, req.RequestType)
 	if err != nil {
 		return nil, err
 	}
@@ -145,7 +165,11 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 		missingMetricQuerySet[name] = true
 	}
 
-	queries, steps, err := q.buildQueries(orgID, req, dependencyQueries, missingMetricQuerySet, event)
+	promqlOpts, err := q.promqlOptions(ctx, orgID, req)
+	if err != nil {
+		return nil, err
+	}
+	queries, steps, err := q.buildQueries(orgID, req, dependencyQueries, missingMetricQuerySet, event, promqlOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -153,7 +177,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 	preseededResults := make(map[string]any)
 	for _, name := range missingMetricQueries {
 		switch req.RequestType {
-		case qbtypes.RequestTypeTimeSeries:
+		case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 			preseededResults[name] = &qbtypes.TimeSeriesData{QueryName: name}
 		case qbtypes.RequestTypeScalar:
 			preseededResults[name] = &qbtypes.ScalarData{QueryName: name}
@@ -188,12 +212,41 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 	return qbResp, qbErr
 }
 
+// promqlOptions derives the PromQL execution options for a request. With the
+// org's use_prometheus_clickhouse_v2 flag on, queries are shadow-compared
+// against the clickhousev2 provider (serving unaffected, diffs logged; see
+// promql_shadow.go). The X-SigNoz-PromQL-Provider header may instead pin the
+// response to that provider — integration tests and support fetch both
+// results for comparison — so it is deliberately flag-gated too: without the
+// gate the header would be an unaudited switch onto a provider still under
+// validation.
+func (q *querier) promqlOptions(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (promqlOptions, error) {
+	enabled := q.fl.BooleanOrEmpty(ctx, flagger.FeatureUsePrometheusClickhouseV2, featuretypes.NewFlaggerEvaluationContext(orgID))
+	if req.PromQLProvider == "" {
+		if enabled && q.promV2 != nil {
+			return promqlOptions{shadow: q.promV2, shadowSlots: q.shadowSlots}, nil
+		}
+		return promqlOptions{}, nil
+	}
+	if req.PromQLProvider != prometheus.ProviderClickhouseV2 {
+		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "unknown promql provider %q", req.PromQLProvider)
+	}
+	if !enabled {
+		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "promql provider %q requires the use_prometheus_clickhouse_v2 flag", req.PromQLProvider)
+	}
+	if q.promV2 == nil {
+		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "promql provider %q is not available", req.PromQLProvider)
+	}
+	return promqlOptions{serve: q.promV2}, nil
+}
+
 func (q *querier) buildQueries(
 	orgID valuer.UUID,
 	req *qbtypes.QueryRangeRequest,
 	dependencyQueries map[string]bool,
 	missingMetricQuerySet map[string]bool,
 	event *qbtypes.QBEvent,
+	promqlOpts promqlOptions,
 ) (map[string]qbtypes.Query, map[string]qbtypes.Step, error) {
 
 	tmplVars := req.Variables
@@ -218,7 +271,7 @@ func (q *querier) buildQueries(
 			if !ok {
 				return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid promql query spec %T", query.Spec)
 			}
-			promqlQuery := newPromqlQuery(q.logger, q.promEngine, promQuery, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars)
+			promqlQuery := newPromqlQuery(q.logger, q.promEngine, promQuery, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars, promqlOpts)
 			queries[promQuery.Name] = promqlQuery
 			steps[promQuery.Name] = promQuery.Step
 		case qbtypes.QueryTypeClickHouseSQL:
@@ -281,15 +334,22 @@ func (q *querier) buildQueries(
 				if missingMetricQuerySet[spec.Name] {
 					continue
 				}
+				requestType := req.RequestType
+				if requestType == qbtypes.RequestTypeHeatmap && spec.Disabled {
+					// A disabled query in a heatmap request feeds a formula, and the
+					// formula converts time series into heatmap data, so its inputs
+					// run as time series queries.
+					requestType = qbtypes.RequestTypeTimeSeries
+				}
 				spec.ShiftBy = extractShiftFromBuilderQuery(spec)
-				timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
+				timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, requestType)
 				var bq *builderQuery[qbtypes.MetricAggregation]
 
 				if spec.Source == telemetrytypes.SourceMeter {
 					event.Source = telemetrytypes.SourceMeter.StringValue()
-					bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
+					bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, query.Type, spec, timeRange, requestType, tmplVars, builderConfig{})
 				} else {
-					bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
+					bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, query.Type, spec, timeRange, requestType, tmplVars, builderConfig{})
 				}
 
 				queries[spec.Name] = bq
@@ -362,7 +422,7 @@ func (q *querier) populateQBEvent(event *qbtypes.QBEvent, queries []qbtypes.Quer
 //     resolved: never-seen metrics and dormant metrics (seen but no data in
 //     the query window).
 //   - err: Internal when a metadata fetch fails.
-func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID, queries []qbtypes.QueryEnvelope, start, end uint64) (missingMetricQueries []string, metricWarnings []string, err error) {
+func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID, queries []qbtypes.QueryEnvelope, start, end uint64, requestType qbtypes.RequestType) (missingMetricQueries []string, metricWarnings []string, err error) {
 	metricNames := make([]string, 0)
 	for idx := range queries {
 		if queries[idx].Type != qbtypes.QueryTypeBuilder {
@@ -420,6 +480,13 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID,
 			if err := spec.Aggregations[i].ValidateForTypeAndTemporality(); err != nil {
 				return nil, nil, err
 			}
+			// Only the enabled query is used to render the heatmap, so bucket
+			// options are only applied to the enabled query.
+			if requestType == qbtypes.RequestTypeHeatmap && !spec.Disabled {
+				if err := spec.Aggregations[i].VerifyAndApplyBucketOptions(spec.BucketOptions); err != nil {
+					return nil, nil, err
+				}
+			}
 			if reducedMetricsSet[spec.Aggregations[i].MetricName] {
 				spec.Aggregations[i].Reduced = true
 			}
@@ -626,7 +693,7 @@ func (q *querier) run(
 			if val, ok := result.Value.(*qbtypes.RawData); ok && val != nil {
 				return len(val.Rows) != 0
 			}
-		case qbtypes.RequestTypeTimeSeries:
+		case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 			if val, ok := result.Value.(*qbtypes.TimeSeriesData); ok && val != nil {
 				if len(val.Aggregations) != 0 {
 					anyNonEmpty := false
@@ -876,7 +943,7 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
 	switch qt := originalQuery.(type) {
 	case *promqlQuery:
 		queryCopy := qt.query.Copy()
-		return newPromqlQuery(q.logger, qt.promEngine, queryCopy, timeRange, qt.requestType, qt.vars)
+		return newPromqlQuery(q.logger, qt.promEngine, queryCopy, timeRange, qt.requestType, qt.vars, qt.opts)
 
 	case *chSQLQuery:
 		queryCopy := qt.query.Copy()
@@ -947,7 +1014,7 @@ func (q *querier) mergeResults(cached *qbtypes.Result, fresh []*qbtypes.Result)
 
 		// Merge all fresh results including the first one
 		switch merged.Type {
-		case qbtypes.RequestTypeTimeSeries:
+		case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 			// Pass nil as cached value to ensure proper merging of all fresh results
 			merged.Value = q.mergeTimeSeriesResults(nil, fresh)
 		}
@@ -970,7 +1037,7 @@ func (q *querier) mergeResults(cached *qbtypes.Result, fresh []*qbtypes.Result)
 	}
 
 	switch merged.Type {
-	case qbtypes.RequestTypeTimeSeries:
+	case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:
 		merged.Value = q.mergeTimeSeriesResults(cached.Value.(*qbtypes.TimeSeriesData), fresh)
 	}
 
@@ -991,6 +1058,16 @@ func (q *querier) mergeResults(cached *qbtypes.Result, fresh []*qbtypes.Result)
 	return merged
 }
 
+func mergeBucketUpperBounds(cachedValue *qbtypes.TimeSeriesData, freshResults []*qbtypes.Result) map[int][]float64 {
+	upperBoundSources := make([]*qbtypes.TimeSeriesData, 0, len(freshResults)+1)
+	upperBoundSources = append(upperBoundSources, cachedValue)
+	for _, result := range freshResults {
+		freshTS, _ := result.Value.(*qbtypes.TimeSeriesData)
+		upperBoundSources = append(upperBoundSources, freshTS)
+	}
+	return qbtypes.MergeBucketUpperBounds(upperBoundSources...)
+}
+
 // mergeTimeSeriesResults merges time series data.
 func (q *querier) mergeTimeSeriesResults(cachedValue *qbtypes.TimeSeriesData, freshResults []*qbtypes.Result) *qbtypes.TimeSeriesData {
 
@@ -999,12 +1076,15 @@ func (q *querier) mergeTimeSeriesResults(cachedValue *qbtypes.TimeSeriesData, fr
 	// Map to store aggregation bucket metadata
 	bucketMetadata := make(map[int]*qbtypes.AggregationBucket)
 
+	mergedUpperBounds := mergeBucketUpperBounds(cachedValue, freshResults)
+
 	// Process cached data if available
 	if cachedValue != nil && cachedValue.Aggregations != nil {
 		for _, aggBucket := range cachedValue.Aggregations {
 			if seriesMap[aggBucket.Index] == nil {
 				seriesMap[aggBucket.Index] = make(map[string]*qbtypes.TimeSeries)
 			}
+			aggBucket.ReindexValuesToNewUpperBounds(mergedUpperBounds[aggBucket.Index])
 			if bucketMetadata[aggBucket.Index] == nil {
 				bucketMetadata[aggBucket.Index] = aggBucket
 			}
@@ -1056,6 +1136,7 @@ func (q *querier) mergeTimeSeriesResults(cachedValue *qbtypes.TimeSeriesData, fr
 		}
 
 		for _, aggBucket := range freshTS.Aggregations {
+			aggBucket.ReindexValuesToNewUpperBounds(mergedUpperBounds[aggBucket.Index])
 			for _, series := range aggBucket.Series {
 				key := qbtypes.GetUniqueSeriesKey(series.Labels)
 

pkg/querier/querier_test.go

@@ -48,6 +48,7 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
 		nil, // telemetryStore
 		metadataStore,
 		nil,                // prometheus
+		nil,                // promV2
 		nil,                // traceStmtBuilder
 		nil,                // aiTraceStmtBuilder
 		nil,                // logStmtBuilder
@@ -121,6 +122,7 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
+		nil, // promV2
 		nil, // traceStmtBuilder
 		nil, // aiTraceStmtBuilder
 		nil, // logStmtBuilder

pkg/querier/signozquerier/provider.go

@@ -18,6 +18,7 @@ import (
 func NewFactory(
 	telemetryStore telemetrystore.TelemetryStore,
 	prometheus prometheus.Prometheus,
+	promV2 prometheus.Prometheus,
 	metadataStore telemetrytypes.MetadataStore,
 	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
@@ -41,6 +42,7 @@ func NewFactory(
 				telemetryStore,
 				metadataStore,
 				prometheus,
+				promV2,
 				traceStmtBuilder,
 				aiTraceStmtBuilder,
 				logStmtBuilder,

pkg/query-service/app/clickhouseReader/reader.go

@@ -35,7 +35,6 @@ import (
 	errorsV2 "github.com/SigNoz/signoz/pkg/errors"
 
 	"github.com/prometheus/prometheus/promql"
-	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/util/stats"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
@@ -231,43 +230,41 @@ func NewReader(
 }
 
 func (r *ClickHouseReader) GetInstantQueryMetricsResult(ctx context.Context, queryParams *model.InstantQueryMetricsParams) (*promql.Result, *stats.QueryStats, *model.ApiError) {
-	res, err := r.prometheus.Query(ctx, queryParams.Query, queryParams.Time)
-	var qs stats.QueryStats
+	qry, err := r.prometheus.Engine().NewInstantQuery(ctx, r.prometheus.Storage(), nil, queryParams.Query, queryParams.Time)
 	if err != nil {
-		var parseErrs parser.ParseErrors
-		if errorsV2.As(err, &parseErrs) {
-			return nil, nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
-		}
-		// Evaluation errors travel inside the result, as the engine reports
-		// them; the handler maps them from there.
-		return &promql.Result{Err: err}, &qs, nil
+		return nil, nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
 	}
 
+	res := qry.Exec(ctx)
+
 	// Optional stats field in response if parameter "stats" is not empty.
-	if queryParams.Stats != "" && res.Stats != nil {
-		qs = stats.NewQueryStats(res.Stats)
+	var qs stats.QueryStats
+	if queryParams.Stats != "" {
+		qs = stats.NewQueryStats(qry.Stats())
 	}
 
-	return &promql.Result{Value: res.Value, Warnings: res.Warnings}, &qs, nil
+	qry.Close()
+	return res, &qs, nil
+
 }
 
 func (r *ClickHouseReader) GetQueryRangeResult(ctx context.Context, query *model.QueryRangeParams) (*promql.Result, *stats.QueryStats, *model.ApiError) {
-	res, err := r.prometheus.QueryRange(ctx, query.Query, query.Start, query.End, query.Step)
-	var qs stats.QueryStats
+	qry, err := r.prometheus.Engine().NewRangeQuery(ctx, r.prometheus.Storage(), nil, query.Query, query.Start, query.End, query.Step)
+
 	if err != nil {
-		var parseErrs parser.ParseErrors
-		if errorsV2.As(err, &parseErrs) {
-			return nil, nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
-		}
-		return &promql.Result{Err: err}, &qs, nil
+		return nil, nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
 	}
 
+	res := qry.Exec(ctx)
+
 	// Optional stats field in response if parameter "stats" is not empty.
-	if query.Stats != "" && res.Stats != nil {
-		qs = stats.NewQueryStats(res.Stats)
+	var qs stats.QueryStats
+	if query.Stats != "" {
+		qs = stats.NewQueryStats(qry.Stats())
 	}
 
-	return &promql.Result{Value: res.Value, Warnings: res.Warnings}, &qs, nil
+	qry.Close()
+	return res, &qs, nil
 }
 
 func (r *ClickHouseReader) GetServicesList(ctx context.Context) (*[]string, error) {

pkg/query-service/app/server.go

@@ -2,9 +2,19 @@ package app
 
 import (
 	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"slices"
 
+	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/queryparser"
 
+	"github.com/gorilla/handlers"
+
+	"github.com/rs/cors"
+	"github.com/soheilhy/cmux"
+
 	"github.com/SigNoz/signoz/pkg/http/middleware"
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
 	"github.com/SigNoz/signoz/pkg/query-service/app/clickhouseReader"
@@ -13,15 +23,31 @@ import (
 	"github.com/SigNoz/signoz/pkg/query-service/app/opamp"
 	opAmpModel "github.com/SigNoz/signoz/pkg/query-service/app/opamp/model"
 	"github.com/SigNoz/signoz/pkg/signoz"
+	"github.com/SigNoz/signoz/pkg/web"
 
 	"log/slog"
 
+	"go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
+	"go.opentelemetry.io/otel/propagation"
+
 	"github.com/SigNoz/signoz/pkg/query-service/constants"
+	"github.com/SigNoz/signoz/pkg/query-service/healthcheck"
+	"github.com/SigNoz/signoz/pkg/query-service/utils"
 )
 
-// Server runs auxiliary servers (opamp) alongside the signoz apiserver
+// Server runs HTTP, Mux and a grpc server
 type Server struct {
+	config signoz.Config
+	signoz *signoz.SigNoz
+
+	// public http router
+	httpConn     net.Listener
+	httpServer   *http.Server
+	httpHostPort string
+
 	opampServer *opamp.Server
+
+	unavailableChannel chan healthcheck.Status
 }
 
 // NewServer creates and initializes Server
@@ -64,20 +90,20 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 		return nil, err
 	}
 
-	// Register the legacy query-service routes on the apiserver router. The
-	// apiserver owns the HTTP server and applies the middleware chain at serve
-	// time, so these routes get the same treatment as the apiserver routes.
-	r := signoz.APIServer.Router()
-	am := middleware.NewAuthZ(signoz.Instrumentation.Logger(), signoz.Modules.OrgGetter, signoz.Authz)
+	s := &Server{
+		config:             config,
+		signoz:             signoz,
+		httpHostPort:       constants.HTTPHostPort,
+		unavailableChannel: make(chan healthcheck.Status),
+	}
 
-	apiHandler.RegisterRoutes(r, am)
-	apiHandler.RegisterLogsRoutes(r, am)
-	apiHandler.RegisterIntegrationRoutes(r, am)
-	apiHandler.RegisterQueryRangeV3Routes(r, am)
-	apiHandler.RegisterQueryRangeV4Routes(r, am)
-	apiHandler.RegisterMessagingQueuesRoutes(r, am)
-	apiHandler.RegisterThirdPartyApiRoutes(r, am)
-	apiHandler.RegisterTraceFunnelsRoutes(r, am)
+	httpServer, err := s.createPublicServer(apiHandler, signoz.Web)
+
+	if err != nil {
+		return nil, err
+	}
+
+	s.httpServer = httpServer
 
 	opAmpModel.Init(signoz.SQLStore, signoz.Instrumentation.Logger(), signoz.Modules.OrgGetter)
 
@@ -95,8 +121,6 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 		return nil, err
 	}
 
-	s := &Server{}
-
 	s.opampServer = opamp.InitializeServer(
 		&opAmpModel.AllAgents,
 		agentConfMgr,
@@ -106,18 +130,146 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 	return s, nil
 }
 
-// Start starts the opamp websocket server. The HTTP API server is started by
-// the signoz registry.
-func (s *Server) Start(ctx context.Context) error {
-	slog.Info("Starting OpAmp Websocket server", "addr", constants.OpAmpWsEndpoint)
-	if err := s.opampServer.Start(constants.OpAmpWsEndpoint); err != nil {
+// HealthCheckStatus returns health check status channel a client can subscribe to
+func (s Server) HealthCheckStatus() chan healthcheck.Status {
+	return s.unavailableChannel
+}
+
+func (s *Server) createPublicServer(api *APIHandler, web web.Web) (*http.Server, error) {
+	r := NewRouter()
+
+	r.Use(middleware.NewRecovery(s.signoz.Instrumentation.Logger()).Wrap)
+	r.Use(otelmux.Middleware(
+		"apiserver",
+		otelmux.WithMeterProvider(s.signoz.Instrumentation.MeterProvider()),
+		otelmux.WithTracerProvider(s.signoz.Instrumentation.TracerProvider()),
+		otelmux.WithPropagators(propagation.NewCompositeTextMapPropagator(propagation.Baggage{}, propagation.TraceContext{})),
+		otelmux.WithFilter(func(r *http.Request) bool {
+			return !slices.Contains([]string{"/api/v1/health"}, r.URL.Path)
+		}),
+	))
+	r.Use(middleware.NewIdentN(s.signoz.IdentNResolver, s.signoz.Sharder, s.signoz.Instrumentation.Logger()).Wrap)
+	r.Use(middleware.NewTimeout(s.signoz.Instrumentation.Logger(),
+		s.config.APIServer.Timeout.ExcludedRoutes,
+		s.config.APIServer.Timeout.Default,
+		s.config.APIServer.Timeout.Max,
+	).Wrap)
+	r.Use(middleware.NewResource(s.signoz.Instrumentation.Logger()).Wrap)
+	r.Use(middleware.NewAudit(s.signoz.Instrumentation.Logger(), s.config.APIServer.Logging.ExcludedRoutes, s.signoz.Auditor).Wrap)
+	r.Use(middleware.NewComment().Wrap)
+
+	am := middleware.NewAuthZ(s.signoz.Instrumentation.Logger(), s.signoz.Modules.OrgGetter, s.signoz.Authz)
+
+	api.RegisterRoutes(r, am)
+	api.RegisterLogsRoutes(r, am)
+	api.RegisterIntegrationRoutes(r, am)
+	api.RegisterQueryRangeV3Routes(r, am)
+	api.RegisterQueryRangeV4Routes(r, am)
+	api.RegisterMessagingQueuesRoutes(r, am)
+	api.RegisterThirdPartyApiRoutes(r, am)
+	api.RegisterTraceFunnelsRoutes(r, am)
+
+	err := s.signoz.APIServer.AddToRouter(r)
+	if err != nil {
+		return nil, err
+	}
+
+	c := cors.New(cors.Options{
+		AllowedOrigins: []string{"*"},
+		AllowedMethods: []string{"GET", "DELETE", "POST", "PUT", "PATCH", "OPTIONS"},
+		AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "cache-control"},
+	})
+
+	handler := c.Handler(r)
+
+	handler = handlers.CompressHandler(handler)
+
+	err = web.AddToRouter(r)
+	if err != nil {
+		return nil, err
+	}
+
+	routePrefix := s.config.Global.ExternalPath()
+	if routePrefix != "" {
+		prefixed := http.StripPrefix(routePrefix, handler)
+		handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			switch req.URL.Path {
+			case "/api/v1/health", "/api/v2/healthz", "/api/v2/readyz", "/api/v2/livez":
+				r.ServeHTTP(w, req)
+				return
+			}
+
+			prefixed.ServeHTTP(w, req)
+		})
+	}
+
+	return &http.Server{
+		Handler: handler,
+	}, nil
+}
+
+// initListeners initialises listeners of the server
+func (s *Server) initListeners() error {
+	// listen on public port
+	var err error
+	publicHostPort := s.httpHostPort
+	if publicHostPort == "" {
+		return fmt.Errorf("constants.HTTPHostPort is required")
+	}
+
+	s.httpConn, err = net.Listen("tcp", publicHostPort)
+	if err != nil {
 		return err
 	}
 
+	slog.Info(fmt.Sprintf("Query server started listening on %s...", s.httpHostPort))
+
+	return nil
+}
+
+// Start listening on http and private http port concurrently
+func (s *Server) Start(ctx context.Context) error {
+	err := s.initListeners()
+	if err != nil {
+		return err
+	}
+
+	var httpPort int
+	if port, err := utils.GetPort(s.httpConn.Addr()); err == nil {
+		httpPort = port
+	}
+
+	go func() {
+		slog.Info("Starting HTTP server", "port", httpPort, "addr", s.httpHostPort)
+
+		switch err := s.httpServer.Serve(s.httpConn); err {
+		case nil, http.ErrServerClosed, cmux.ErrListenerClosed:
+			// normal exit, nothing to do
+		default:
+			slog.Error("Could not start HTTP server", errors.Attr(err))
+		}
+		s.unavailableChannel <- healthcheck.Unavailable
+	}()
+
+	go func() {
+		slog.Info("Starting OpAmp Websocket server", "addr", constants.OpAmpWsEndpoint)
+		err := s.opampServer.Start(constants.OpAmpWsEndpoint)
+		if err != nil {
+			slog.Error("opamp ws server failed to start", errors.Attr(err))
+			s.unavailableChannel <- healthcheck.Unavailable
+		}
+	}()
+
 	return nil
 }
 
 func (s *Server) Stop(ctx context.Context) error {
+	if s.httpServer != nil {
+		if err := s.httpServer.Shutdown(context.Background()); err != nil {
+			return err
+		}
+	}
+
 	s.opampServer.Stop()
 
 	return nil

pkg/query-service/constants/constants.go

@@ -10,7 +10,11 @@ import (
 	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
-const OpAmpWsEndpoint = "0.0.0.0:4320" // address for opamp websocket
+const (
+	HTTPHostPort    = "0.0.0.0:8080" // Address to serve http (query service)
+	PrivateHostPort = "0.0.0.0:8085" // Address to server internal services like alert manager
+	OpAmpWsEndpoint = "0.0.0.0:4320" // address for opamp websocket
+)
 
 const MaxAllowedPointsInTimeSeries = 300
 

pkg/query-service/healthcheck/handler.go

@@ -0,0 +1,12 @@
+package healthcheck
+
+const (
+	// Unavailable indicates the service is not able to handle requests
+	Unavailable Status = iota
+	// Ready indicates the service is ready to handle requests
+	Ready
+	// Broken indicates that the healthcheck itself is broken, not serving HTTP
+	Broken
+)
+
+type Status int

pkg/query-service/rules/manager_test.go

@@ -156,7 +156,7 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 			triggeredTestAlerts := []map[*alertmanagertypes.PostableAlert][]string{}
 
 			// Variable to store promProvider for cleanup
-			var promProvider prometheus.Prometheus
+			var promProvider *prometheustest.Provider
 
 			// Create manager using test factory with hooks
 			mgr := NewTestManager(t, &TestManagerOptions{
@@ -181,29 +181,74 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 				TelemetryStoreHook: func(store telemetrystore.TelemetryStore) {
 					mockStore := store.(*telemetrystoretest.Provider)
 
-					// Grid the TestNotification eval computes over (see
-					// Timestamps on base_rule); nil args match any window.
+					// Set up Prometheus-specific mock data
+					// Fingerprint columns for Prometheus queries
+					fingerprintCols := []cmock.ColumnType{
+						{Name: "fingerprint", Type: "UInt64"},
+						{Name: "any(labels)", Type: "String"},
+					}
+
+					// Samples columns for Prometheus queries
+					samplesCols := []cmock.ColumnType{
+						{Name: "metric_name", Type: "String"},
+						{Name: "fingerprint", Type: "UInt64"},
+						{Name: "unix_milli", Type: "Int64"},
+						{Name: "value", Type: "Float64"},
+						{Name: "flags", Type: "UInt32"},
+					}
+
+					// Calculate query time range similar to Prometheus rule tests
+					// TestNotification uses time.Now().UTC() for evaluation
+					// We calculate the query window based on current time to match what the actual evaluation will use
 					evalTime := baseTime
 					evalWindowMs := int64(5 * 60 * 1000) // 5 minutes in ms
-					gridEnd := (evalTime.UnixMilli() / 60000) * 60000
-					gridStart := gridEnd - evalWindowMs
+					evalTimeMs := evalTime.UnixMilli()
+					queryStart := ((evalTimeMs-2*evalWindowMs)/60000)*60000 + 1 // truncate to minute + 1ms
+					queryEnd := (evalTimeMs / 60000) * 60000                    // truncate to minute
 
-					tsList := make([]int64, 0, len(tc.Values))
-					vList := make([]float64, 0, len(tc.Values))
+					// Create fingerprint data
+					fingerprint := uint64(12345)
+					labelsJSON := `{"__name__":"test_metric"}`
+					fingerprintData := [][]any{
+						{fingerprint, labelsJSON},
+					}
+					fingerprintRows := cmock.NewRows(fingerprintCols, fingerprintData)
+
+					// Create samples data from test case values, calculating timestamps relative to baseTime
+					validSamplesData := make([][]any, 0)
 					for _, v := range tc.Values {
 						// Skip NaN and Inf values in the samples data
 						if math.IsNaN(v.Value) || math.IsInf(v.Value, 0) {
 							continue
 						}
-						tsList = append(tsList, baseTime.Add(v.Offset).UnixMilli())
-						vList = append(vList, v.Value)
+						// Calculate timestamp relative to baseTime
+						sampleTimestamp := baseTime.Add(v.Offset).UnixMilli()
+						validSamplesData = append(validSamplesData, []any{
+							"test_metric",
+							fingerprint,
+							sampleTimestamp,
+							v.Value,
+							uint32(0), // flags - 0 means normal value
+						})
 					}
-					grid := prometheustest.LastSampleGrid(tsList, vList, gridStart, gridEnd, 60_000, 300_000)
+					samplesRows := cmock.NewRows(samplesCols, validSamplesData)
 
 					mock := mockStore.Mock()
-					mock.ExpectQuery("SELECT gkey").
-						WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-						WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{{`[["__name__","test_metric"]]`, grid}}))
+
+					// Mock the fingerprint query (for Prometheus label matching)
+					mock.ExpectQuery("SELECT fingerprint, any").
+						WithArgs("test_metric").
+						WillReturnRows(fingerprintRows)
+
+					// Mock the samples query (for Prometheus metric data)
+					mock.ExpectQuery("SELECT metric_name, fingerprint, unix_milli").
+						WithArgs(
+							"test_metric",
+							"test_metric",
+							queryStart,
+							queryEnd,
+						).
+						WillReturnRows(samplesRows)
 
 					// Create Prometheus provider for this test
 					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, store)
@@ -237,6 +282,7 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 				assert.Empty(t, triggeredTestAlerts)
 			}
 
+			promProvider.Close()
 		})
 	}
 }

pkg/query-service/rules/managertestfactory.go

@@ -131,7 +131,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
 	meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
 	require.NoError(t, err)
 	bucketCache := querier.NewBucketCache(providerSettings, cache, 0, 0)
-	providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger)
+	providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, nil, metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger)
 	mockQuerier, err := providerFactory.New(context.Background(), providerSettings, querier.Config{})
 	require.NoError(t, err)
 

pkg/query-service/rules/prom_rule.go

@@ -376,11 +376,17 @@ func (r *PromRule) String() string {
 }
 
 func (r *PromRule) RunAlertQuery(ctx context.Context, qs string, start, end time.Time, interval time.Duration) (promql.Matrix, error) {
-	res, err := r.prometheus.QueryRange(ctx, qs, start, end, interval)
+	q, err := r.prometheus.Engine().NewRangeQuery(ctx, r.prometheus.Storage(), nil, qs, start, end, interval)
 	if err != nil {
 		return nil, err
 	}
 
+	res := q.Exec(ctx)
+
+	if res.Err != nil {
+		return nil, res.Err
+	}
+
 	switch typ := res.Value.(type) {
 	case promql.Vector:
 		series := make([]promql.Series, 0, len(typ))

pkg/query-service/rules/prom_rule_test.go

@@ -9,8 +9,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	cmock "github.com/SigNoz/clickhouse-go-mock"
 	pql "github.com/prometheus/prometheus/promql"
+	cmock "github.com/SigNoz/clickhouse-go-mock"
 
 	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/prometheus"
@@ -768,11 +768,26 @@ func TestPromRuleUnitCombinations(t *testing.T) {
 		},
 	}
 
+	// time_series_v4 cols of interest
+	fingerprintCols := []cmock.ColumnType{
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "any(labels)", Type: "String"},
+	}
+
+	// samples_v4 columns
+	samplesCols := []cmock.ColumnType{
+		{Name: "metric_name", Type: "String"},
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "unix_milli", Type: "Int64"},
+		{Name: "value", Type: "Float64"},
+		{Name: "flags", Type: "UInt32"},
+	}
+
 	// see Timestamps on base_rule
 	evalWindowMs := int64(5 * 60 * 1000) // 5 minutes in ms
 	evalTimeMs := evalTime.UnixMilli()
-	queryEnd := (evalTimeMs / 60000) * 60000 // truncate to minute
-	gridStart := queryEnd - evalWindowMs
+	queryStart := ((evalTimeMs-2*evalWindowMs)/60000)*60000 + 1 // truncate to minute + 1ms
+	queryEnd := (evalTimeMs / 60000) * 60000                    // truncate to minute
 
 	cases := []struct {
 		targetUnit string
@@ -889,19 +904,43 @@ func TestPromRuleUnitCombinations(t *testing.T) {
 	for idx, c := range cases {
 		telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{})
 
-		tsList := make([]int64, len(c.values))
-		vList := make([]float64, len(c.values))
-		for i, v := range c.values {
-			tsList[i] = v.timestamp.UnixMilli()
-			vList[i] = v.value
+		// single fingerprint with labels JSON
+		fingerprint := uint64(12345)
+		labelsJSON := `{"__name__":"test_metric"}`
+		fingerprintData := [][]any{
+			{fingerprint, labelsJSON},
 		}
-		grid := prometheustest.LastSampleGrid(tsList, vList, gridStart, queryEnd, 60_000, 300_000)
+		fingerprintRows := cmock.NewRows(fingerprintCols, fingerprintData)
 
-		// args: $1-$3=group-key join conditions, $4-$6=samples conditions
+		// create samples data from test case values
+		samplesData := make([][]any, len(c.values))
+		for i, v := range c.values {
+			samplesData[i] = []any{
+				"test_metric",
+				fingerprint,
+				v.timestamp.UnixMilli(),
+				v.value,
+				uint32(0), // flags - 0 means normal value, 1 means stale, we are not doing staleness tests
+			}
+		}
+		samplesRows := cmock.NewRows(samplesCols, samplesData)
+
+		// args: $1=metric_name (the __name__ matcher maps onto the column)
 		telemetryStore.Mock().
-			ExpectQuery("SELECT gkey").
-			WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-			WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{{`[["__name__","test_metric"]]`, grid}}))
+			ExpectQuery("SELECT fingerprint, any").
+			WithArgs("test_metric").
+			WillReturnRows(fingerprintRows)
+
+		// args: $1=metric_name IN (discovered names), $2=metric_name (subquery), $3=start, $4=end
+		telemetryStore.Mock().
+			ExpectQuery("SELECT metric_name, fingerprint, unix_milli").
+			WithArgs(
+				"test_metric",
+				"test_metric",
+				queryStart,
+				queryEnd,
+			).
+			WillReturnRows(samplesRows)
 
 		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
@@ -931,12 +970,14 @@ func TestPromRuleUnitCombinations(t *testing.T) {
 		rule, err := NewPromRule("69", valuer.GenerateUUID(), &postableRule, logger, promProvider, externalUrl)
 		if err != nil {
 			assert.NoError(t, err)
+			promProvider.Close()
 			continue
 		}
 
 		alertsFound, err := rule.Eval(context.Background(), evalTime)
 		if err != nil {
 			assert.NoError(t, err)
+			promProvider.Close()
 			continue
 		}
 
@@ -954,6 +995,7 @@ func TestPromRuleUnitCombinations(t *testing.T) {
 			assert.Equal(t, c.expectAlerts, foundCount, "case %d", idx)
 		}
 
+		promProvider.Close()
 	}
 }
 
@@ -985,6 +1027,12 @@ func TestPromRuleNoData(t *testing.T) {
 		},
 	}
 
+	// time_series_v4 cols of interest
+	fingerprintCols := []cmock.ColumnType{
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "any(labels)", Type: "String"},
+	}
+
 	cases := []struct {
 		values []struct {
 			timestamp time.Time
@@ -1006,11 +1054,15 @@ func TestPromRuleNoData(t *testing.T) {
 	for idx, c := range cases {
 		telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{})
 
+		// no data
+		fingerprintData := [][]any{}
+		fingerprintRows := cmock.NewRows(fingerprintCols, fingerprintData)
+
 		// no rows == no data
 		telemetryStore.Mock().
-			ExpectQuery("SELECT gkey").
-			WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-			WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{}))
+			ExpectQuery("SELECT fingerprint, any").
+			WithArgs("test_metric").
+			WillReturnRows(fingerprintRows)
 
 		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
@@ -1035,12 +1087,14 @@ func TestPromRuleNoData(t *testing.T) {
 		rule, err := NewPromRule("69", valuer.GenerateUUID(), &postableRule, logger, promProvider, externalUrl)
 		if err != nil {
 			assert.NoError(t, err)
+			promProvider.Close()
 			continue
 		}
 
 		alertsFound, err := rule.Eval(context.Background(), evalTime)
 		if err != nil {
 			assert.NoError(t, err)
+			promProvider.Close()
 			continue
 		}
 
@@ -1053,6 +1107,7 @@ func TestPromRuleNoData(t *testing.T) {
 			}
 		}
 
+		promProvider.Close()
 	}
 }
 
@@ -1084,11 +1139,24 @@ func TestMultipleThresholdPromRule(t *testing.T) {
 		},
 	}
 
+	fingerprintCols := []cmock.ColumnType{
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "any(labels)", Type: "String"},
+	}
+
+	samplesCols := []cmock.ColumnType{
+		{Name: "metric_name", Type: "String"},
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "unix_milli", Type: "Int64"},
+		{Name: "value", Type: "Float64"},
+		{Name: "flags", Type: "UInt32"},
+	}
+
 	// see .Timestamps of base rule
 	evalWindowMs := int64(5 * 60 * 1000)
 	evalTimeMs := evalTime.UnixMilli()
+	queryStart := ((evalTimeMs-2*evalWindowMs)/60000)*60000 + 1
 	queryEnd := (evalTimeMs / 60000) * 60000
-	gridStart := queryEnd - evalWindowMs
 
 	cases := []struct {
 		targetUnit string
@@ -1182,19 +1250,39 @@ func TestMultipleThresholdPromRule(t *testing.T) {
 	for idx, c := range cases {
 		telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{})
 
-		tsList := make([]int64, len(c.values))
-		vList := make([]float64, len(c.values))
-		for i, v := range c.values {
-			tsList[i] = v.timestamp.UnixMilli()
-			vList[i] = v.value
+		fingerprint := uint64(12345)
+		labelsJSON := `{"__name__":"test_metric"}`
+		fingerprintData := [][]any{
+			{fingerprint, labelsJSON},
 		}
-		grid := prometheustest.LastSampleGrid(tsList, vList, gridStart, queryEnd, 60_000, 300_000)
+		fingerprintRows := cmock.NewRows(fingerprintCols, fingerprintData)
+
+		samplesData := make([][]any, len(c.values))
+		for i, v := range c.values {
+			samplesData[i] = []any{
+				"test_metric",
+				fingerprint,
+				v.timestamp.UnixMilli(),
+				v.value,
+				uint32(0),
+			}
+		}
+		samplesRows := cmock.NewRows(samplesCols, samplesData)
 
-		// args: $1-$3=group-key join conditions, $4-$6=samples conditions
 		telemetryStore.Mock().
-			ExpectQuery("SELECT gkey").
-			WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-			WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{{`[["__name__","test_metric"]]`, grid}}))
+			ExpectQuery("SELECT fingerprint, any").
+			WithArgs("test_metric").
+			WillReturnRows(fingerprintRows)
+
+		telemetryStore.Mock().
+			ExpectQuery("SELECT metric_name, fingerprint, unix_milli").
+			WithArgs(
+				"test_metric",
+				"test_metric",
+				queryStart,
+				queryEnd,
+			).
+			WillReturnRows(samplesRows)
 
 		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
@@ -1231,12 +1319,14 @@ func TestMultipleThresholdPromRule(t *testing.T) {
 		rule, err := NewPromRule("69", valuer.GenerateUUID(), &postableRule, logger, promProvider, externalUrl)
 		if err != nil {
 			assert.NoError(t, err)
+			promProvider.Close()
 			continue
 		}
 
 		alertsFound, err := rule.Eval(context.Background(), evalTime)
 		if err != nil {
 			assert.NoError(t, err)
+			promProvider.Close()
 			continue
 		}
 
@@ -1254,6 +1344,7 @@ func TestMultipleThresholdPromRule(t *testing.T) {
 			assert.Equal(t, c.expectAlerts, foundCount, "case %d", idx)
 		}
 
+		promProvider.Close()
 	}
 }
 
@@ -1287,6 +1378,27 @@ func TestPromRule_NoData(t *testing.T) {
 		},
 	}
 
+	// time_series_v4 cols of interest
+	fingerprintCols := []cmock.ColumnType{
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "any(labels)", Type: "String"},
+	}
+
+	// samples_v4 columns
+	samplesCols := []cmock.ColumnType{
+		{Name: "metric_name", Type: "String"},
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "unix_milli", Type: "Int64"},
+		{Name: "value", Type: "Float64"},
+		{Name: "flags", Type: "UInt32"},
+	}
+
+	// see Timestamps on base_rule
+	evalWindowMs := int64(5 * 60 * 1000) // 5 minutes in ms
+	evalTimeMs := evalTime.UnixMilli()
+	queryStart := ((evalTimeMs-2*evalWindowMs)/60000)*60000 + 1 // truncate to minute + 1ms
+	queryEnd := (evalTimeMs / 60000) * 60000                    // truncate to minute
+
 	cases := []struct {
 		description   string
 		alertOnAbsent bool
@@ -1318,11 +1430,18 @@ func TestPromRule_NoData(t *testing.T) {
 
 			telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{})
 
-			// no rows == no data
+			// single fingerprint with labels JSON
+			fingerprint := uint64(12345)
+			labelsJSON := `{"__name__":"test_metric"}`
 			telemetryStore.Mock().
-				ExpectQuery("SELECT gkey").
-				WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-				WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{}))
+				ExpectQuery("SELECT fingerprint, any").
+				WithArgs("test_metric").
+				WillReturnRows(cmock.NewRows(fingerprintCols, [][]any{{fingerprint, labelsJSON}}))
+
+			telemetryStore.Mock().
+				ExpectQuery("SELECT metric_name, fingerprint, unix_milli").
+				WithArgs("test_metric", "test_metric", queryStart, queryEnd).
+				WillReturnRows(cmock.NewRows(samplesCols, [][]any{}))
 
 			promProvider := prometheustest.New(
 				context.Background(),
@@ -1331,6 +1450,7 @@ func TestPromRule_NoData(t *testing.T) {
 				telemetryStore,
 			)
 			defer func() {
+				_ = promProvider.Close()
 			}()
 
 			externalUrl := mustParseURL(t, "http://localhost:8080")
@@ -1390,6 +1510,19 @@ func TestPromRule_NoData_AbsentFor(t *testing.T) {
 		},
 	}
 
+	fingerprintCols := []cmock.ColumnType{
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "any(labels)", Type: "String"},
+	}
+
+	samplesCols := []cmock.ColumnType{
+		{Name: "metric_name", Type: "String"},
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "unix_milli", Type: "Int64"},
+		{Name: "value", Type: "Float64"},
+		{Name: "flags", Type: "UInt32"},
+	}
+
 	cases := []struct {
 		description        string
 		absentFor          uint64        // grace period in minutes
@@ -1423,30 +1556,43 @@ func TestPromRule_NoData_AbsentFor(t *testing.T) {
 
 			telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{})
 
-			// Grid an eval at this time evaluates over (see Timestamps on
-			// base_rule).
-			calcGrid := func(evalTime time.Time) (int64, int64) {
-				gridEnd := (evalTime.UnixMilli() / 60000) * 60000
-				return gridEnd - evalWindow.Milliseconds(), gridEnd
+			fingerprint := uint64(12345)
+			labelsJSON := `{"__name__":"test_metric"}`
+
+			// Helper to calculate query time range for an eval time
+			calcQueryRange := func(evalTime time.Time) (int64, int64) {
+				evalTimeMs := evalTime.UnixMilli()
+				queryStart := ((evalTimeMs-2*evalWindow.Milliseconds())/60000)*60000 + 1
+				queryEnd := (evalTimeMs / 60000) * 60000
+				return queryStart, queryEnd
 			}
 
-			// First eval (t1) - with data: points in the past relative to t1
-			gridStart1, gridEnd1 := calcGrid(t1)
-			grid1 := prometheustest.LastSampleGrid(
-				[]int64{baseTime.UnixMilli(), baseTime.Add(1 * time.Minute).UnixMilli(), baseTime.Add(2 * time.Minute).UnixMilli()},
-				[]float64{100, 100, 100},
-				gridStart1, gridEnd1, 60_000, 300_000,
-			)
+			// First eval (t1) - with data
+			queryStart1, queryEnd1 := calcQueryRange(t1)
 			telemetryStore.Mock().
-				ExpectQuery("SELECT gkey").
-				WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-				WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{{`[["__name__","test_metric"]]`, grid1}}))
+				ExpectQuery("SELECT fingerprint, any").
+				WithArgs("test_metric").
+				WillReturnRows(cmock.NewRows(fingerprintCols, [][]any{{fingerprint, labelsJSON}}))
+			telemetryStore.Mock().
+				ExpectQuery("SELECT metric_name, fingerprint, unix_milli").
+				WithArgs("test_metric", "test_metric", queryStart1, queryEnd1).
+				WillReturnRows(cmock.NewRows(samplesCols, [][]any{
+					// Data points in the past relative to t1
+					{"test_metric", fingerprint, baseTime.UnixMilli(), 100.0, uint32(0)},
+					{"test_metric", fingerprint, baseTime.Add(1 * time.Minute).UnixMilli(), 100.0, uint32(0)},
+					{"test_metric", fingerprint, baseTime.Add(2 * time.Minute).UnixMilli(), 100.0, uint32(0)},
+				}))
 
 			// Second eval (t2) - no data
+			queryStart2, queryEnd2 := calcQueryRange(t2)
 			telemetryStore.Mock().
-				ExpectQuery("SELECT gkey").
-				WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-				WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{}))
+				ExpectQuery("SELECT fingerprint, any").
+				WithArgs("test_metric").
+				WillReturnRows(cmock.NewRows(fingerprintCols, [][]any{{fingerprint, labelsJSON}}))
+			telemetryStore.Mock().
+				ExpectQuery("SELECT metric_name, fingerprint, unix_milli").
+				WithArgs("test_metric", "test_metric", queryStart2, queryEnd2).
+				WillReturnRows(cmock.NewRows(samplesCols, [][]any{})) // empty - no data
 
 			promProvider := prometheustest.New(
 				context.Background(),
@@ -1455,6 +1601,7 @@ func TestPromRule_NoData_AbsentFor(t *testing.T) {
 				telemetryStore,
 			)
 			defer func() {
+				_ = promProvider.Close()
 			}()
 
 			externalUrl := mustParseURL(t, "http://localhost:8080")
@@ -1505,15 +1652,32 @@ func TestPromRuleEval_RequireMinPoints(t *testing.T) {
 		},
 	}
 
-	sampleTs := []int64{baseTime.UnixMilli(), baseTime.Add(time.Minute).UnixMilli(), baseTime.Add(2 * time.Minute).UnixMilli()}
-	sampleVs := []float64{100, 150, 250}
+	fingerprintCols := []cmock.ColumnType{
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "any(labels)", Type: "String"},
+	}
+	fingerprint := uint64(12345)
+	fingerprintData := [][]any{{fingerprint, `{"__name__":"test_metric"}`}}
+
+	samplesCols := []cmock.ColumnType{
+		{Name: "metric_name", Type: "String"},
+		{Name: "fingerprint", Type: "UInt64"},
+		{Name: "unix_milli", Type: "Int64"},
+		{Name: "value", Type: "Float64"},
+		{Name: "flags", Type: "UInt32"},
+	}
+	samplesData := [][]any{
+		{"test_metric", fingerprint, baseTime.UnixMilli(), 100.0, 0},
+		{"test_metric", fingerprint, baseTime.Add(time.Minute).UnixMilli(), 150.0, 0},
+		{"test_metric", fingerprint, baseTime.Add(2 * time.Minute).UnixMilli(), 250.0, 0},
+	}
 	targetForAlert := 200.0
 	targetForNoAlert := 500.0
 
 	// see Timestamps on base_rule
 	evalTimeMs := evalTime.UnixMilli()
-	queryEnd := (evalTimeMs / 60000) * 60000 // truncate to minute
-	gridStart := queryEnd - evalWindow.Milliseconds()
+	queryStart := ((evalTimeMs-evalWindow.Milliseconds()-lookBackDelta.Milliseconds())/60000)*60000 + 1 // truncate to minute + 1ms
+	queryEnd := (evalTimeMs / 60000) * 60000                                                            // truncate to minute
 
 	cases := []struct {
 		description       string
@@ -1582,11 +1746,14 @@ func TestPromRuleEval_RequireMinPoints(t *testing.T) {
 
 		t.Run(c.description, func(t *testing.T) {
 			telemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &queryMatcherAny{})
-			grid := prometheustest.LastSampleGrid(sampleTs, sampleVs, gridStart, queryEnd, 60_000, lookBackDelta.Milliseconds())
 			telemetryStore.Mock().
-				ExpectQuery("SELECT gkey").
-				WithArgs("test_metric", nil, nil, "test_metric", nil, nil).
-				WillReturnRows(cmock.NewRows(prometheustest.GridCols, [][]any{{`[["__name__","test_metric"]]`, grid}}))
+				ExpectQuery("SELECT fingerprint, any").
+				WithArgs("test_metric").
+				WillReturnRows(cmock.NewRows(fingerprintCols, fingerprintData))
+			telemetryStore.Mock().
+				ExpectQuery("SELECT metric_name, fingerprint, unix_milli").
+				WithArgs("test_metric", "test_metric", queryStart, queryEnd).
+				WillReturnRows(cmock.NewRows(samplesCols, samplesData))
 			promProvider := prometheustest.New(
 				context.Background(),
 				instrumentationtest.New().ToProviderSettings(),
@@ -1594,6 +1761,7 @@ func TestPromRuleEval_RequireMinPoints(t *testing.T) {
 				telemetryStore,
 			)
 			defer func() {
+				_ = promProvider.Close()
 			}()
 
 			externalUrl := mustParseURL(t, "http://localhost:8080")

pkg/query-service/rules/setups_test.go

@@ -40,6 +40,7 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
+		nil, // promV2
 		nil, // traceStmtBuilder
 		nil, // aiTraceStmtBuilder
 		nil, // logStmtBuilder
@@ -75,6 +76,7 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
+		nil, // promV2
 		nil, // traceStmtBuilder
 		nil, // aiTraceStmtBuilder
 		logStmtBuilder,
@@ -111,6 +113,7 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
+		nil, // promV2
 		traceStmtBuilder,
 		nil, // aiTraceStmtBuilder
 		nil, // logStmtBuilder

pkg/signoz/openapi.go

@@ -10,14 +10,12 @@ import (
 	"github.com/SigNoz/signoz/pkg/alertmanager"
 	"github.com/SigNoz/signoz/pkg/apiserver"
 	"github.com/SigNoz/signoz/pkg/apiserver/signozapiserver"
-	"github.com/SigNoz/signoz/pkg/auditor"
 	"github.com/SigNoz/signoz/pkg/authz"
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/gateway"
 	"github.com/SigNoz/signoz/pkg/global"
 	"github.com/SigNoz/signoz/pkg/http/handler"
-	"github.com/SigNoz/signoz/pkg/identn"
 	"github.com/SigNoz/signoz/pkg/instrumentation"
 	"github.com/SigNoz/signoz/pkg/licensing"
 	"github.com/SigNoz/signoz/pkg/modules/aiobservability"
@@ -44,11 +42,9 @@ import (
 	"github.com/SigNoz/signoz/pkg/prometheus"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/ruler"
-	"github.com/SigNoz/signoz/pkg/sharder"
 	"github.com/SigNoz/signoz/pkg/statsreporter"
 	"github.com/SigNoz/signoz/pkg/subscription"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/web"
 	"github.com/SigNoz/signoz/pkg/zeus"
 	"github.com/swaggest/jsonschema-go"
 	"github.com/swaggest/openapi-go"
@@ -105,11 +101,6 @@ func NewOpenAPI(ctx context.Context, instrumentation instrumentation.Instrumenta
 		struct{ ruler.Handler }{},
 		struct{ statsreporter.Handler }{},
 		struct{ savedview.Handler }{},
-		global.Config{},
-		struct{ identn.IdentNResolver }{},
-		struct{ sharder.Sharder }{},
-		struct{ auditor.Auditor }{},
-		struct{ web.Web }{},
 		struct{ quickfilter.Module }{},
 		struct{ quickfilter.Handler }{},
 	).New(ctx, instrumentation.ToProviderSettings(), apiserver.Config{})

pkg/signoz/provider.go

@@ -45,6 +45,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/pprof/httppprof"
 	"github.com/SigNoz/signoz/pkg/pprof/nooppprof"
 	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheus"
 	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/querier/signozquerier"
@@ -269,6 +270,7 @@ func NewTelemetryStoreProviderFactories() factory.NamedMap[factory.ProviderFacto
 
 func NewPrometheusProviderFactories(telemetryStore telemetrystore.TelemetryStore) factory.NamedMap[factory.ProviderFactory[prometheus.Prometheus, prometheus.Config]] {
 	return factory.MustNewNamedMap(
+		clickhouseprometheus.NewFactory(telemetryStore),
 		clickhouseprometheusv2.NewFactory(telemetryStore),
 	)
 }
@@ -311,13 +313,13 @@ func NewStatsReporterProviderFactories(aggregator statsreporter.Aggregator, orgG
 	)
 }
 
-func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
+func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, promV2 prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], aiTraceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
 	return factory.MustNewNamedMap(
-		signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
+		signozquerier.NewFactory(telemetryStore, prometheus, promV2, metadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
 	)
 }
 
-func NewAPIServerProviderFactories(orgGetter organization.Getter, authz authz.AuthZ, modules Modules, handlers Handlers, globalConfig global.Config, gatewayService gateway.Gateway, identNResolver identn.IdentNResolver, sharder sharder.Sharder, auditor auditor.Auditor, web web.Web) factory.NamedMap[factory.ProviderFactory[apiserver.APIServer, apiserver.Config]] {
+func NewAPIServerProviderFactories(orgGetter organization.Getter, authz authz.AuthZ, modules Modules, handlers Handlers, globalConfig global.Config, gatewayService gateway.Gateway) factory.NamedMap[factory.ProviderFactory[apiserver.APIServer, apiserver.Config]] {
 	return factory.MustNewNamedMap(
 		signozapiserver.NewFactory(
 			orgGetter,
@@ -359,11 +361,6 @@ func NewAPIServerProviderFactories(orgGetter organization.Getter, authz authz.Au
 			handlers.RulerHandler,
 			handlers.StatsHandler,
 			handlers.SavedView,
-			globalConfig,
-			identNResolver,
-			sharder,
-			auditor,
-			web,
 			modules.QuickFilter,
 			handlers.QuickFilter,
 		),

pkg/signoz/provider_test.go

@@ -102,10 +102,6 @@ func TestNewProviderFactories(t *testing.T) {
 			Handlers{},
 			global.Config{},
 			nil,
-			nil,
-			nil,
-			nil,
-			nil,
 		)
 	})
 }

pkg/signoz/signoz.go

@@ -40,6 +40,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/modules/tag/impltag"
 	"github.com/SigNoz/signoz/pkg/modules/user/impluser"
 	"github.com/SigNoz/signoz/pkg/prometheus"
+	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/queryparser"
 	"github.com/SigNoz/signoz/pkg/ruler"
@@ -308,6 +309,11 @@ func New(
 
 	retentionGetter := implretention.NewGetter(implretention.NewStore(sqlstore))
 
+	// promV2 is the clickhousev2 provider handed to the querier for shadow
+	// comparison and pinned serving (declared before the serving provider,
+	// whose variable shadows the package name below).
+	var promV2 prometheus.Prometheus
+
 	// Initialize prometheus from the available prometheus provider factories
 	prometheus, err := factory.NewProviderFromNamedMap(
 		ctx,
@@ -320,6 +326,23 @@ func New(
 		return nil, err
 	}
 
+	// With the default provider, also stand up the clickhousev2 provider for
+	// the querier: PromQL queries shadow-compare against it behind the
+	// use_prometheus_clickhouse_v2 flag (see pkg/querier/promql_shadow.go).
+	// It never serves by default. An explicit
+	// prometheus::provider: clickhousev2 makes v2 the serving provider
+	// outright, so there is nothing to compare against.
+	if config.Prometheus.Provider() == "clickhouse" {
+		v2Config := config.Prometheus
+		// The v2 engine only evaluates shadow and pinned queries; disable its
+		// active query tracker so two trackers never share a file.
+		v2Config.ActiveQueryTrackerConfig.Enabled = false
+		promV2, err = clickhouseprometheusv2.New(ctx, providerSettings, v2Config, telemetrystore)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Assemble the query stack (metadata store, statement builders, bucket cache) once,
 	// and reuse the single metadata store everywhere downstream.
 	telemetryMetadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, err := newQueryStack(ctx, providerSettings, config, telemetrystore, cache, flagger)
@@ -332,7 +355,7 @@ func New(
 		ctx,
 		providerSettings,
 		config.Querier,
-		NewQuerierProviderFactories(telemetrystore, prometheus, telemetryMetadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
+		NewQuerierProviderFactories(telemetrystore, prometheus, promV2, telemetryMetadataStore, traceStmtBuilder, aiTraceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
 		config.Querier.Provider(),
 	)
 	if err != nil {
@@ -612,20 +635,13 @@ func New(
 		ctx,
 		providerSettings,
 		config.APIServer,
-		NewAPIServerProviderFactories(orgGetter, authz, modules, handlers, config.Global, gateway, identNResolver, sharder, auditor, web),
+		NewAPIServerProviderFactories(orgGetter, authz, modules, handlers, config.Global, gateway),
 		"signoz",
 	)
 	if err != nil {
 		return nil, err
 	}
 
-	// Register the API server with the registry so its lifecycle is managed
-	// alongside the other services and it shows up in the health endpoint.
-	err = registry.Add(ctx, factory.NewNamedService(factory.MustNewName("apiserver"), apiserverInstance))
-	if err != nil {
-		return nil, err
-	}
-
 	return &SigNoz{
 		Registry:               registry,
 		Analytics:              analytics,

pkg/statementbuilder/meterstatementbuilder/statement_builder.go

@@ -129,7 +129,7 @@ func (b *meterQueryStatementBuilder) buildPipelineStatement(
 	}
 
 	// final SELECT
-	return b.metricsStatementBuilder.BuildFinalSelect(cteFragments, cteArgs, query)
+	return b.metricsStatementBuilder.BuildFinalSelect(cteFragments, cteArgs, qbtypes.RequestTypeTimeSeries, query)
 }
 
 func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(

pkg/statementbuilder/metricsstatementbuilder/reduced_test.go

@@ -122,7 +122,7 @@ func TestReducedStatementBuilder(t *testing.T) {
 			name:  "histogram_p99",
 			query: reducedQuery("test.metric.bucket", metrictypes.HistogramType, metrictypes.Cumulative, metrictypes.TimeAggregationUnspecified, metrictypes.SpaceAggregationPercentile99),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(`sum`) / 300 AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points FINAL INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(`sum`) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points FINAL INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
 				Args:  []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746997200000), uint64(1747172760000), "test.metric.bucket", uint64(1746999900000), uint64(1747172760000)},
 			},
 		},
@@ -130,7 +130,7 @@ func TestReducedStatementBuilder(t *testing.T) {
 			name:  "histogram_p99_group_by",
 			query: withGroupBy(reducedQuery("test.metric.bucket", metrictypes.HistogramType, metrictypes.Cumulative, metrictypes.TimeAggregationUnspecified, metrictypes.SpaceAggregationPercentile99), "service.name"),
 			expected: qbtypes.Statement{
-				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(`sum`) / 300 AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points FINAL INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
+				Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts) UNION ALL SELECT * FROM (WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(`sum`) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points FINAL INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts SETTINGS do_not_merge_across_partitions_select_final = 1, optimize_move_to_prewhere_if_final = 1",
 				Args:  []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746997200000), uint64(1747172760000), "test.metric.bucket", uint64(1746999900000), uint64(1747172760000)},
 			},
 		},

pkg/statementbuilder/metricsstatementbuilder/statement_builder.go

@@ -4,9 +4,13 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"math"
 	"slices"
+	"strconv"
+	"strings"
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -113,7 +117,7 @@ func (b *StatementBuilder) Build(
 	orgID valuer.UUID,
 	start uint64,
 	end uint64,
-	_ qbtypes.RequestType,
+	requestType qbtypes.RequestType,
 	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
 	variables map[string]qbtypes.VariableItem,
 ) (*qbtypes.Statement, error) {
@@ -125,13 +129,14 @@ func (b *StatementBuilder) Build(
 
 	start, end = querybuilder.AdjustedMetricTimeRange(start, end, uint64(query.StepInterval.Seconds()), query)
 
-	return b.buildPipelineStatement(ctx, orgID, start, end, query, keys, variables)
+	return b.buildPipelineStatement(ctx, orgID, start, end, requestType, query, keys, variables)
 }
 
 func (b *StatementBuilder) buildPipelineStatement(
 	ctx context.Context,
 	orgID valuer.UUID,
 	start, end uint64,
+	requestType qbtypes.RequestType,
 	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
 	keys map[string][]*telemetrytypes.TelemetryFieldKey,
 	variables map[string]qbtypes.VariableItem,
@@ -144,7 +149,7 @@ func (b *StatementBuilder) buildPipelineStatement(
 	cteQuery := query
 	if query.Aggregations[0].Type == metrictypes.HistogramType {
 		query.GroupBy = slices.DeleteFunc(slices.Clone(query.GroupBy), isHistogramBucket)
-		cteQuery = histogramCTEQuery(query)
+		cteQuery = rewriteQueryForHistogramCTE(query)
 	}
 
 	agg := cteQuery.Aggregations[0]
@@ -216,7 +221,7 @@ func (b *StatementBuilder) buildPipelineStatement(
 		}
 	}
 
-	mainStmt, err := b.BuildFinalSelect(cteFragments, cteArgs, query)
+	mainStmt, err := b.BuildFinalSelect(cteFragments, cteArgs, requestType, query)
 	if err != nil {
 		return nil, err
 	}
@@ -224,13 +229,27 @@ func (b *StatementBuilder) buildPipelineStatement(
 	if reducedFragments == nil {
 		return mainStmt, nil
 	}
-	reducedStmt, err := b.BuildFinalSelect(reducedFragments, reducedArgs, query)
+	reducedStmt, err := b.BuildFinalSelect(reducedFragments, reducedArgs, requestType, query)
 	if err != nil {
 		return nil, err
 	}
 	return unionStatements(mainStmt, reducedStmt, query)
 }
 
+func rewriteQueryForHistogramCTE(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation] {
+	query.GroupBy = append(slices.Clone(query.GroupBy), qbtypes.GroupByKey{
+		TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: histogramBucketKey},
+	})
+
+	// The CTE yields one observation count per `le` per step. The space
+	// aggregation the caller asked for is applied downstream, over the `le` array.
+	query.Aggregations = slices.Clone(query.Aggregations)
+	query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationIncrease
+	query.Aggregations[0].SpaceAggregation = metrictypes.SpaceAggregationSum
+
+	return query
+}
+
 func unionStatements(main, reduced *qbtypes.Statement, query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) (*qbtypes.Statement, error) {
 	orderBy := "ts"
 	for i, g := range query.GroupBy {
@@ -758,11 +777,9 @@ func (b *StatementBuilder) buildSpatialAggregationCTE(
 func (b *StatementBuilder) BuildFinalSelect(
 	cteFragments []string,
 	cteArgs [][]any,
+	requestType qbtypes.RequestType,
 	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
 ) (*qbtypes.Statement, error) {
-	metricType := query.Aggregations[0].Type
-	spaceAgg := query.Aggregations[0].SpaceAggregation
-
 	combined := querybuilder.CombineCTEs(cteFragments)
 
 	var args []any
@@ -770,6 +787,22 @@ func (b *StatementBuilder) BuildFinalSelect(
 		args = append(args, a...)
 	}
 
+	if requestType == qbtypes.RequestTypeHeatmap {
+		return buildHeatmapFinalSelect(combined, args, query)
+	}
+	return buildAggregationFinalSelect(combined, args, query)
+}
+
+// buildAggregationFinalSelect reads __spatial_aggregation_cte as one value per
+// (group, timestamp), which is what every request type but heatmap wants.
+func buildAggregationFinalSelect(
+	combined string,
+	args []any,
+	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
+) (*qbtypes.Statement, error) {
+	metricType := query.Aggregations[0].Type
+	spaceAgg := query.Aggregations[0].SpaceAggregation
+
 	sb := sqlbuilder.NewSelectBuilder()
 
 	if metricType == metrictypes.HistogramType && spaceAgg.IsPercentile() {
@@ -842,24 +875,150 @@ func (b *StatementBuilder) BuildFinalSelect(
 	return &qbtypes.Statement{Query: combined + q, Args: append(args, a...)}, nil
 }
 
-const histogramBucketKey = "le"
+const (
+	histogramBucketKey = "le"
+
+	heatmapValueAlias = "__result_0"
+	heatmapWindow     = "__heatmap_window"
+)
 
 func isHistogramBucket(k qbtypes.GroupByKey) bool { return k.Name == histogramBucketKey }
 
-func histogramCTEQuery(query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation] {
-	query.GroupBy = append(slices.Clone(query.GroupBy), qbtypes.GroupByKey{
-		TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: histogramBucketKey},
-	})
-
-	query.Aggregations = slices.Clone(query.Aggregations)
-	if query.Aggregations[0].SpaceAggregation.IsPercentile() {
-		query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationRate
-	} else {
-		query.Aggregations[0].TimeAggregation = metrictypes.TimeAggregationIncrease
+// buildHeatmapFinalSelect turns __spatial_aggregation_cte into one row per
+// heatmap cell: (ts, group labels..., bucket upper bound, count).
+func buildHeatmapFinalSelect(
+	combined string,
+	args []any,
+	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
+) (*qbtypes.Statement, error) {
+	if query.Aggregations[0].Type == metrictypes.HistogramType {
+		return buildHistogramHeatmapFinalSelect(combined, args, query)
 	}
-	query.Aggregations[0].SpaceAggregation = metrictypes.SpaceAggregationSum
+	return buildValueHeatmapFinalSelect(combined, args, query)
+}
 
-	return query
+// buildHistogramHeatmapFinalSelect differences the cumulative per-`le` counts in
+// __spatial_aggregation_cte into a count per bucket. A bucket runs from the `le`
+// below it up to its own, so the `le=+Inf` row reaches the reader as the
+// overflow and the lowest `le` as a bucket open below, which is where a
+// negative observation would have been counted.
+func buildHistogramHeatmapFinalSelect(
+	combined string,
+	args []any,
+	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
+) (*qbtypes.Statement, error) {
+	groupAliases := GroupByAliases(query.GroupBy)
+	partitionBy := append(append([]string{}, groupAliases...), "ts")
+
+	sb := sqlbuilder.NewSelectBuilder()
+	sb.Select("ts")
+	sb.SelectMore(groupAliases...)
+	sb.SelectMore(fmt.Sprintf(
+		"lagInFrame(toFloat64(%s), 1, toFloat64('-Inf')) OVER %s AS %s",
+		histogramBucketKey, heatmapWindow, qbtypes.HeatmapBucketMinColumn,
+	))
+	sb.SelectMore(fmt.Sprintf("toFloat64(%s) AS %s", histogramBucketKey, qbtypes.HeatmapBucketMaxColumn))
+	// a partial scrape can break monotonicity across `le`, and a negative cell
+	// count has no meaning
+	sb.SelectMore(fmt.Sprintf(
+		"greatest(value - lagInFrame(value, 1, 0) OVER %s, 0) AS %s",
+		heatmapWindow, heatmapValueAlias,
+	))
+	// sqlbuilder has no WINDOW clause; appending it to FROM lands it between FROM
+	// and ORDER BY, since these statements carry no WHERE or GROUP BY
+	sb.From(fmt.Sprintf(
+		"__spatial_aggregation_cte WINDOW %s AS (PARTITION BY %s ORDER BY toFloat64(%s))",
+		heatmapWindow, strings.Join(partitionBy, ", "), histogramBucketKey,
+	))
+	sb.OrderBy(groupAliases...)
+	sb.OrderBy("ts", fmt.Sprintf("toFloat64(%s)", histogramBucketKey))
+
+	q, a := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return &qbtypes.Statement{Query: combined + q, Args: append(args, a...)}, nil
+}
+
+// buildValueHeatmapFinalSelect places each spatially aggregated value in a
+// bucket of the requested axis. __spatial_aggregation_cte holds one row per
+// (group, timestamp), so every cell counts exactly one.
+func buildValueHeatmapFinalSelect(
+	combined string,
+	args []any,
+	query qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation],
+) (*qbtypes.Statement, error) {
+	bucketMin, bucketMax, err := renderHeatmapBucketExprs(*query.Aggregations[0].HeatmapBucketing)
+	if err != nil {
+		return nil, err
+	}
+
+	groupAliases := GroupByAliases(query.GroupBy)
+
+	sb := sqlbuilder.NewSelectBuilder()
+	sb.Select("ts")
+	sb.SelectMore(groupAliases...)
+	sb.SelectMore(fmt.Sprintf("%s AS %s", bucketMin, qbtypes.HeatmapBucketMinColumn))
+	sb.SelectMore(fmt.Sprintf("%s AS %s", bucketMax, qbtypes.HeatmapBucketMaxColumn))
+	sb.SelectMore(fmt.Sprintf("toFloat64(1) AS %s", heatmapValueAlias))
+	sb.From("__spatial_aggregation_cte")
+	sb.OrderBy(groupAliases...)
+	sb.OrderBy("ts", qbtypes.HeatmapBucketMaxColumn)
+
+	q, a := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return &qbtypes.Statement{Query: combined + q, Args: append(args, a...)}, nil
+}
+
+// renderHeatmapBucketExprs renders the bucket (min, max] that `value` falls in.
+// Only the bucket under everything the axis covers is open below, and only the
+// one over it is open above.
+func renderHeatmapBucketExprs(bucketing qbtypes.HeatmapBucketing) (minExpr, maxExpr string, err error) {
+	switch bucketing.Kind {
+	case qbtypes.BucketsKindLinear:
+		return renderLinearBucketExprs(bucketing)
+	case qbtypes.BucketsKindLog:
+		return renderLogBucketExprs()
+	default:
+		return "", "", errors.NewInvalidInputf(errors.CodeInvalidInput,
+			"unsupported bucketsScaling %q for heatmap requests", bucketing.Kind.StringValue())
+	}
+}
+
+func renderLinearBucketExprs(bucketing qbtypes.HeatmapBucketing) (string, string, error) {
+	maxValue := formatFloat(bucketing.MaxValue)
+	numBuckets := strconv.Itoa(bucketing.NumBuckets)
+	index := fmt.Sprintf("least(greatest(ceil(value * %s / %s), 1), %s)", numBuckets, maxValue, numBuckets)
+
+	minExpr := fmt.Sprintf(
+		"multiIf(value <= 0, toFloat64('-Inf'), value > %s, toFloat64(%s), (%s - 1) * %s / %s)",
+		maxValue, maxValue, index, maxValue, numBuckets,
+	)
+	maxExpr := fmt.Sprintf(
+		"multiIf(value <= 0, toFloat64(0), value > %s, toFloat64('+Inf'), %s * %s / %s)",
+		maxValue, index, maxValue, numBuckets,
+	)
+	return minExpr, maxExpr, nil
+}
+
+// ClickHouse buckets at MaxLogScale whatever HeatmapBucketing.LogScale asks for;
+// postprocessing folds the axis down afterwards.
+func renderLogBucketExprs() (string, string, error) {
+	bucketsPerDoubling := formatFloat(math.Exp2(qbtypes.MaxLogScale))
+	lowest := formatFloat(qbtypes.MinLogUpperBound)
+	highest := formatFloat(qbtypes.MaxLogUpperBound)
+
+	minExpr := fmt.Sprintf(
+		"multiIf(value <= 0, toFloat64('-Inf'), value <= %s, toFloat64(0), value > %s, toFloat64(%s), pow(2, (ceil(log2(value) * %s) - 1) / %s))",
+		lowest, highest, highest, bucketsPerDoubling, bucketsPerDoubling,
+	)
+	maxExpr := fmt.Sprintf(
+		"multiIf(value <= 0, toFloat64(0), value <= %s, %s, value > %s, toFloat64('+Inf'), pow(2, ceil(log2(value) * %s) / %s))",
+		lowest, lowest, highest, bucketsPerDoubling, bucketsPerDoubling,
+	)
+	return minExpr, maxExpr, nil
+}
+
+// formatFloat renders a float64 as the shortest literal that reads back as the
+// same value, so an upper bound computed from it is identical on every row.
+func formatFloat(v float64) string {
+	return strconv.FormatFloat(v, 'g', -1, 64)
 }
 
 func GroupByColumnAlias(i int, name string) string {

pkg/statementbuilder/metricsstatementbuilder/stmt_builder_test.go

@@ -212,7 +212,7 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
+				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value) AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
 				Args:  []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "cartservice", "signoz_latency", uint64(1747947390000), uint64(1747983420000)},
 			},
 			expectedErr: nil,
@@ -250,7 +250,7 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value)/30 AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
+				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value) AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND JSONExtractString(labels, 'service.name') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
 				Args:  []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "cartservice", "signoz_latency", uint64(1747947390000), uint64(1747983420000)},
 			},
 			expectedErr: nil,
@@ -284,6 +284,202 @@ func TestStatementBuilder(t *testing.T) {
 			},
 			expectedErr: nil,
 		},
+		{
+			name:        "test_histogram_heatmap_counting_aggregations",
+			requestType: qbtypes.RequestTypeHeatmap,
+			query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+				Signal:       telemetrytypes.SignalMetrics,
+				StepInterval: qbtypes.Step{Duration: 60 * time.Second},
+				Aggregations: []qbtypes.MetricAggregation{
+					{
+						MetricName:       "signoz_latency",
+						Type:             metrictypes.HistogramType,
+						Temporality:      metrictypes.Delta,
+						TimeAggregation:  metrictypes.TimeAggregationIncrease,
+						SpaceAggregation: metrictypes.SpaceAggregationCount,
+					},
+				},
+				GroupBy: []qbtypes.GroupByKey{
+					{
+						TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+							Name: "service.name",
+						},
+					},
+				},
+			},
+			expected: qbtypes.Statement{
+				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value) AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, lagInFrame(toFloat64(le), 1, toFloat64('-Inf')) OVER __heatmap_window AS __bucket_min, toFloat64(le) AS __bucket_max, greatest(value - lagInFrame(value, 1, 0) OVER __heatmap_window, 0) AS __result_0 FROM __spatial_aggregation_cte WINDOW __heatmap_window AS (PARTITION BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY toFloat64(le)) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts, toFloat64(le)",
+				Args:  []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "signoz_latency", uint64(1747947360000), uint64(1747983420000)},
+			},
+			expectedErr: nil,
+		},
+		{
+			// a heatmap counts observations per `le`, so a histogram's aggregations
+			// are rewritten to increase/count whatever was asked for — this builds
+			// the same statement as test_histogram_heatmap_counting_aggregations
+			name:        "test_histogram_heatmap_overrules_requested_aggregations",
+			requestType: qbtypes.RequestTypeHeatmap,
+			query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+				Signal:       telemetrytypes.SignalMetrics,
+				StepInterval: qbtypes.Step{Duration: 60 * time.Second},
+				Aggregations: []qbtypes.MetricAggregation{
+					{
+						MetricName:       "signoz_latency",
+						Type:             metrictypes.HistogramType,
+						Temporality:      metrictypes.Delta,
+						TimeAggregation:  metrictypes.TimeAggregationRate,
+						SpaceAggregation: metrictypes.SpaceAggregationPercentile95,
+					},
+				},
+				GroupBy: []qbtypes.GroupByKey{
+					{
+						TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+							Name: "service.name",
+						},
+					},
+				},
+			},
+			expected: qbtypes.Statement{
+				Query: "WITH __spatial_aggregation_cte AS (SELECT toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(value) AS value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, lagInFrame(toFloat64(le), 1, toFloat64('-Inf')) OVER __heatmap_window AS __bucket_min, toFloat64(le) AS __bucket_max, greatest(value - lagInFrame(value, 1, 0) OVER __heatmap_window, 0) AS __result_0 FROM __spatial_aggregation_cte WINDOW __heatmap_window AS (PARTITION BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY toFloat64(le)) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts, toFloat64(le)",
+				Args:  []any{"signoz_latency", uint64(1747936800000), uint64(1747983420000), "delta", "signoz_latency", uint64(1747947360000), uint64(1747983420000)},
+			},
+			expectedErr: nil,
+		},
+		{
+			name:        "test_gauge_heatmap_log",
+			requestType: qbtypes.RequestTypeHeatmap,
+			query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+				Signal:       telemetrytypes.SignalMetrics,
+				StepInterval: qbtypes.Step{Duration: 60 * time.Second},
+				Aggregations: []qbtypes.MetricAggregation{
+					{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						Temporality:      metrictypes.Unspecified,
+						TimeAggregation:  metrictypes.TimeAggregationAvg,
+						SpaceAggregation: metrictypes.SpaceAggregationSum,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{
+							Kind:       qbtypes.BucketsKindLog,
+							LogScale:   qbtypes.MaxLogScale,
+							NumBuckets: qbtypes.DefaultNumBuckets,
+						},
+					},
+				},
+				GroupBy: []qbtypes.GroupByKey{
+					{
+						TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+							Name: "host.name",
+						},
+					},
+				},
+			},
+			expected: qbtypes.Statement{
+				Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `__GROUP_BY_KEY_0_host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_host.name`) SELECT ts, `__GROUP_BY_KEY_0_host.name`, multiIf(value <= 0, toFloat64('-Inf'), value <= 2.3283064365386963e-10, toFloat64(0), value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(value) * 16) - 1) / 16)) AS __bucket_min, multiIf(value <= 0, toFloat64(0), value <= 2.3283064365386963e-10, 2.3283064365386963e-10, value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(value) * 16) / 16)) AS __bucket_max, toFloat64(1) AS __result_0 FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_host.name`, ts, __bucket_max",
+				Args:  []any{"system.memory.usage", uint64(1747936800000), uint64(1747983420000), "unspecified", "system.memory.usage", uint64(1747947360000), uint64(1747983420000), 0},
+			},
+			expectedErr: nil,
+		},
+		{
+			name:        "test_gauge_heatmap_linear",
+			requestType: qbtypes.RequestTypeHeatmap,
+			query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+				Signal:       telemetrytypes.SignalMetrics,
+				StepInterval: qbtypes.Step{Duration: 60 * time.Second},
+				Aggregations: []qbtypes.MetricAggregation{
+					{
+						MetricName:       "system.memory.usage",
+						Type:             metrictypes.GaugeType,
+						Temporality:      metrictypes.Unspecified,
+						TimeAggregation:  metrictypes.TimeAggregationAvg,
+						SpaceAggregation: metrictypes.SpaceAggregationSum,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{
+							Kind:       qbtypes.BucketsKindLinear,
+							LogScale:   qbtypes.MaxLogScale,
+							MaxValue:   500,
+							NumBuckets: 25,
+						},
+					},
+				},
+				GroupBy: []qbtypes.GroupByKey{
+					{
+						TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+							Name: "host.name",
+						},
+					},
+				},
+			},
+			expected: qbtypes.Statement{
+				Query: "WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_host.name`, avg(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'host.name') AS `__GROUP_BY_KEY_0_host.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_host.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_host.name` ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_host.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_host.name`) SELECT ts, `__GROUP_BY_KEY_0_host.name`, multiIf(value <= 0, toFloat64('-Inf'), value > 500, toFloat64(500), (least(greatest(ceil(value * 25 / 500), 1), 25) - 1) * 500 / 25) AS __bucket_min, multiIf(value <= 0, toFloat64(0), value > 500, toFloat64('+Inf'), least(greatest(ceil(value * 25 / 500), 1), 25) * 500 / 25) AS __bucket_max, toFloat64(1) AS __result_0 FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_host.name`, ts, __bucket_max",
+				Args:  []any{"system.memory.usage", uint64(1747936800000), uint64(1747983420000), "unspecified", "system.memory.usage", uint64(1747947360000), uint64(1747983420000), 0},
+			},
+			expectedErr: nil,
+		},
+		{
+			// cumulative keeps CanShortCircuitDelta false, so the counts reach the
+			// bucket differencing through the temporal CTE rather than the delta
+			// fast path
+			name:        "test_histogram_heatmap_cumulative",
+			requestType: qbtypes.RequestTypeHeatmap,
+			query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+				Signal:       telemetrytypes.SignalMetrics,
+				StepInterval: qbtypes.Step{Duration: 60 * time.Second},
+				Aggregations: []qbtypes.MetricAggregation{
+					{
+						MetricName:       "http_server_duration_bucket",
+						Type:             metrictypes.HistogramType,
+						Temporality:      metrictypes.Cumulative,
+						TimeAggregation:  metrictypes.TimeAggregationRate,
+						SpaceAggregation: metrictypes.SpaceAggregationPercentile95,
+					},
+				},
+				GroupBy: []qbtypes.GroupByKey{
+					{
+						TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+							Name: "service.name",
+						},
+					},
+				},
+			},
+			expected: qbtypes.Statement{
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, lagInFrame(toFloat64(le), 1, toFloat64('-Inf')) OVER __heatmap_window AS __bucket_min, toFloat64(le) AS __bucket_max, greatest(value - lagInFrame(value, 1, 0) OVER __heatmap_window, 0) AS __result_0 FROM __spatial_aggregation_cte WINDOW __heatmap_window AS (PARTITION BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY toFloat64(le)) ORDER BY `__GROUP_BY_KEY_0_service.name`, ts, toFloat64(le)",
+				Args:  []any{"http_server_duration_bucket", uint64(1747936800000), uint64(1747983420000), "cumulative", "http_server_duration_bucket", uint64(1747947300000), uint64(1747983420000), 0},
+			},
+			expectedErr: nil,
+		},
+		{
+			name:        "test_sum_heatmap_cumulative",
+			requestType: qbtypes.RequestTypeHeatmap,
+			query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
+				Signal:       telemetrytypes.SignalMetrics,
+				StepInterval: qbtypes.Step{Duration: 60 * time.Second},
+				Aggregations: []qbtypes.MetricAggregation{
+					{
+						MetricName:       "signoz_calls_total",
+						Type:             metrictypes.SumType,
+						Temporality:      metrictypes.Cumulative,
+						TimeAggregation:  metrictypes.TimeAggregationIncrease,
+						SpaceAggregation: metrictypes.SpaceAggregationSum,
+						HeatmapBucketing: &qbtypes.HeatmapBucketing{
+							Kind:       qbtypes.BucketsKindLog,
+							LogScale:   qbtypes.MaxLogScale,
+							NumBuckets: qbtypes.DefaultNumBuckets,
+						},
+					},
+				},
+				GroupBy: []qbtypes.GroupByKey{
+					{
+						TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+							Name: "service.name",
+						},
+					},
+				},
+			},
+			expected: qbtypes.Statement{
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, `__GROUP_BY_KEY_0_service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, multiIf(value <= 0, toFloat64('-Inf'), value <= 2.3283064365386963e-10, toFloat64(0), value > 1.8446744073709552e+19, toFloat64(1.8446744073709552e+19), pow(2, (ceil(log2(value) * 16) - 1) / 16)) AS __bucket_min, multiIf(value <= 0, toFloat64(0), value <= 2.3283064365386963e-10, 2.3283064365386963e-10, value > 1.8446744073709552e+19, toFloat64('+Inf'), pow(2, ceil(log2(value) * 16) / 16)) AS __bucket_max, toFloat64(1) AS __result_0 FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts, __bucket_max",
+				Args:  []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", "signoz_calls_total", uint64(1747947300000), uint64(1747983420000), 0},
+			},
+			expectedErr: nil,
+		},
 		{
 			name:        "test_gauge_avg_sum",
 			requestType: qbtypes.RequestTypeTimeSeries,
@@ -342,7 +538,7 @@ func TestStatementBuilder(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
+				Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, `le`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name`, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name`, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`, `le`) SELECT ts, `__GROUP_BY_KEY_0_service.name`, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.950) AS value FROM __spatial_aggregation_cte GROUP BY `__GROUP_BY_KEY_0_service.name`, ts ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
 				Args:  []any{"http_server_duration_bucket", uint64(1747936800000), uint64(1747983420000), "cumulative", "http_server_duration_bucket", uint64(1747947360000), uint64(1747983420000), 0},
 			},
 			expectedErr: nil,

pkg/types/dashboardtypes/perses_dashboard_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/SigNoz/signoz/pkg/errors"
+	qb "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/perses/spec/go/dashboard"
 	"github.com/stretchr/testify/assert"
@@ -524,6 +525,149 @@ func TestInvalidateUnknownPluginKind(t *testing.T) {
 	}
 }
 
+// TestHeatmapPanelQueryKinds pins the panel allowlist to what validateHeatmap
+// accepts in querybuildertypesv5: everything but a trace operator.
+func TestHeatmapPanelQueryKinds(t *testing.T) {
+	testCases := []struct {
+		description     string
+		queryPluginKind string
+		queryPluginSpec string
+		expectedAllowed bool
+	}{
+		{
+			description:     "a metrics builder query is allowed",
+			queryPluginKind: "signoz/BuilderQuery",
+			queryPluginSpec: `{"name": "A", "signal": "metrics", "aggregations": [
+				{"metricName": "http.server.request.duration", "timeAggregation": "increase", "spaceAggregation": "sum"}
+			]}`,
+			expectedAllowed: true,
+		},
+		{
+			description:     "a promql query is allowed",
+			queryPluginKind: "signoz/PromQLQuery",
+			queryPluginSpec: `{"name": "A", "query": "sum by (le) (increase(signoz_latency_bucket[5m]))"}`,
+			expectedAllowed: true,
+		},
+		{
+			description:     "a clickhouse query is allowed",
+			queryPluginKind: "signoz/ClickHouseSQL",
+			queryPluginSpec: `{"name": "A", "query": "SELECT ts, bucket, value FROM cells"}`,
+			expectedAllowed: true,
+		},
+		{
+			description:     "a formula is allowed",
+			queryPluginKind: "signoz/Formula",
+			queryPluginSpec: `{"name": "F1", "expression": "A / B"}`,
+			expectedAllowed: true,
+		},
+		{
+			description:     "a composite query is allowed, since a formula needs its disabled inputs alongside it",
+			queryPluginKind: "signoz/CompositeQuery",
+			queryPluginSpec: `{"queries": [
+				{"type": "builder_query", "spec": {"name": "A", "signal": "metrics", "disabled": true, "aggregations": [
+					{"metricName": "http.server.request.duration", "timeAggregation": "increase", "spaceAggregation": "sum"}
+				]}},
+				{"type": "builder_formula", "spec": {"name": "F1", "expression": "A * 2"}}
+			]}`,
+			expectedAllowed: true,
+		},
+		{
+			description:     "a trace operator is refused",
+			queryPluginKind: "signoz/TraceOperator",
+			queryPluginSpec: `{"name": "T1", "expression": "A => B"}`,
+			expectedAllowed: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.description, func(t *testing.T) {
+			data := fmt.Sprintf(`{
+				"variables": [],
+				"panels": {
+					"p1": {
+						"kind": "Panel",
+						"spec": {
+							"links": [],
+							"plugin": {"kind": "signoz/HeatmapPanel", "spec": {}},
+							"queries": [{
+								"kind": "heatmap",
+								"spec": {
+									"plugin": {"kind": %q, "spec": %s}
+								}
+							}]
+						}
+					}
+				},
+				"links": [],
+				"layouts": []
+			}`, testCase.queryPluginKind, testCase.queryPluginSpec)
+
+			_, err := unmarshalDashboard([]byte(data))
+
+			if testCase.expectedAllowed {
+				require.NoError(t, err)
+				return
+			}
+
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "is not supported by panel kind")
+		})
+	}
+}
+
+func TestValidateHeatmapDashboard(t *testing.T) {
+	data, err := os.ReadFile("testdata/perses_heatmap_panel.json")
+	require.NoError(t, err, "reading example file")
+
+	spec, err := unmarshalDashboard(data)
+	require.NoError(t, err, "unmarshal and validate failed")
+
+	require.IsType(t, &HeatmapPanelSpec{}, spec.Panels["p1"].Spec.Plugin.Spec)
+	panelSpec := spec.Panels["p1"].Spec.Plugin.Spec.(*HeatmapPanelSpec)
+	assert.Equal(t, "log", panelSpec.Axes.YScale.ValueOrDefault())
+	assert.Equal(t, "ember", panelSpec.ChartAppearance.Colors.Palette.ValueOrDefault())
+	assert.Equal(t, "sqrt", panelSpec.ChartAppearance.Colors.Scale.ValueOrDefault())
+	assert.Equal(t, 8, panelSpec.ChartAppearance.Colors.Steps)
+
+	dashboard := &DashboardV2{Spec: *spec}
+	request, err := dashboard.GetPanelQuery(1, 2, "p1")
+	require.NoError(t, err, "building the panel's query failed")
+	assert.Equal(t, qb.RequestTypeHeatmap, request.RequestType)
+
+	require.Len(t, request.CompositeQuery.Queries, 3)
+	numerator, ok := request.CompositeQuery.Queries[0].Spec.(qb.QueryBuilderQuery[qb.MetricAggregation])
+	require.True(t, ok, "expected a metrics builder query")
+	assert.True(t, numerator.Disabled)
+	require.NotNil(t, numerator.BucketOptions)
+	require.IsType(t, qb.LogBucketsSpec{}, numerator.BucketOptions.Spec)
+	assert.Equal(t, 4, *numerator.BucketOptions.Spec.(qb.LogBucketsSpec).Scale)
+
+	denominator, ok := request.CompositeQuery.Queries[1].Spec.(qb.QueryBuilderQuery[qb.MetricAggregation])
+	require.True(t, ok, "expected a metrics builder query")
+	assert.True(t, denominator.Disabled)
+	require.NotNil(t, denominator.BucketOptions)
+	require.IsType(t, qb.LinearBucketsSpec{}, denominator.BucketOptions.Spec)
+	assert.Equal(t, float64(1000), denominator.BucketOptions.Spec.(qb.LinearBucketsSpec).MaxValue)
+
+	formula, ok := request.CompositeQuery.Queries[2].Spec.(qb.QueryBuilderFormula)
+	require.True(t, ok, "expected a formula")
+	require.NotNil(t, formula.BucketOptions)
+	assert.Equal(t, qb.BucketsKindLog, formula.BucketOptions.Kind)
+	require.IsType(t, qb.LogBucketsSpec{}, formula.BucketOptions.Spec)
+	assert.Equal(t, 2, *formula.BucketOptions.Spec.(qb.LogBucketsSpec).Scale)
+
+	require.NoError(t, request.Validate(), "the request built from the panel is not a valid heatmap request")
+
+	// the panel read back out of storage draws the same heatmap
+	stored, err := json.Marshal(spec)
+	require.NoError(t, err, "marshal dashboard failed")
+	reread, err := unmarshalDashboard(stored)
+	require.NoError(t, err, "the stored dashboard does not validate")
+	rereadRequest, err := (&DashboardV2{Spec: *reread}).GetPanelQuery(1, 2, "p1")
+	require.NoError(t, err, "building the stored panel's query failed")
+	assert.Equal(t, request, rereadRequest)
+}
+
 func TestInvalidateOneInvalidPanel(t *testing.T) {
 	data := []byte(`{
 		"variables": [],

pkg/types/dashboardtypes/perses_plugin_wrappers.go

@@ -35,6 +35,7 @@ func (PanelPlugin) PrepareJSONSchema(s *jsonschema.Schema) error {
 		string(PanelKindTable):      schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTablePanelSpec"),
 		string(PanelKindHistogram):  schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHistogramPanelSpec"),
 		string(PanelKindList):       schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesListPanelSpec"),
+		string(PanelKindHeatmap):    schemaRef("DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesHeatmapPanelSpec"),
 	})
 }
 
@@ -65,6 +66,7 @@ func (PanelPlugin) JSONSchemaOneOf() []any {
 		PanelPluginVariant[TablePanelSpec]{Kind: string(PanelKindTable)},
 		PanelPluginVariant[HistogramPanelSpec]{Kind: string(PanelKindHistogram)},
 		PanelPluginVariant[ListPanelSpec]{Kind: string(PanelKindList)},
+		PanelPluginVariant[HeatmapPanelSpec]{Kind: string(PanelKindHeatmap)},
 	}
 }
 
@@ -228,6 +230,7 @@ var (
 		PanelKindTable:      func() any { return new(TablePanelSpec) },
 		PanelKindHistogram:  func() any { return new(HistogramPanelSpec) },
 		PanelKindList:       func() any { return new(ListPanelSpec) },
+		PanelKindHeatmap:    func() any { return new(HeatmapPanelSpec) },
 	}
 	queryPluginSpecs = map[QueryPluginKind]func() any{
 		QueryKindBuilder:       func() any { return new(BuilderQuerySpec) },
@@ -250,6 +253,7 @@ var (
 		PanelKindPieChart:   {QueryKindBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
 		PanelKindTable:      {QueryKindBuilder, QueryKindComposite, QueryKindFormula, QueryKindTraceOperator, QueryKindClickHouseSQL},
 		PanelKindList:       {QueryKindBuilder},
+		PanelKindHeatmap:    {QueryKindBuilder, QueryKindComposite, QueryKindFormula, QueryKindPromQL, QueryKindClickHouseSQL},
 	}
 )
 

pkg/types/dashboardtypes/perses_signoz_plugins.go

@@ -173,10 +173,11 @@ const (
 	PanelKindTable      PanelPluginKind = "signoz/TablePanel"
 	PanelKindHistogram  PanelPluginKind = "signoz/HistogramPanel"
 	PanelKindList       PanelPluginKind = "signoz/ListPanel"
+	PanelKindHeatmap    PanelPluginKind = "signoz/HeatmapPanel"
 )
 
 func (PanelPluginKind) Enum() []any {
-	return []any{PanelKindTimeSeries, PanelKindBarChart, PanelKindNumber, PanelKindPieChart, PanelKindTable, PanelKindHistogram, PanelKindList}
+	return []any{PanelKindTimeSeries, PanelKindBarChart, PanelKindNumber, PanelKindPieChart, PanelKindTable, PanelKindHistogram, PanelKindList, PanelKindHeatmap}
 }
 
 type TimeSeriesPanelSpec struct {
@@ -237,6 +238,56 @@ type ListPanelSpec struct {
 	SelectFields []telemetrytypes.TelemetryFieldKey `json:"selectFields,omitzero" validate:"dive"`
 }
 
+type HeatmapPanelSpec struct {
+	Visualization   BasicVisualization     `json:"visualization"`
+	Formatting      PanelFormatting        `json:"formatting"`
+	Axes            HeatmapAxes            `json:"axes"`
+	Legend          Legend                 `json:"legend"`
+	ChartAppearance HeatmapChartAppearance `json:"chartAppearance"`
+}
+
+// HeatmapAxes carries only the Y scale. The shared Axes type models a value
+// axis with soft bounds, where a heatmap's Y axis is the bucket boundaries the
+// response already fixed.
+type HeatmapAxes struct {
+	YScale HeatmapYScale `json:"yScale"`
+}
+
+type HeatmapChartAppearance struct {
+	Colors HeatmapColors `json:"colors"`
+}
+
+type HeatmapColors struct {
+	Mode    HeatmapColorMode  `json:"mode"`
+	Palette HeatmapPalette    `json:"palette"`
+	Scale   HeatmapColorScale `json:"scale"`
+	Steps   int               `json:"steps" validate:"omitempty,min=2,max=128"`
+	// MinCount and MaxCount clamp the colour scale; nil derives them from the
+	// grid, 0 and the highest count in it.
+	MinCount *float64 `json:"minCount"`
+	MaxCount *float64 `json:"maxCount"`
+	// Fill applies in opacity mode; empty means the selected group's legend colour.
+	Fill string `json:"fill"`
+}
+
+func (c *HeatmapColors) UnmarshalJSON(data []byte) error {
+	type alias HeatmapColors
+	var tmp alias
+	if err := json.Unmarshal(data, &tmp); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid heatmap colors")
+	}
+	*c = HeatmapColors(tmp)
+	return c.validate()
+}
+
+func (c HeatmapColors) validate() error {
+	if c.MinCount != nil && c.MaxCount != nil && *c.MinCount > *c.MaxCount {
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput,
+			"heatmap colors.minCount %v is greater than colors.maxCount %v", *c.MinCount, *c.MaxCount)
+	}
+	return nil
+}
+
 // ══════════════════════════════════════════════
 // Panel common types
 // ══════════════════════════════════════════════
@@ -709,3 +760,168 @@ func (p *PrecisionOption) UnmarshalJSON(data []byte) error {
 		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid precision option %q: must be `0`, `1`, `2`, `3`, `4`, or `full`", v)
 	}
 }
+
+type HeatmapColorMode struct{ valuer.String }
+
+var (
+	HeatmapColorModePalette = HeatmapColorMode{valuer.NewString("palette")} // default
+	HeatmapColorModeOpacity = HeatmapColorMode{valuer.NewString("opacity")}
+)
+
+func (HeatmapColorMode) Enum() []any {
+	return []any{HeatmapColorModePalette, HeatmapColorModeOpacity}
+}
+
+func (m HeatmapColorMode) ValueOrDefault() string {
+	if m.IsZero() {
+		return HeatmapColorModePalette.StringValue()
+	}
+	return m.StringValue()
+}
+
+func (m HeatmapColorMode) MarshalJSON() ([]byte, error) {
+	return json.Marshal(m.ValueOrDefault())
+}
+
+func (m *HeatmapColorMode) UnmarshalJSON(data []byte) error {
+	var v string
+	if err := json.Unmarshal(data, &v); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid heatmap color mode: must be a string, one of `palette` or `opacity`")
+	}
+	mode := HeatmapColorMode{valuer.NewString(v)}
+	switch mode {
+	case HeatmapColorModePalette, HeatmapColorModeOpacity:
+		*m = mode
+		return nil
+	default:
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid heatmap color mode %q: must be `palette` or `opacity`", v)
+	}
+}
+
+type HeatmapPalette struct{ valuer.String }
+
+var (
+	HeatmapPaletteIce      = HeatmapPalette{valuer.NewString("ice")}
+	HeatmapPaletteMoss     = HeatmapPalette{valuer.NewString("moss")}
+	HeatmapPaletteRust     = HeatmapPalette{valuer.NewString("rust")}
+	HeatmapPaletteGraphite = HeatmapPalette{valuer.NewString("graphite")}
+	HeatmapPaletteEmber    = HeatmapPalette{valuer.NewString("ember")}
+	HeatmapPaletteLagoon   = HeatmapPalette{valuer.NewString("lagoon")}
+	HeatmapPaletteOrchid   = HeatmapPalette{valuer.NewString("orchid")}
+	HeatmapPaletteVerdant  = HeatmapPalette{valuer.NewString("verdant")}
+	HeatmapPaletteLava     = HeatmapPalette{valuer.NewString("lava")} // default
+	HeatmapPaletteBeacon   = HeatmapPalette{valuer.NewString("beacon")}
+)
+
+func (HeatmapPalette) Enum() []any {
+	return []any{
+		HeatmapPaletteIce, HeatmapPaletteMoss, HeatmapPaletteRust, HeatmapPaletteGraphite,
+		HeatmapPaletteEmber, HeatmapPaletteLagoon, HeatmapPaletteOrchid, HeatmapPaletteVerdant,
+		HeatmapPaletteLava, HeatmapPaletteBeacon,
+	}
+}
+
+func (p HeatmapPalette) ValueOrDefault() string {
+	if p.IsZero() {
+		return HeatmapPaletteLava.StringValue()
+	}
+	return p.StringValue()
+}
+
+func (p HeatmapPalette) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.ValueOrDefault())
+}
+
+func (p *HeatmapPalette) UnmarshalJSON(data []byte) error {
+	var v string
+	if err := json.Unmarshal(data, &v); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid heatmap palette: must be a string, one of `ice`, `moss`, `rust`, `graphite`, `ember`, `lagoon`, `orchid`, `verdant`, `lava`, or `beacon`")
+	}
+	palette := HeatmapPalette{valuer.NewString(v)}
+	switch palette {
+	case HeatmapPaletteIce, HeatmapPaletteMoss, HeatmapPaletteRust, HeatmapPaletteGraphite,
+		HeatmapPaletteEmber, HeatmapPaletteLagoon, HeatmapPaletteOrchid, HeatmapPaletteVerdant,
+		HeatmapPaletteLava, HeatmapPaletteBeacon:
+		*p = palette
+		return nil
+	default:
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid heatmap palette %q: must be `ice`, `moss`, `rust`, `graphite`, `ember`, `lagoon`, `orchid`, `verdant`, `lava`, or `beacon`", v)
+	}
+}
+
+type HeatmapYScale struct{ valuer.String }
+
+var (
+	HeatmapYScaleAuto   = HeatmapYScale{valuer.NewString("auto")} // default
+	HeatmapYScaleLinear = HeatmapYScale{valuer.NewString("linear")}
+	HeatmapYScaleLog    = HeatmapYScale{valuer.NewString("log")}
+	HeatmapYScaleSymlog = HeatmapYScale{valuer.NewString("symlog")}
+)
+
+func (HeatmapYScale) Enum() []any {
+	return []any{HeatmapYScaleAuto, HeatmapYScaleLinear, HeatmapYScaleLog, HeatmapYScaleSymlog}
+}
+
+func (s HeatmapYScale) ValueOrDefault() string {
+	if s.IsZero() {
+		return HeatmapYScaleAuto.StringValue()
+	}
+	return s.StringValue()
+}
+
+func (s HeatmapYScale) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.ValueOrDefault())
+}
+
+func (s *HeatmapYScale) UnmarshalJSON(data []byte) error {
+	var v string
+	if err := json.Unmarshal(data, &v); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid heatmap y scale: must be a string, one of `auto`, `linear`, `log`, or `symlog`")
+	}
+	scale := HeatmapYScale{valuer.NewString(v)}
+	switch scale {
+	case HeatmapYScaleAuto, HeatmapYScaleLinear, HeatmapYScaleLog, HeatmapYScaleSymlog:
+		*s = scale
+		return nil
+	default:
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid heatmap y scale %q: must be `auto`, `linear`, `log`, or `symlog`", v)
+	}
+}
+
+type HeatmapColorScale struct{ valuer.String }
+
+var (
+	HeatmapColorScaleLog    = HeatmapColorScale{valuer.NewString("log")} // default
+	HeatmapColorScaleSqrt   = HeatmapColorScale{valuer.NewString("sqrt")}
+	HeatmapColorScaleLinear = HeatmapColorScale{valuer.NewString("linear")}
+)
+
+func (HeatmapColorScale) Enum() []any {
+	return []any{HeatmapColorScaleLog, HeatmapColorScaleSqrt, HeatmapColorScaleLinear}
+}
+
+func (s HeatmapColorScale) ValueOrDefault() string {
+	if s.IsZero() {
+		return HeatmapColorScaleLog.StringValue()
+	}
+	return s.StringValue()
+}
+
+func (s HeatmapColorScale) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.ValueOrDefault())
+}
+
+func (s *HeatmapColorScale) UnmarshalJSON(data []byte) error {
+	var v string
+	if err := json.Unmarshal(data, &v); err != nil {
+		return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid heatmap color scale: must be a string, one of `log`, `sqrt`, or `linear`")
+	}
+	scale := HeatmapColorScale{valuer.NewString(v)}
+	switch scale {
+	case HeatmapColorScaleLog, HeatmapColorScaleSqrt, HeatmapColorScaleLinear:
+		*s = scale
+		return nil
+	default:
+		return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "invalid heatmap color scale %q: must be `log`, `sqrt`, or `linear`", v)
+	}
+}

pkg/types/dashboardtypes/testdata/perses_heatmap_panel.json

@@ -0,0 +1,149 @@
+{
+  "display": {
+    "name": "latency",
+    "description": "how request duration is distributed"
+  },
+  "variables": [],
+  "panels": {
+    "p1": {
+      "kind": "Panel",
+      "spec": {
+        "display": {
+          "name": "request duration",
+          "description": ""
+        },
+        "plugin": {
+          "kind": "signoz/HeatmapPanel",
+          "spec": {
+            "visualization": {
+              "timePreference": "global_time"
+            },
+            "formatting": {
+              "unit": "s",
+              "decimalPrecision": "3"
+            },
+            "axes": {
+              "yScale": "log"
+            },
+            "legend": {
+              "position": "right",
+              "mode": "table"
+            },
+            "chartAppearance": {
+              "colors": {
+                "mode": "palette",
+                "palette": "ember",
+                "scale": "sqrt",
+                "steps": 8,
+                "minCount": 0,
+                "maxCount": 500
+              }
+            }
+          }
+        },
+        "links": [],
+        "queries": [
+          {
+            "kind": "heatmap",
+            "spec": {
+              "plugin": {
+                "kind": "signoz/CompositeQuery",
+                "spec": {
+                  "queries": [
+                    {
+                      "type": "builder_query",
+                      "spec": {
+                        "name": "A",
+                        "signal": "metrics",
+                        "disabled": true,
+                        "stepInterval": 60,
+                        "aggregations": [
+                          {
+                            "metricName": "http.server.request.duration",
+                            "timeAggregation": "increase",
+                            "spaceAggregation": "sum"
+                          }
+                        ],
+                        "groupBy": [
+                          {
+                            "name": "service.name"
+                          }
+                        ],
+                        "bucketOptions": {
+                          "kind": "log",
+                          "spec": {
+                            "scale": 4
+                          }
+                        }
+                      }
+                    },
+                    {
+                      "type": "builder_query",
+                      "spec": {
+                        "name": "B",
+                        "signal": "metrics",
+                        "disabled": true,
+                        "stepInterval": 60,
+                        "aggregations": [
+                          {
+                            "metricName": "http.server.request.count",
+                            "timeAggregation": "increase",
+                            "spaceAggregation": "sum"
+                          }
+                        ],
+                        "groupBy": [
+                          {
+                            "name": "service.name"
+                          }
+                        ],
+                        "bucketOptions": {
+                          "kind": "linear",
+                          "spec": {
+                            "maxValue": 1000,
+                            "numBuckets": 20
+                          }
+                        }
+                      }
+                    },
+                    {
+                      "type": "builder_formula",
+                      "spec": {
+                        "name": "F1",
+                        "expression": "A / B",
+                        "bucketOptions": {
+                          "kind": "log",
+                          "spec": {
+                            "scale": 2
+                          }
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  },
+  "links": [],
+  "layouts": [
+    {
+      "kind": "Grid",
+      "spec": {
+        "items": [
+          {
+            "x": 0,
+            "y": 0,
+            "width": 12,
+            "height": 8,
+            "content": {
+              "$ref": "#/spec/panels/p1"
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

pkg/types/querybuildertypes/querybuildertypesv5/builder_elements.go

@@ -540,6 +540,8 @@ type MetricAggregation struct {
 	// reduce to operator for metric scalar requests
 	ReduceTo ReduceTo `json:"reduceTo,omitzero"`
 
+	HeatmapBucketing *HeatmapBucketing `json:"-"`
+
 	Reduced bool `json:"-"`
 }
 
@@ -554,6 +556,10 @@ func (m MetricAggregation) Copy() MetricAggregation {
 		valueFilterCopy := *m.ValueFilter
 		c.ValueFilter = &valueFilterCopy
 	}
+	if m.HeatmapBucketing != nil {
+		bucketingCopy := *m.HeatmapBucketing
+		c.HeatmapBucketing = &bucketingCopy
+	}
 	return c
 }
 

pkg/types/querybuildertypes/querybuildertypesv5/builder_query.go

@@ -28,6 +28,10 @@ type QueryBuilderQuery[T any] struct {
 	// currently supported: []Aggregation, []MetricAggregation
 	Aggregations []T `json:"aggregations,omitzero"`
 
+	// BucketOptions is the bucket axis to count this query's values into. Only a
+	// heatmap request reads it, and only from the query it draws.
+	BucketOptions *BucketOptions `json:"bucketOptions,omitempty"`
+
 	// disabled if true, the query will not be executed
 	Disabled bool `json:"disabled"`
 
@@ -154,6 +158,11 @@ func (q QueryBuilderQuery[T]) Copy() QueryBuilderQuery[T] {
 		c.Having = q.Having.Copy()
 	}
 
+	if q.BucketOptions != nil {
+		bucketOptionsCopy := *q.BucketOptions
+		c.BucketOptions = &bucketOptionsCopy
+	}
+
 	return c
 }
 

pkg/types/querybuildertypes/querybuildertypesv5/formula.go

@@ -36,6 +36,10 @@ type QueryBuilderFormula struct {
 	// functions to apply to the formula result
 	Functions []Function `json:"functions,omitzero"`
 
+	// BucketOptions is the bucket axis to count the formula's results into.
+	// Only a heatmap request reads it, and only from the query it draws.
+	BucketOptions *BucketOptions `json:"bucketOptions,omitempty"`
+
 	Legend string `json:"legend"`
 }
 
@@ -61,6 +65,11 @@ func (f QueryBuilderFormula) Copy() QueryBuilderFormula {
 		c.Having = f.Having.Copy()
 	}
 
+	if f.BucketOptions != nil {
+		bucketOptionsCopy := *f.BucketOptions
+		c.BucketOptions = &bucketOptionsCopy
+	}
+
 	return c
 }
 

pkg/types/querybuildertypes/querybuildertypesv5/heatmap.go

@@ -0,0 +1,308 @@
+package querybuildertypesv5
+
+import (
+	"math"
+	"slices"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/metrictypes"
+)
+
+const (
+	// A heatmap statement reports a row's bucket as (min, max] under these two
+	// aliases, telling them apart from the single value column every other
+	// aggregation returns. Only the upper bounds reach the response; the lower
+	// ones are what says whether two rows bucketed the same way.
+	HeatmapBucketMinColumn = "__bucket_min"
+	HeatmapBucketMaxColumn = "__bucket_max"
+
+	DefaultNumBuckets = 60
+
+	// MaxLogScale is the resolution ClickHouse buckets every log heatmap at:
+	// 2^MaxLogScale bands per doubling. It is both the default and the finest
+	// available, since a coarser LogBucketsSpec.Scale folds down from it.
+	MaxLogScale = 4
+	// MinLogScale is one band per 16x, the coarsest axis worth rendering.
+	MinLogScale = -4
+
+	// A positive value approaching zero runs its band index off to -inf, so
+	// without a clamp one near-zero sample would stretch the axis by thousands
+	// of bands once AddHeatmapBucketsWithNoCounts spans it.
+	MinLogBandIndex = -512 // 2^-32, about 2.3e-10
+	MaxLogBandIndex = 1024 // 2^64, about 1.8e19
+)
+
+// MinLogUpperBound and MaxLogUpperBound are the ends the log axis is clamped
+// to. They do not vary with the requested scale.
+var (
+	MinLogUpperBound = math.Exp2(float64(MinLogBandIndex) / math.Exp2(MaxLogScale))
+	MaxLogUpperBound = math.Exp2(float64(MaxLogBandIndex) / math.Exp2(MaxLogScale))
+)
+
+// HeatmapBucketing is the bucket axis a heatmap statement builds in ClickHouse,
+// resolved from BucketOptions once the metric type is known. It stays nil for
+// histograms, whose upper bounds come from their own `le` labels.
+type HeatmapBucketing struct {
+	Kind BucketsKind
+	// LogScale is the resolution the caller asked for. ClickHouse always buckets
+	// at MaxLogScale, and postprocessing folds the axis down to this.
+	LogScale int
+	// MaxValue and NumBuckets are linear only.
+	MaxValue   float64
+	NumBuckets int
+}
+
+// ToHeatmapBucketing fills in what the caller left unset. An absent
+// BucketOptions resolves to the finest log axis, the one kind that needs nothing
+// from the caller.
+func (b *BucketOptions) ToHeatmapBucketing() HeatmapBucketing {
+	resolved := HeatmapBucketing{
+		Kind:       BucketsKindLog,
+		LogScale:   MaxLogScale,
+		NumBuckets: DefaultNumBuckets,
+	}
+	if b == nil {
+		return resolved
+	}
+
+	switch spec := b.Spec.(type) {
+	case LinearBucketsSpec:
+		resolved.Kind = BucketsKindLinear
+		resolved.MaxValue = spec.MaxValue
+		if spec.NumBuckets > 0 {
+			resolved.NumBuckets = spec.NumBuckets
+		}
+	case LogBucketsSpec:
+		if spec.Scale != nil {
+			resolved.LogScale = *spec.Scale
+		}
+	}
+
+	return resolved
+}
+
+// This cannot be called in validateHeatmap cuz type is resolved in querier.go.
+func (a *MetricAggregation) VerifyAndApplyBucketOptions(bucketOptions *BucketOptions) error {
+	switch a.Type {
+	case metrictypes.HistogramType:
+		if bucketOptions != nil {
+			return errors.NewInvalidInputf(errors.CodeInvalidInput,
+				"bucketOptions are not supported for histogram metrics: %q takes its bucket axis from its own `le` labels, so nothing in the spec would be applied", a.MetricName)
+		}
+		a.HeatmapBucketing = nil
+		return nil
+	// A summary carries no upper bounds of its own either, and its samples reach
+	// the final select the same way a gauge's do, so it buckets identically.
+	case metrictypes.GaugeType, metrictypes.SumType, metrictypes.SummaryType:
+		bucketing := bucketOptions.ToHeatmapBucketing()
+		a.HeatmapBucketing = &bucketing
+		return nil
+	case metrictypes.UnspecifiedType:
+		return errors.NewInvalidInputf(errors.CodeInvalidInput,
+			"heatmaps need a metric whose type is known: no type is recorded for %q, so its bucket axis cannot be chosen", a.MetricName)
+	case metrictypes.ExpHistogramType:
+		return errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported,
+			"heatmaps are not supported for exponential histograms yet: %q keeps its bucket counts in a sketch column, which needs its own reader", a.MetricName)
+	default:
+		return errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported,
+			"heatmaps are not supported for %s metrics", a.Type.StringValue())
+	}
+}
+
+func MergeBucketUpperBounds(tsData ...*TimeSeriesData) map[int][]float64 {
+	upperBoundsByAggregation := map[int][]float64{}
+
+	for _, data := range tsData {
+		if data == nil {
+			continue
+		}
+		for _, aggBucket := range data.Aggregations {
+			if len(aggBucket.Meta.Buckets) == 0 {
+				continue
+			}
+			upperBoundsByAggregation[aggBucket.Index] = append(upperBoundsByAggregation[aggBucket.Index], aggBucket.Meta.Buckets...)
+		}
+	}
+
+	for index, upperBounds := range upperBoundsByAggregation {
+		slices.Sort(upperBounds)
+		upperBoundsByAggregation[index] = slices.Compact(upperBounds)
+	}
+
+	return upperBoundsByAggregation
+}
+
+// DownscaleHeatmapResolution folds the MaxLogScale axis ClickHouse buckets at
+// down to toScale, merging every 2^(MaxLogScale-toScale) adjacent bands into
+// one. The coarser upper bounds are a subset of the finer ones, so the fold is
+// exact.
+func DownscaleHeatmapResolution(tsData *TimeSeriesData, toScale int) {
+	if tsData == nil || toScale >= MaxLogScale {
+		return
+	}
+	for _, aggBucket := range tsData.Aggregations {
+		downscaleHeatmapResolutionForAggregation(aggBucket, toScale)
+	}
+}
+
+func downscaleHeatmapResolutionForAggregation(aggBucket *AggregationBucket, toScale int) {
+	if aggBucket == nil || len(aggBucket.Meta.Buckets) == 0 {
+		return
+	}
+
+	factor := int(math.Exp2(float64(MaxLogScale - toScale)))
+
+	// Merging is by index in the exponential mapping, not by position in
+	// Meta.Buckets, which lists only the upper bounds some series reached.
+	coarseUpperBounds := make([]float64, 0, len(aggBucket.Meta.Buckets))
+	upperBoundToCoarseIndex := make(map[float64]int, len(aggBucket.Meta.Buckets))
+	mergedInto := make([]int, len(aggBucket.Meta.Buckets))
+	for index, upperBound := range aggBucket.Meta.Buckets {
+		coarsened := coarsenUpperBound(upperBound, toScale, factor)
+		coarseIndex, ok := upperBoundToCoarseIndex[coarsened]
+		if !ok {
+			coarseIndex = len(coarseUpperBounds)
+			coarseUpperBounds = append(coarseUpperBounds, coarsened)
+			upperBoundToCoarseIndex[coarsened] = coarseIndex
+		}
+		mergedInto[index] = coarseIndex
+	}
+
+	overflowIndex := len(coarseUpperBounds)
+	for _, series := range aggBucket.Series {
+		for _, point := range series.Values {
+			if len(point.Values) == 0 {
+				continue
+			}
+			coarseCounts := make([]float64, overflowIndex+1)
+			for index, count := range point.Values {
+				if index >= len(mergedInto) {
+					coarseCounts[overflowIndex] += count
+					continue
+				}
+				coarseCounts[mergedInto[index]] += count
+			}
+			point.Values = coarseCounts
+		}
+	}
+	aggBucket.Meta.Buckets = coarseUpperBounds
+}
+
+// coarsenUpperBound moves an upper bound from the MaxLogScale exponential axis
+// onto the toScale one. The zero band has no exponent to rescale and stays put.
+func coarsenUpperBound(upperBound float64, toScale, factor int) float64 {
+	if upperBound <= 0 || math.IsInf(upperBound, 0) || math.IsNaN(upperBound) {
+		return upperBound
+	}
+	index := int(math.Round(math.Log2(upperBound) * math.Exp2(MaxLogScale)))
+	merged := int(math.Ceil(float64(index) / float64(factor)))
+	return math.Exp2(float64(merged) / math.Exp2(float64(toScale)))
+}
+
+// AddHeatmapBucketsWithNoCounts spans the range from the rung below the lowest
+// upper bound some series reached to the highest. Meta.Buckets leaves the ones
+// in between out entirely, so without this a gap renders with its two sides
+// touching and the lowest bucket reaches the floor of the axis.
+//
+// Only a value-derived axis can be spanned: its upper bounds come from an index
+// that is a pure function of the value, so the ones in between are known without
+// having seen them. Nothing says what sits between two `le` labels.
+func AddHeatmapBucketsWithNoCounts(tsData *TimeSeriesData, bucketing HeatmapBucketing) {
+	if tsData == nil {
+		return
+	}
+	for _, aggBucket := range tsData.Aggregations {
+		addHeatmapBucketsWithNoCountsForAggregation(aggBucket, bucketing)
+	}
+}
+
+func addHeatmapBucketsWithNoCountsForAggregation(aggBucket *AggregationBucket, bucketing HeatmapBucketing) {
+	if aggBucket == nil || len(aggBucket.Meta.Buckets) == 0 {
+		return
+	}
+
+	// The zero bucket holds everything at or below zero. It has no index on either
+	// axis and sits below every other upper bound, so it keeps index 0 and the
+	// fill runs over the rest.
+	offset := 0
+	if aggBucket.Meta.Buckets[0] <= 0 {
+		offset = 1
+	}
+	positive := aggBucket.Meta.Buckets[offset:]
+	if len(positive) == 0 {
+		return
+	}
+
+	// Only finite upper bounds have an index, and the fill sizes a slice from
+	// one. Nothing should put +Inf or NaN on the axis, but bail if it happens.
+	indexes := make([]int, len(positive))
+	for i, upperBound := range positive {
+		if math.IsInf(upperBound, 0) || math.IsNaN(upperBound) {
+			return
+		}
+		indexes[i] = bucketing.calculateIndexOfUpperBound(upperBound)
+	}
+	lowest, highest := slices.Min(indexes), slices.Max(indexes)
+
+	// The lowest bucket reached needs the rung under it on the axis, or its own
+	// lower bound goes unstated and it reads as holding everything below. That
+	// one rung is enough: everything under it merges into the single leading
+	// bucket, which no series reached.
+	from := lowest - 1
+	if offset == 1 && bucketing.calculateUpperBoundAtIndex(from) <= aggBucket.Meta.Buckets[0] {
+		from = lowest
+	}
+
+	denseUpperBounds := append([]float64{}, aggBucket.Meta.Buckets[:offset]...)
+	for index := from; index <= highest; index++ {
+		denseUpperBounds = append(denseUpperBounds, bucketing.calculateUpperBoundAtIndex(index))
+	}
+	if len(denseUpperBounds) == len(aggBucket.Meta.Buckets) {
+		return
+	}
+
+	// Counts map through their index rather than by matching upper bounds, so a
+	// regenerated upper bound differing from ClickHouse's in its last bit still
+	// lands where it came from.
+	shiftedTo := make([]int, len(aggBucket.Meta.Buckets))
+	for i, index := range indexes {
+		shiftedTo[i+offset] = index - from + offset
+	}
+
+	overflowIndex := len(denseUpperBounds)
+	for _, series := range aggBucket.Series {
+		for _, point := range series.Values {
+			if len(point.Values) == 0 {
+				continue
+			}
+			denseCounts := make([]float64, overflowIndex+1)
+			for index, count := range point.Values {
+				if index >= len(shiftedTo) {
+					denseCounts[overflowIndex] += count
+					continue
+				}
+				denseCounts[shiftedTo[index]] += count
+			}
+			point.Values = denseCounts
+		}
+	}
+	aggBucket.Meta.Buckets = denseUpperBounds
+}
+
+// calculateIndexOfUpperBound and calculateUpperBoundAtIndex are inverses over
+// the axis being returned, so they read h.LogScale rather than the MaxLogScale
+// ClickHouse bucketed at: k * maxValue / numBuckets on a linear axis,
+// 2^(k / 2^scale) on a log one.
+func (h HeatmapBucketing) calculateIndexOfUpperBound(upperBound float64) int {
+	if h.Kind == BucketsKindLinear {
+		return int(math.Round(upperBound * float64(h.NumBuckets) / h.MaxValue))
+	}
+	return int(math.Round(math.Log2(upperBound) * math.Exp2(float64(h.LogScale))))
+}
+
+func (h HeatmapBucketing) calculateUpperBoundAtIndex(index int) float64 {
+	if h.Kind == BucketsKindLinear {
+		return float64(index) * h.MaxValue / float64(h.NumBuckets)
+	}
+	return math.Exp2(float64(index) / math.Exp2(float64(h.LogScale)))
+}

pkg/types/querybuildertypes/querybuildertypesv5/req.go

@@ -388,6 +388,14 @@ type QueryRangeRequest struct {
 	// NoCache is a flag to disable caching for the request.
 	NoCache bool `json:"noCache,omitempty"`
 
+	// PromQLProvider serves this request's PromQL queries via the named
+	// prometheus provider ("clickhousev2") instead of the default — the same
+	// data read through a different implementation. It is set from the
+	// X-SigNoz-PromQL-Provider header by the API handler, never from the
+	// body: a rollout-scoped comparison hook for integration tests and
+	// support should not become part of the public request schema.
+	PromQLProvider string `json:"-"`
+
 	FormatOptions *FormatOptions `json:"formatOptions,omitempty"`
 }
 
@@ -726,3 +734,130 @@ func (r *QueryRangeRequest) GetQueriesSupportingZeroDefault() map[string]bool {
 
 	return canDefaultZero
 }
+
+type BucketOptions struct {
+	Kind BucketsKind `json:"kind"`
+	Spec any         `json:"spec"`
+}
+
+type BucketsKind struct {
+	valuer.String
+}
+
+var (
+	BucketsKindLinear = BucketsKind{valuer.NewString("linear")}
+	BucketsKindLog    = BucketsKind{valuer.NewString("log")}
+)
+
+// Enum implements jsonschema.Enum.
+func (BucketsKind) Enum() []any {
+	return []any{
+		BucketsKindLinear,
+		BucketsKindLog,
+	}
+}
+
+// LinearBucketsSpec divides (0, MaxValue] into NumBuckets equal bands.
+type LinearBucketsSpec struct {
+	// Everything above MaxValue is counted in the trailing overflow band. Evenly
+	// spaced upper bounds have no top to divide without it, so it is required.
+	MaxValue   float64 `json:"maxValue" required:"true"`
+	NumBuckets int     `json:"numBuckets,omitempty"`
+}
+
+// LogBucketsSpec spaces upper bounds at 2^Scale bands per doubling, the mapping
+// an exponential histogram uses.
+type LogBucketsSpec struct {
+	// ClickHouse always buckets at MaxLogScale and the surplus is folded away
+	// afterwards, so every Scale reads the same cache entry. MaxLogScale applies
+	// when unset.
+	Scale *int `json:"scale,omitempty"`
+}
+
+func (b *BucketOptions) UnmarshalJSON(data []byte) error {
+	var shadow struct {
+		Kind BucketsKind     `json:"kind"`
+		Spec json.RawMessage `json:"spec"`
+	}
+	if err := binding.JSON.BindBody(bytes.NewReader(data), &shadow, binding.WithDisallowUnknownFields(true)); err != nil {
+		return err
+	}
+
+	b.Kind = shadow.Kind
+
+	// An absent spec is a malformed pair rather than a request for defaults;
+	// `"spec": {}` asks for those.
+	if len(shadow.Spec) == 0 {
+		return errors.NewInvalidInputf(errors.CodeInvalidInput,
+			"bucketOptions spec is required, use an empty object for the kind's defaults")
+	}
+
+	switch shadow.Kind {
+	case BucketsKindLinear:
+		var spec LinearBucketsSpec
+		if err := binding.JSON.BindBody(bytes.NewReader(shadow.Spec), &spec, binding.WithDisallowUnknownFields(true), binding.WithUnknownFieldContext("linear buckets spec")); err != nil {
+			return err
+		}
+		b.Spec = spec
+
+	case BucketsKindLog:
+		var spec LogBucketsSpec
+		if err := binding.JSON.BindBody(bytes.NewReader(shadow.Spec), &spec, binding.WithDisallowUnknownFields(true), binding.WithUnknownFieldContext("log buckets spec")); err != nil {
+			return err
+		}
+		b.Spec = spec
+
+	default:
+		return errors.NewInvalidInputf(
+			errors.CodeInvalidInput,
+			"invalid bucketOptions kind: %s",
+			shadow.Kind.StringValue(),
+		).WithAdditional(
+			"Valid bucket kinds are: linear, log",
+		)
+	}
+
+	return nil
+}
+
+// bucketOptionsLinear and bucketOptionsLog are the OpenAPI schemas for the two
+// BucketOptions variants. They have to be named types: the reflector turns an
+// anonymous one into an inline subschema, leaving the discriminator mapping in
+// PrepareJSONSchema pointing at components that were never emitted. `kind` is
+// required:"true" on both so oapi-codegen renders the discriminator non-pointer.
+type bucketOptionsLinear struct {
+	Kind BucketsKind       `json:"kind" required:"true" description:"How the upper bounds are spaced."`
+	Spec LinearBucketsSpec `json:"spec" required:"true" description:"The evenly spaced bucket specification."`
+}
+
+type bucketOptionsLog struct {
+	Kind BucketsKind    `json:"kind" required:"true" description:"How the upper bounds are spaced."`
+	Spec LogBucketsSpec `json:"spec" required:"true" description:"The logarithmic bucket specification."`
+}
+
+var _ jsonschema.OneOfExposer = BucketOptions{}
+
+func (BucketOptions) JSONSchemaOneOf() []any {
+	return []any{
+		bucketOptionsLinear{},
+		bucketOptionsLog{},
+	}
+}
+
+var _ jsonschema.Preparer = BucketOptions{}
+
+// PrepareJSONSchema marks the options as a `kind`-discriminated union;
+// signoz.attachDiscriminators promotes it and strips the base properties.
+func (BucketOptions) PrepareJSONSchema(s *jsonschema.Schema) error {
+	if s.ExtraProperties == nil {
+		s.ExtraProperties = map[string]any{}
+	}
+	s.ExtraProperties["x-signoz-discriminator"] = map[string]any{
+		"propertyName": "kind",
+		"mapping": map[string]string{
+			BucketsKindLinear.StringValue(): "#/components/schemas/Querybuildertypesv5BucketOptionsLinear",
+			BucketsKindLog.StringValue():    "#/components/schemas/Querybuildertypesv5BucketOptionsLog",
+		},
+	}
+	return nil
+}