refactor(querybuilder): derive the allowed table function message from the map

Values the map by the spelling to name back to the caller, so the message
comes off the same data the lookup uses and cannot drift from it. Drops
the test that existed only to catch that drift.

.github/workflows/cacheci.yml

@@ -0,0 +1,92 @@
+name: cacheci
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: write
+
+# Cancelling mid-rotation is safe: the sequential delete-then-save order
+# leaves at most one key missing at any moment.
+concurrency:
+  group: cacheci
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: restore
+        id: restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ runner.temp }}/cacheci
+          key: tests-primary
+          restore-keys: |
+            tests-secondary
+      - name: inject
+        if: steps.restore.outputs.cache-matched-key != ''
+        run: |
+          cat > "$RUNNER_TEMP/inject.Dockerfile" <<'EOF'
+          FROM busybox:1.37
+          RUN --mount=type=cache,target=/root/.cache/go-build \
+              --mount=type=cache,target=/go/pkg/mod \
+              --mount=type=cache,target=/pnpm/store \
+              --mount=type=bind,target=/restored \
+              tar -xf /restored/go-build.tar -C /root/.cache/go-build && \
+              tar -xf /restored/go-mod.tar -C /go/pkg/mod && \
+              tar -xf /restored/pnpm-store.tar -C /pnpm/store
+          EOF
+          docker build -f "$RUNNER_TEMP/inject.Dockerfile" "$RUNNER_TEMP/cacheci"
+      - name: build
+        run: |
+          docker build -f cmd/enterprise/Dockerfile.integration --build-arg TARGETARCH=amd64 --build-arg ZEUSURL=http://zeus:8080 .
+          docker build -f cmd/enterprise/Dockerfile.with-web.integration --build-arg TARGETARCH=amd64 --build-arg ZEUSURL=http://zeus:8080 .
+      # docker cp instead of --output type=local (the local exporter stalls on
+      # multi-GB outputs); tarballs instead of raw trees so the host never hits
+      # the permission and symlink semantics that broke docker cp.
+      - name: extract
+        run: |
+          rm -rf "$RUNNER_TEMP/cacheci"
+          mkdir -p "$RUNNER_TEMP/cacheci" "$RUNNER_TEMP/extract-context"
+          cat > "$RUNNER_TEMP/extract.Dockerfile" <<'EOF'
+          FROM busybox:1.37
+          RUN --mount=type=cache,target=/root/.cache/go-build \
+              --mount=type=cache,target=/go/pkg/mod \
+              --mount=type=cache,target=/pnpm/store \
+              mkdir -p /out && \
+              tar -cf /out/go-build.tar -C /root/.cache/go-build . && \
+              tar -cf /out/go-mod.tar -C /go/pkg/mod . && \
+              tar -cf /out/pnpm-store.tar -C /pnpm/store .
+          EOF
+          docker build -f "$RUNNER_TEMP/extract.Dockerfile" -t cacheci-extract "$RUNNER_TEMP/extract-context"
+          id=$(docker create cacheci-extract)
+          docker cp "$id":/out/. "$RUNNER_TEMP/cacheci/"
+          docker rm "$id"
+      # Fixed cache keys are immutable, so each key must be deleted before it
+      # can be saved again. Rotating primary and secondary one after the other
+      # keeps at least one key restorable for concurrent test runs.
+      - name: delete-primary
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh cache delete tests-primary --repo "$GITHUB_REPOSITORY" || true
+      - name: save-primary
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ runner.temp }}/cacheci
+          key: tests-primary
+      - name: delete-secondary
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh cache delete tests-secondary --repo "$GITHUB_REPOSITORY" || true
+      - name: save-secondary
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ runner.temp }}/cacheci
+          key: tests-secondary

.github/workflows/e2eci.yaml

@@ -75,6 +75,30 @@ jobs:
           docker rm pw
           echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" >> "$GITHUB_ENV"
           cd tests/e2e && pnpm playwright install-deps ${{ matrix.project }}
+      # Restore-only: the cacheci workflow owns cache saves. Seeds the
+      # BuildKit cache mounts so the in-test image build is incremental.
+      - name: restore
+        id: restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ runner.temp }}/cacheci
+          key: tests-primary
+          restore-keys: |
+            tests-secondary
+      - name: inject
+        if: steps.restore.outputs.cache-matched-key != ''
+        run: |
+          cat > "$RUNNER_TEMP/inject.Dockerfile" <<'EOF'
+          FROM busybox:1.37
+          RUN --mount=type=cache,target=/root/.cache/go-build \
+              --mount=type=cache,target=/go/pkg/mod \
+              --mount=type=cache,target=/pnpm/store \
+              --mount=type=bind,target=/restored \
+              tar -xf /restored/go-build.tar -C /root/.cache/go-build && \
+              tar -xf /restored/go-mod.tar -C /go/pkg/mod && \
+              tar -xf /restored/pnpm-store.tar -C /pnpm/store
+          EOF
+          docker build -f "$RUNNER_TEMP/inject.Dockerfile" "$RUNNER_TEMP/cacheci"
       - name: bring-up-stack
         run: |
           cd tests && \

.github/workflows/integrationci.yaml

@@ -110,6 +110,30 @@ jobs:
           sudo mv chromedriver-linux64/chromedriver /usr/local/bin/chromedriver
           chromedriver -version
           google-chrome-stable --version
+      # Restore-only: the cacheci workflow owns cache saves. Seeds the
+      # BuildKit cache mounts so the in-test image build is incremental.
+      - name: restore
+        id: restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ runner.temp }}/cacheci
+          key: tests-primary
+          restore-keys: |
+            tests-secondary
+      - name: inject
+        if: steps.restore.outputs.cache-matched-key != ''
+        run: |
+          cat > "$RUNNER_TEMP/inject.Dockerfile" <<'EOF'
+          FROM busybox:1.37
+          RUN --mount=type=cache,target=/root/.cache/go-build \
+              --mount=type=cache,target=/go/pkg/mod \
+              --mount=type=cache,target=/pnpm/store \
+              --mount=type=bind,target=/restored \
+              tar -xf /restored/go-build.tar -C /root/.cache/go-build && \
+              tar -xf /restored/go-mod.tar -C /go/pkg/mod && \
+              tar -xf /restored/pnpm-store.tar -C /pnpm/store
+          EOF
+          docker build -f "$RUNNER_TEMP/inject.Dockerfile" "$RUNNER_TEMP/cacheci"
       - name: run
         run: |
           cd tests && \

Makefile

@@ -209,8 +209,8 @@ py-lint: ## Run ruff check across the shared tests project
 	@cd tests && uv run ruff check --fix .
 
 .PHONY: py-test-setup
-py-test-setup: ## Bring up the shared SigNoz backend used by integration and e2e tests
-	@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --capture=no integration/bootstrap/setup.py::test_setup
+py-test-setup: ## Bring up the shared SigNoz backend used by integration and e2e tests, rebuilding signoz from the current sources
+	@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --rebuild --capture=no integration/bootstrap/setup.py::test_setup
 
 .PHONY: py-test-teardown
 py-test-teardown: ## Tear down the shared SigNoz backend

cmd/enterprise/Dockerfile.integration

@@ -4,9 +4,13 @@ ARG OS="linux"
 ARG TARGETARCH
 ARG ZEUSURL
 
+# HOME comes from the build user, not the image config; declare it so the
+# /root paths below trace to it.
+ENV HOME=/root
+
 # This path is important for stacktraces
 WORKDIR $GOPATH/src/github.com/signoz/signoz
-WORKDIR /root
+WORKDIR $HOME
 
 RUN set -eux; \
 	apt-get update; \
@@ -14,23 +18,36 @@ RUN set -eux; \
 		g++ \
 		gcc \
 		libc6-dev \
-		make \
 		pkg-config \
 	; \
 	rm -rf /var/lib/apt/lists/*
 
+# Keep the literal cache-mount targets below in sync with these. The caches
+# are shared with Dockerfile.with-web.integration (same target paths).
+ENV GOCACHE=$HOME/.cache/go-build
+ENV GOMODCACHE=$GOPATH/pkg/mod
+
 COPY go.mod go.sum ./
 
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod \
+	go mod download
 
 COPY ./cmd/ ./cmd/
 COPY ./ee/ ./ee/
 COPY ./pkg/ ./pkg/
 COPY ./templates /root/templates
 
-COPY Makefile Makefile
-RUN TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
-RUN mv /root/linux-${TARGETARCH}/signoz /root/signoz
+# Invoked directly instead of via make so Makefile changes don't invalidate
+# this layer; the Makefile's git-derived ldflags resolve to empty in here
+# anyway (.git is dockerignored).
+RUN --mount=type=cache,target=/go/pkg/mod \
+	--mount=type=cache,target=/root/.cache/go-build \
+	GOARCH=${TARGETARCH} GOOS=${OS} go build -C ./cmd/enterprise -race -tags timetzdata -o /root/signoz \
+	-ldflags "-s -w \
+		-X github.com/SigNoz/signoz/pkg/version.version=integration \
+		-X github.com/SigNoz/signoz/pkg/version.variant=enterprise \
+		-X github.com/SigNoz/signoz/ee/zeus.url=${ZEUSURL} \
+		-X github.com/SigNoz/signoz/ee/zeus.deprecatedURL=${ZEUSURL}/api/v1"
 
 RUN chmod 755 /root /root/signoz
 

cmd/enterprise/Dockerfile.with-web.integration

@@ -1,10 +1,23 @@
 FROM node:22-bookworm AS build
 
 WORKDIR /opt/
-COPY ./frontend/ ./
+
+# HOME comes from the build user, not the image config.
+ENV HOME=/root
+# pnpm's store lives at $PNPM_HOME/store — a dedicated directory pnpm
+# manages. Keep the literal cache-mount targets below in sync.
+ENV PNPM_HOME=/pnpm
 ENV NODE_OPTIONS=--max-old-space-size=8192
+
 RUN CI=1 npm i -g pnpm@10
-RUN CI=1 pnpm install
+
+# pnpm fetch resolves from the lockfile alone and runs no lifecycle scripts;
+# the repo's postinstall needs source files that are not copied yet.
+COPY ./frontend/package.json ./frontend/pnpm-lock.yaml ./frontend/pnpm-workspace.yaml ./
+RUN --mount=type=cache,target=/pnpm/store CI=1 pnpm fetch
+
+COPY ./frontend/ ./
+RUN --mount=type=cache,target=/pnpm/store CI=1 pnpm install --offline
 RUN CI=1 pnpm build
 
 FROM golang:1.25-bookworm
@@ -13,9 +26,13 @@ ARG OS="linux"
 ARG TARGETARCH
 ARG ZEUSURL
 
+# HOME comes from the build user, not the image config; declare it so the
+# /root paths below trace to it.
+ENV HOME=/root
+
 # This path is important for stacktraces
 WORKDIR $GOPATH/src/github.com/signoz/signoz
-WORKDIR /root
+WORKDIR $HOME
 
 RUN set -eux; \
 	apt-get update; \
@@ -23,23 +40,36 @@ RUN set -eux; \
 		g++ \
 		gcc \
 		libc6-dev \
-		make \
 		pkg-config \
 	; \
 	rm -rf /var/lib/apt/lists/*
 
+# Keep the literal cache-mount targets below in sync with these. The caches
+# are shared with Dockerfile.integration (same target paths).
+ENV GOCACHE=$HOME/.cache/go-build
+ENV GOMODCACHE=$GOPATH/pkg/mod
+
 COPY go.mod go.sum ./
 
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod \
+	go mod download
 
 COPY ./cmd/ ./cmd/
 COPY ./ee/ ./ee/
 COPY ./pkg/ ./pkg/
 COPY ./templates /root/templates
 
-COPY Makefile Makefile
-RUN TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
-RUN mv /root/linux-${TARGETARCH}/signoz /root/signoz
+# Invoked directly instead of via make so Makefile changes don't invalidate
+# this layer; the Makefile's git-derived ldflags resolve to empty in here
+# anyway (.git is dockerignored).
+RUN --mount=type=cache,target=/go/pkg/mod \
+	--mount=type=cache,target=/root/.cache/go-build \
+	GOARCH=${TARGETARCH} GOOS=${OS} go build -C ./cmd/enterprise -race -tags timetzdata -o /root/signoz \
+	-ldflags "-s -w \
+		-X github.com/SigNoz/signoz/pkg/version.version=integration \
+		-X github.com/SigNoz/signoz/pkg/version.variant=enterprise \
+		-X github.com/SigNoz/signoz/ee/zeus.url=${ZEUSURL} \
+		-X github.com/SigNoz/signoz/ee/zeus.deprecatedURL=${ZEUSURL}/api/v1"
 
 COPY --from=build /opt/build ./web/
 

docs/contributing/tests/e2e.md

@@ -30,11 +30,11 @@ yarn install:browsers   # one-time Playwright browser install
 
 ### Starting the Test Environment
 
-To spin up the backend stack (SigNoz, ClickHouse, Postgres, Zookeeper, Zeus mock, gateway mock, seeder, migrator-with-web) and keep it running:
+To spin up the backend stack (SigNoz, ClickHouse, Postgres, ClickHouse Keeper, Zeus mock, gateway mock, seeder, migrator-with-web) and keep it running:
 
 ```bash
 cd tests
-uv run pytest --basetemp=./tmp/ -vv --reuse --with-web \
+uv run pytest --basetemp=./tmp/ -vv --reuse --rebuild --with-web \
   e2e/bootstrap/setup.py::test_setup
 ```
 
@@ -45,8 +45,13 @@ This command will:
 - Start the HTTP seeder container (`tests/seeder/` — exposing `/telemetry/{traces,logs,metrics}` POST + DELETE)
 - Write backend coordinates to `tests/e2e/.env.local` (loaded by `playwright.config.ts` via dotenv)
 - Keep containers running via the `--reuse` flag
+- Rebuild the SigNoz container from the current sources via the `--rebuild` flag
 
-The `--with-web` flag builds the frontend into the SigNoz container — required for E2E. The build takes ~4 mins on a cold start.
+The `--with-web` flag builds the frontend into the SigNoz container — required for E2E. The build takes ~4 mins on a cold start; later builds are incremental.
+
+### Rebuilding After Source Changes
+
+The `--with-web` image bakes the built frontend in, so neither backend nor frontend changes are picked up while `--reuse` keeps the container running. `--rebuild` fixes that for both: it kills the SigNoz container, rebuilds the image incrementally (go build cache + pnpm store — a frontend-only change rebuilds in about a minute), and starts a fresh one while databases, mocks, migrations, and the seeder stay reused. The setup command above passes it, so the iteration loop is: change code → re-run the setup command → re-run your specs. `--rebuild` requires `--reuse` and cannot be combined with `--teardown` or `--clean`.
 
 ### Stopping the Test Environment
 
@@ -281,13 +286,16 @@ The full `playwright.config.ts` is the source of truth. Common things to tweak:
 The same pytest flags integration tests expose work here, since E2E reuses the shared fixture graph:
 
 - `--reuse` — keep containers warm between runs (required for all iteration).
+- `--rebuild` — recreate the SigNoz container from the current sources (backend and, with `--with-web`, frontend) while the rest of the stack stays up. Requires `--reuse`.
 - `--teardown` — tear everything down.
+- `--clean` — prune the docker build caches, forcing the next image build to start cold.
 - `--with-web` — build the frontend into the SigNoz container. **Required for E2E**; integration tests don't need it.
-- `--sqlstore-provider`, `--postgres-version`, `--clickhouse-version`, etc. — see `docs/contributing/integration.md`.
+- `--sqlstore-provider`, `--postgres-version`, `--clickhouse-version`, etc. — see `docs/contributing/tests/integration.md`.
 
 ## What should I remember?
 
-- **Always use the `--reuse` flag** when setting up the E2E stack. `--with-web` adds a ~4 min frontend build; you only want to pay that once.
+- **Always use the `--reuse` flag** when setting up the E2E stack. `--with-web` adds a ~4 min frontend build on a cold start; later builds are incremental.
+- **Changed backend or frontend code? Re-run the setup command** — it passes `--rebuild`, swapping the SigNoz container for one built from your current sources while the rest of the stack stays up.
 - **Don't teardown before setup.** `--reuse` correctly handles partially-set-up state, so chaining teardown → setup wastes time.
 - **Prefer UI-driven flows.** Playwright captures BE requests in the trace; a parallel `fetch` probe is almost always redundant. Drop to `page.request.*` only when the UI can't reach what you need.
 - **Use `page.waitForResponse` on UI clicks** to assert BE contracts — it still exercises the UI trigger path.

docs/contributing/tests/integration.md

@@ -37,13 +37,34 @@ make py-test-setup
 Under the hood this runs, from `tests/`:
 
 ```bash
-uv run pytest --basetemp=./tmp/ -vv --reuse integration/bootstrap/setup.py::test_setup
+uv run pytest --basetemp=./tmp/ -vv --reuse --rebuild --capture=no integration/bootstrap/setup.py::test_setup
 ```
 
 This command will:
-- Start all required services (ClickHouse, PostgreSQL, Zookeeper, SigNoz, Zeus mock, gateway mock)
+- Start all required services (ClickHouse, PostgreSQL, ClickHouse Keeper, SigNoz, Zeus mock, gateway mock)
 - Register an admin user
 - Keep containers running via the `--reuse` flag
+- Rebuild the SigNoz container from the current sources via the `--rebuild` flag
+
+### Rebuilding After Source Changes
+
+`--reuse` keeps the running SigNoz container, which means backend source changes are not picked up. `--rebuild` fixes exactly that: it kills the existing SigNoz container, rebuilds the image (incremental — only changed packages recompile thanks to the build cache), and starts a fresh one, while everything else (databases, mocks, migrations) stays reused. `make py-test-setup` passes it by default, so the iteration loop is simply:
+
+```bash
+make py-test-setup      # (re)build signoz from your current sources
+uv run pytest --basetemp=./tmp/ -vv --reuse integration/tests/<suite>/
+# ... edit backend code or tests ...
+make py-test-setup      # pick up the backend changes
+uv run pytest --basetemp=./tmp/ -vv --reuse integration/tests/<suite>/
+```
+
+The same applies to the e2e stack. `--rebuild` requires `--reuse` and cannot be combined with `--teardown` or `--clean`.
+
+Some suites define their own SigNoz variant in a suite-local `conftest.py` (`create_signoz(..., cache_key=...)` — e.g. `basepath`, `metricreduction`, `querier_json_body`). Those containers are not touched by `make py-test-setup`, which only rebuilds the default instance. For such suites, pass `--rebuild` on the suite run itself — it rebuilds every SigNoz variant the run instantiates:
+
+```bash
+uv run pytest --basetemp=./tmp/ -vv --reuse --rebuild integration/tests/<suite>/
+```
 
 ### Stopping the Test Environment
 
@@ -56,11 +77,21 @@ make py-test-teardown
 Which runs:
 
 ```bash
-uv run pytest --basetemp=./tmp/ -vv --teardown integration/bootstrap/setup.py::test_teardown
+uv run pytest --basetemp=./tmp/ -vv --teardown --capture=no integration/bootstrap/setup.py::test_teardown
 ```
 
 This destroys the running integration test setup and cleans up resources.
 
+### Cleaning the Image Build Cache
+
+The `signoz:integration` image build keeps its Go build and module caches in BuildKit cache mounts, so rebuilds only recompile what changed. These caches survive `--teardown` (they belong to the Docker builder, not to any container). If a cache ever needs to be nuked — suspected corruption, disk pressure, or to force a genuinely cold build — pass the `--clean` flag:
+
+```bash
+uv run pytest --basetemp=./tmp/ -vv --teardown --clean integration/bootstrap/setup.py::test_teardown
+```
+
+`--clean` prunes the docker build artifacts backing the incremental image build at session start, so the next build starts from a clean slate. Images and regular layer cache stay intact, but note the pruning is host-wide — it clears build caches for other projects too, not just SigNoz's. The flag composes with any invocation — passing it on a normal `--reuse` run simply makes the next image build start cold (~3–4 minutes instead of seconds).
+
 ## Understanding the Integration Test Framework
 
 Python and pytest form the foundation of the integration testing framework. Testcontainers are used to spin up disposable integration environments. WireMock is used to spin up **test doubles** of external services (Zeus cloud API, gateway, etc.).
@@ -99,7 +130,7 @@ tests/
 │       ├── passwordauthn/
 │       ├── querier/
 │       └── ...
-└── e2e/                     # Playwright suite (see docs/contributing/e2e.md)
+└── e2e/                     # Playwright suite (see docs/contributing/tests/e2e.md)
 ```
 
 Each test suite follows these principles:
@@ -224,9 +255,9 @@ Tests can be configured using pytest options:
 - `--sqlstore-provider` — Choose the SQL store provider (default: `postgres`)
 - `--sqlite-mode` — SQLite journal mode: `delete` or `wal` (default: `delete`). Only relevant when `--sqlstore-provider=sqlite`.
 - `--postgres-version` — PostgreSQL version (default: `15`)
-- `--clickhouse-version` — ClickHouse version (default: `25.5.6`)
-- `--zookeeper-version` — Zookeeper version (default: `3.7.1`)
-- `--schema-migrator-version` — SigNoz schema migrator version (default: `v0.144.2`)
+- `--clickhouse-version` — ClickHouse version, also used for ClickHouse Keeper (default: `25.12.5`)
+- `--schema-migrator-version` — SigNoz schema migrator version (default: `v0.144.6`)
+- `--with-web` — Build the frontend into the SigNoz image (required for e2e)
 
 Example:
 
@@ -239,6 +270,7 @@ uv run pytest --basetemp=./tmp/ -vv --reuse \
 ## What should I remember?
 
 - **Always use the `--reuse` flag** when setting up the environment or running tests to keep containers warm. Without it every run rebuilds the stack (~4 mins).
+- **Changed backend code? Re-run `make py-test-setup`** — it passes `--rebuild`, swapping the SigNoz container for one built from your current sources while the rest of the stack stays up.
 - **Use the `--teardown` flag** only when cleaning up — mixing `--teardown` with `--reuse` is a contradiction.
 - **Do not pre-emptively teardown before setup.** If the stack is partially up, `--reuse` picks up from wherever it is. `make py-test-teardown` then `make py-test-setup` wastes minutes.
 - **Follow the naming convention** with two-digit numeric prefixes (`01_`, `02_`) for ordered test execution within a suite.
@@ -247,5 +279,5 @@ uv run pytest --basetemp=./tmp/ -vv --reuse \
 - **Use descriptive test names** that clearly indicate what is being tested.
 - **Leverage fixtures** for common setup. The shared fixture package is at `tests/fixtures/` — reuse before adding new ones.
 - **Test both success and failure scenarios** (4xx / 5xx paths) to ensure robust functionality.
-- **Run `make py-fmt` and `make py-lint` before committing** Python changes — black + isort + autoflake + pylint.
+- **Run `make py-fmt` and `make py-lint` before committing** Python changes — ruff format + ruff check.
 - **`--sqlite-mode=wal` does not work on macOS.** The integration test environment runs SigNoz inside a Linux container with the SQLite database file mounted from the macOS host. WAL mode requires shared memory between connections, and connections crossing the VM boundary (macOS host ↔ Linux container) cannot share the WAL index, resulting in `SQLITE_IOERR_SHORT_READ`. WAL mode is tested in CI on Linux only.

go.mod

@@ -4,7 +4,7 @@ go 1.25.7
 
 require (
 	dario.cat/mergo v1.0.2
-	github.com/AfterShip/clickhouse-sql-parser v0.5.3
+	github.com/AfterShip/clickhouse-sql-parser v0.5.4
 	github.com/ClickHouse/clickhouse-go/v2 v2.44.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/clickhouse-go-mock v0.14.0

go.sum

@@ -66,8 +66,8 @@ dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
 filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
-github.com/AfterShip/clickhouse-sql-parser v0.5.3 h1:6iap8XGjuSjD3w7r1UNrg66ljBugcv2P39s4eo/ZLRw=
-github.com/AfterShip/clickhouse-sql-parser v0.5.3/go.mod h1:Qi3qvPTfZb/aFwI5V4WFOahgjsLJa4MzVijIAfwOhDw=
+github.com/AfterShip/clickhouse-sql-parser v0.5.4 h1:yiCQaMq8EO+dpKdnpP9YYd/ne6MSuOXgsMsNL33NiTI=
+github.com/AfterShip/clickhouse-sql-parser v0.5.4/go.mod h1:Qi3qvPTfZb/aFwI5V4WFOahgjsLJa4MzVijIAfwOhDw=
 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=

pkg/flagger/registry.go

@@ -14,8 +14,6 @@ var (
 	FeatureEnableAIObservability  = featuretypes.MustNewName("enable_ai_observability")
 	FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
 	FeatureUseInfraMonitoringV2   = featuretypes.MustNewName("use_infra_monitoring_v2")
-
-	FeatureUsePrometheusClickhouseV2 = featuretypes.MustNewName("use_prometheus_clickhouse_v2")
 )
 
 func MustNewRegistry() featuretypes.Registry {
@@ -108,14 +106,6 @@ func MustNewRegistry() featuretypes.Registry {
 			DefaultVariant: featuretypes.MustNewName("disabled"),
 			Variants:       featuretypes.NewBooleanVariants(),
 		},
-		&featuretypes.Feature{
-			Name:           FeatureUsePrometheusClickhouseV2,
-			Kind:           featuretypes.KindBoolean,
-			Stage:          featuretypes.StageExperimental,
-			Description:    "Runs PromQL queries on the clickhousev2 provider alongside the served engine result and logs any difference; serving is unaffected.",
-			DefaultVariant: featuretypes.MustNewName("disabled"),
-			Variants:       featuretypes.NewBooleanVariants(),
-		},
 	)
 	if err != nil {
 		panic(err)

pkg/prometheus/config.go

@@ -24,10 +24,6 @@ type Config struct {
 
 	// Timeout is the maximum time a query is allowed to run before being aborted.
 	Timeout time.Duration `mapstructure:"timeout"`
-
-	// ProviderName selects the storage provider: "clickhouse" (default) or
-	// "clickhousev2".
-	ProviderName string `mapstructure:"provider"`
 }
 
 func NewConfigFactory() factory.ConfigFactory {
@@ -41,8 +37,7 @@ func newConfig() factory.Config {
 			Path:          "",
 			MaxConcurrent: 20,
 		},
-		Timeout:      2 * time.Minute,
-		ProviderName: "clickhouse",
+		Timeout: 2 * time.Minute,
 	}
 }
 
@@ -50,15 +45,9 @@ func (c Config) Validate() error {
 	if c.Timeout <= 0 {
 		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::timeout must be greater than 0")
 	}
-	if c.ProviderName != "" && c.ProviderName != "clickhouse" && c.ProviderName != "clickhousev2" {
-		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::provider must be one of [clickhouse, clickhousev2], got %q", c.ProviderName)
-	}
 	return nil
 }
 
 func (c Config) Provider() string {
-	if c.ProviderName == "" {
-		return "clickhouse"
-	}
-	return c.ProviderName
+	return "clickhouse"
 }

pkg/prometheus/prometheus.go

@@ -35,9 +35,3 @@ type StatementRecorder interface {
 type StatementCapturer interface {
 	CapturingStorage() (storage.Queryable, StatementRecorder)
 }
-
-// ProviderClickhouseV2 is the clickhousev2 provider name: the factory
-// registration, the prometheus::provider config value and the
-// X-SigNoz-PromQL-Provider request header all use it, so they cannot drift
-// apart.
-const ProviderClickhouseV2 = "clickhousev2"

pkg/querier/api.go

@@ -57,7 +57,6 @@ func (handler *handler) QueryRange(rw http.ResponseWriter, req *http.Request) {
 		render.Error(rw, err)
 		return
 	}
-	queryRangeRequest.PromQLProvider = req.Header.Get("X-SigNoz-PromQL-Provider")
 
 	// Validate the query request
 	if err := queryRangeRequest.Validate(); err != nil {

pkg/querier/preview.go

@@ -231,7 +231,7 @@ func (q *querier) buildPreviewProviders(
 			sub.CompositeQuery = qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{query}}
 		}
 
-		built, _, bErr := q.buildQueries(orgID, &sub, deps, missingMetricQuerySet, event, promqlOptions{})
+		built, _, bErr := q.buildQueries(orgID, &sub, deps, missingMetricQuerySet, event)
 		if bErr != nil {
 			errs[name] = bErr
 			continue

pkg/querier/promql_query.go

@@ -8,12 +8,9 @@ import (
 	"regexp"
 	"sort"
 	"strings"
-	"sync"
 	"text/template"
 	"time"
 
-	"github.com/ClickHouse/clickhouse-go/v2"
-
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -101,24 +98,6 @@ type promqlQuery struct {
 	tr          qbv5.TimeRange
 	requestType qbv5.RequestType
 	vars        map[string]qbv5.VariableItem
-	opts        promqlOptions
-}
-
-// promqlOptions is how a PromQL query relates to the clickhousev2 provider
-// (see querier.promqlOptions for where the fields come from and why they are
-// flag-gated). Both providers are nil for a plain request, so a plain
-// request costs nothing extra.
-type promqlOptions struct {
-	// shadow, when set, runs the query on this provider after serving and
-	// logs any result difference; the response is never affected.
-	shadow prometheus.Prometheus
-	// shadowSlots is the querier-wide admission for shadow runs, shared by
-	// every query so the bound holds per process.
-	shadowSlots chan struct{}
-	// serve, when set, serves the response from this provider instead of the
-	// default path. Comparison callers fetch the default and the pinned
-	// result as two API calls and diff them.
-	serve prometheus.Prometheus
 }
 
 var _ qbv5.Query = (*promqlQuery)(nil)
@@ -131,7 +110,6 @@ func newPromqlQuery(
 	tr qbv5.TimeRange,
 	requestType qbv5.RequestType,
 	variables map[string]qbv5.VariableItem,
-	opts promqlOptions,
 ) *promqlQuery {
 	return &promqlQuery{
 		logger:      logger,
@@ -141,19 +119,10 @@ func newPromqlQuery(
 		tr:          tr,
 		requestType: requestType,
 		vars:        variables,
-		opts:        opts,
 	}
 }
 
 func (q *promqlQuery) Fingerprint() string {
-	// A pinned request must not share cache entries with default serving: a
-	// cached default result would satisfy the pin without running the pinned
-	// provider, and a pinned result would poison normal serving. No
-	// fingerprint means no caching at all — the pin exists to observe a
-	// provider, so a cache in front of it defeats the point.
-	if q.opts.serve != nil {
-		return ""
-	}
 	if q.requestType != qbv5.RequestTypeTimeSeries {
 		return ""
 	}
@@ -283,16 +252,7 @@ func (q *promqlQuery) PreviewStatements(ctx context.Context) ([]prometheus.Captu
 	start := int64(querybuilder.ToNanoSecs(q.tr.From))
 	end := int64(querybuilder.ToNanoSecs(q.tr.To))
 
-	// Attach the same query traits as Execute so the captured statements
-	// match what the live path would run.
-	if expr, parseErr := q.parser.ParseExpr(rendered); parseErr == nil {
-		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
-	}
-
 	capStorage, recorder := storer.CapturingStorage()
-	if capStorage == nil {
-		return nil, nil
-	}
 	qry, err := q.promEngine.Engine().NewRangeQuery(
 		ctx,
 		capStorage,
@@ -336,41 +296,6 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		return nil, err
 	}
 
-	// Attach query traits so the storage can prove step-aligned optimizations
-	// safe (see prometheus.QueryTraits). A parse failure surfaces below via
-	// the engine with the enhanced error message.
-	if expr, parseErr := q.parser.ParseExpr(query); parseErr == nil {
-		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
-	}
-
-	// Accumulate ClickHouse-side scan stats across every storage query this
-	// evaluation issues: progress options propagate to each ClickHouse query
-	// through the context.
-	var statsMu sync.Mutex
-	var rowsScanned, bytesScanned uint64
-	ctx = clickhouse.Context(ctx, clickhouse.WithProgress(func(p *clickhouse.Progress) {
-		statsMu.Lock()
-		rowsScanned += p.Rows
-		bytesScanned += p.Bytes
-		statsMu.Unlock()
-	}))
-
-	began := time.Now()
-
-	// A pinned provider serves directly from it: comparison callers fetch
-	// the default result and the pinned result as two API calls and diff
-	// them.
-	if q.opts.serve != nil {
-		matrix, err := q.serveFromProvider(ctx, query, start, end)
-		if err != nil {
-			if enhanced := tryEnhancePromQLExecError(err); enhanced != nil {
-				return nil, enhanced
-			}
-			return nil, err
-		}
-		return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned), nil
-	}
-
 	qry, err := q.promEngine.Engine().NewRangeQuery(
 		ctx,
 		q.promEngine.Storage(),
@@ -406,34 +331,6 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		return nil, errors.WrapInternalf(promErr, errors.CodeInternal, "error getting matrix from promql query %q", query)
 	}
 
-	if q.opts.shadow != nil {
-		// Shadows detach from the request, so without admission a dashboard
-		// burst would stack unbounded ClickHouse work for up to the shadow
-		// timeout — the concurrency pattern behind the original outages.
-		// Non-blocking: at the cap the comparison is skipped, not queued;
-		// a sampled shadow stream is exactly as useful for rollout evidence.
-		select {
-		case q.opts.shadowSlots <- struct{}{}:
-			// The engine pools the result's sample slices on Close; the
-			// shadow comparison needs a stable copy of what was served.
-			served := copyMatrix(matrix)
-			servedIn := time.Since(began)
-			go func() {
-				defer func() { <-q.opts.shadowSlots }()
-				q.runShadowCompare(context.WithoutCancel(ctx), query, start, end, served, servedIn)
-			}()
-		default:
-			q.logger.DebugContext(ctx, "promql shadow skipped: at concurrency cap", slog.String("query", query))
-		}
-	}
-
-	warnings, _ := res.Warnings.AsStrings(query, 10, 0)
-	return q.toResult(matrix, warnings, began, &statsMu, &rowsScanned, &bytesScanned), nil
-}
-
-// toResult converts an evaluated matrix into the v5 result shape, attaching
-// the ClickHouse scan stats accumulated during evaluation.
-func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) *qbv5.Result {
 	// Hide only known SigNoz storage keys: label names are user data and may
 	// legitimately start with "__" (e.g. __address__), so a blanket dunder
 	// strip mangles user labelsets. The __scope./__resource. prefixes cover
@@ -469,13 +366,7 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti
 		series = append(series, &s)
 	}
 
-	statsMu.Lock()
-	stats := qbv5.ExecStats{
-		RowsScanned:  *rowsScanned,
-		BytesScanned: *bytesScanned,
-		DurationMS:   uint64(time.Since(began).Milliseconds()),
-	}
-	statsMu.Unlock()
+	warnings, _ := res.Warnings.AsStrings(query, 10, 0)
 
 	tsData := &qbv5.TimeSeriesData{
 		QueryName: q.query.Name,
@@ -509,6 +400,6 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti
 		Type:     q.requestType,
 		Value:    payload,
 		Warnings: warnings,
-		Stats:    stats,
-	}
+		// TODO: map promql stats?
+	}, nil
 }

pkg/querier/promql_query_test.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/prometheus"
-	"github.com/SigNoz/signoz/pkg/prometheus/prometheustest"
 	qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/stretchr/testify/assert"
 )
@@ -441,15 +440,3 @@ func TestQuotedMetricOutsideBracesPattern(t *testing.T) {
 		})
 	}
 }
-
-// A pinned request must not share cache entries with default serving: a
-// cached default result would satisfy the pin without running the pinned
-// provider.
-func TestFingerprint_PinnedProviderBypassesCache(t *testing.T) {
-	q := &promqlQuery{
-		logger: slog.Default(),
-		query:  qbv5.PromQuery{Query: "up"},
-		opts:   promqlOptions{serve: &prometheustest.Provider{}},
-	}
-	assert.Empty(t, q.Fingerprint())
-}

pkg/querier/promql_shadow.go

@@ -1,175 +0,0 @@
-package querier
-
-import (
-	"context"
-	"fmt"
-	"log/slog"
-	"math"
-	"sort"
-	"time"
-
-	"github.com/ClickHouse/clickhouse-go/v2"
-	"github.com/SigNoz/signoz/pkg/prometheus"
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/promql"
-)
-
-// shadowTimeout bounds a shadow evaluation; a shadow run must never outlive
-// the request by much or pile up.
-const shadowTimeout = 2 * time.Minute
-
-// runShadowCompare executes the query on the clickhousev2 provider exactly
-// as it would serve (the engine over the v2 querier), compares against the
-// served result and logs the outcome. Serving is never affected: this runs
-// after the response, off the request context, and only logs. The mismatch
-// and failure logs are the rollout evidence — serving cuts over to v2 only
-// after they stay clean.
-func (q *promqlQuery) runShadowCompare(ctx context.Context, query string, startNs, endNs int64, served promql.Matrix, servedIn time.Duration) {
-	defer func() {
-		if r := recover(); r != nil {
-			q.logger.ErrorContext(ctx, "promql shadow comparison panicked", slog.Any("panic", r), slog.String("query", query))
-		}
-	}()
-
-	ctx, cancel := context.WithTimeout(ctx, shadowTimeout)
-	defer cancel()
-
-	// The request context carries the served response's scan-stats progress
-	// callback; without replacing it the shadow's ClickHouse progress would
-	// race into the served stats. The response itself was already sent.
-	ctx = clickhouse.Context(ctx, clickhouse.WithProgress(func(*clickhouse.Progress) {}))
-
-	if expr, parseErr := q.parser.ParseExpr(query); parseErr == nil {
-		ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
-	}
-
-	start, end := time.Unix(0, startNs), time.Unix(0, endNs)
-	began := time.Now()
-	shadow, err := executeOnProvider(ctx, q.opts.shadow, query, start, end, q.query.Step.Duration)
-	shadowIn := time.Since(began)
-
-	logAttrs := []any{
-		slog.String("query", query),
-		slog.Int64("start_ms", startNs/int64(time.Millisecond)),
-		slog.Int64("end_ms", endNs/int64(time.Millisecond)),
-		slog.Duration("step", q.query.Step.Duration),
-		slog.Duration("served_in", servedIn),
-		slog.Duration("shadow_in", shadowIn),
-	}
-
-	if err != nil {
-		// A shadow failure would be a serving failure after rollout; surface
-		// it at the same level as a result mismatch.
-		q.logger.WarnContext(ctx, "promql shadow execution failed", append(logAttrs, slog.Any("error", err))...)
-		return
-	}
-
-	servedNorm := normalizeShadowMatrix(served)
-	shadowNorm := normalizeShadowMatrix(shadow)
-	if diff := diffShadowMatrices(servedNorm, shadowNorm); diff != "" {
-		q.logger.WarnContext(ctx, "promql shadow comparison mismatch", append(logAttrs,
-			slog.String("diff", diff),
-			slog.Int("served_series", len(servedNorm)),
-			slog.Int("shadow_series", len(shadowNorm)),
-		)...)
-		return
-	}
-	// Matches log the timings: served_in vs shadow_in across the fleet is
-	// the perf evidence for the cutover, gathered for free.
-	q.logger.DebugContext(ctx, "promql shadow comparison matched", logAttrs...)
-}
-
-// serveFromProvider evaluates the query the way the pinned provider would
-// serve it.
-func (q *promqlQuery) serveFromProvider(ctx context.Context, query string, startNs, endNs int64) (promql.Matrix, error) {
-	return executeOnProvider(ctx, q.opts.serve, query, time.Unix(0, startNs), time.Unix(0, endNs), q.query.Step.Duration)
-}
-
-// executeOnProvider evaluates the query the way the provider would serve it:
-// the engine over the provider's storage. The returned matrix is an owned
-// copy.
-func executeOnProvider(ctx context.Context, prov prometheus.Prometheus, query string, start, end time.Time, step time.Duration) (promql.Matrix, error) {
-	qry, err := prov.Engine().NewRangeQuery(ctx, prov.Storage(), nil, query, start, end, step)
-	if err != nil {
-		return nil, err
-	}
-	defer qry.Close()
-
-	res := qry.Exec(ctx)
-	if res.Err != nil {
-		return nil, res.Err
-	}
-	matrix, err := res.Matrix()
-	if err != nil {
-		return nil, err
-	}
-	// Close returns the result's sample slices to the engine pool.
-	return copyMatrix(matrix), nil
-}
-
-func copyMatrix(matrix promql.Matrix) promql.Matrix {
-	out := make(promql.Matrix, 0, len(matrix))
-	for _, s := range matrix {
-		floats := make([]promql.FPoint, len(s.Floats))
-		copy(floats, s.Floats)
-		out = append(out, promql.Series{Metric: s.Metric.Copy(), Floats: floats})
-	}
-	return out
-}
-
-// normalizeShadowMatrix sorts by label set for order-independent
-// comparison. Both providers now resolve series identity the same way
-// (empty-valued labels dropped at read, no synthetic fingerprint label
-// since the v1 series-identity fix), so labels need no normalization.
-func normalizeShadowMatrix(matrix promql.Matrix) promql.Matrix {
-	out := make(promql.Matrix, 0, len(matrix))
-	out = append(out, matrix...)
-	sort.Slice(out, func(i, j int) bool { return labels.Compare(out[i].Metric, out[j].Metric) < 0 })
-	return out
-}
-
-// diffShadowMatrices returns a description of the first difference, or "".
-// Values compare with relative tolerance: spatial aggregations accumulate
-// floats in storage order, which differs between the providers in the last
-// ULP.
-func diffShadowMatrices(served, shadow promql.Matrix) string {
-	const relTol = 1e-9
-	if len(served) != len(shadow) {
-		return fmt.Sprintf("series count: served=%d shadow=%d", len(served), len(shadow))
-	}
-	for i := range served {
-		if labels.Compare(served[i].Metric, shadow[i].Metric) != 0 {
-			return fmt.Sprintf("series %d labels: served=%s shadow=%s", i, served[i].Metric, shadow[i].Metric)
-		}
-		if len(served[i].Floats) != len(shadow[i].Floats) {
-			return fmt.Sprintf("series %s points: served=%d shadow=%d", served[i].Metric, len(served[i].Floats), len(shadow[i].Floats))
-		}
-		for j := range served[i].Floats {
-			a, b := served[i].Floats[j], shadow[i].Floats[j]
-			if a.T != b.T {
-				return fmt.Sprintf("series %s point %d ts: served=%d shadow=%d", served[i].Metric, j, a.T, b.T)
-			}
-			// NaN and infinities first: NaN != NaN and Inf-Inf arithmetic
-			// would otherwise make one-sided NaN and Inf-vs-finite compare
-			// as equal (NaN > x and Inf > Inf are both false).
-			if math.IsNaN(a.F) || math.IsNaN(b.F) {
-				if math.IsNaN(a.F) != math.IsNaN(b.F) {
-					return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
-				}
-				continue
-			}
-			if math.IsInf(a.F, 0) || math.IsInf(b.F, 0) {
-				if a.F != b.F {
-					return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
-				}
-				continue
-			}
-			diff := math.Abs(a.F - b.F)
-			scale := math.Max(math.Abs(a.F), math.Abs(b.F))
-			if diff > relTol*math.Max(scale, 1e-300) && diff > 1e-12 {
-				return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
-			}
-		}
-	}
-	return ""
-}

pkg/querier/promql_shadow_test.go

@@ -1,67 +0,0 @@
-package querier
-
-import (
-	"math"
-	"testing"
-
-	"github.com/prometheus/prometheus/model/labels"
-	"github.com/prometheus/prometheus/promql"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNormalizeShadowMatrix(t *testing.T) {
-	matrix := promql.Matrix{
-		{
-			Metric: labels.FromStrings("__name__", "up", "job", "api"),
-			Floats: []promql.FPoint{{T: 1000, F: 1}},
-		},
-		{
-			Metric: labels.FromStrings("a", "1"),
-			Floats: []promql.FPoint{{T: 1000, F: 2}},
-		},
-	}
-	norm := normalizeShadowMatrix(matrix)
-	// sorted by label set; labels pass through untouched — both providers
-	// resolve series identity identically since the v1 series-identity fix
-	assert.Equal(t, labels.FromStrings("__name__", "up", "job", "api"), norm[0].Metric)
-	assert.Equal(t, labels.FromStrings("a", "1"), norm[1].Metric)
-}
-
-func TestDiffShadowMatrices(t *testing.T) {
-	series := func(v float64) promql.Matrix {
-		return promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 1000, F: v}}}}
-	}
-
-	assert.Empty(t, diffShadowMatrices(series(1.5), series(1.5)))
-	// last-ULP differences from storage-order float accumulation are expected
-	assert.Empty(t, diffShadowMatrices(series(0.08888888888888889), series(0.08888888888888888)))
-	assert.Empty(t, diffShadowMatrices(series(math.NaN()), series(math.NaN())))
-
-	assert.Contains(t, diffShadowMatrices(series(1.5), series(1.6)), "value")
-	assert.Contains(t, diffShadowMatrices(series(1.5), promql.Matrix{}), "series count")
-	assert.Contains(t, diffShadowMatrices(
-		series(1.5),
-		promql.Matrix{{Metric: labels.FromStrings("a", "2"), Floats: []promql.FPoint{{T: 1000, F: 1.5}}}},
-	), "labels")
-	assert.Contains(t, diffShadowMatrices(
-		series(1.5),
-		promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 2000, F: 1.5}}}},
-	), "ts")
-}
-
-// One-sided NaN makes every float comparison false, and Inf-Inf arithmetic
-// yields Inf > Inf == false; without explicit handling both divergences log
-// as matched — a shadow comparator that cannot see them would green-light a
-// broken rollout.
-func TestDiffShadowMatrices_SpecialFloats(t *testing.T) {
-	point := func(v float64) promql.Matrix {
-		return promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 1000, F: v}}}}
-	}
-
-	assert.NotEmpty(t, diffShadowMatrices(point(math.NaN()), point(1.5)), "one-sided NaN must diff")
-	assert.NotEmpty(t, diffShadowMatrices(point(1.5), point(math.NaN())), "one-sided NaN must diff either way")
-	assert.NotEmpty(t, diffShadowMatrices(point(math.Inf(1)), point(1.5)), "Inf vs finite must diff")
-	assert.NotEmpty(t, diffShadowMatrices(point(math.Inf(1)), point(math.Inf(-1))), "opposite infinities must diff")
-	assert.Empty(t, diffShadowMatrices(point(math.Inf(1)), point(math.Inf(1))), "equal infinities match")
-	assert.Empty(t, diffShadowMatrices(point(math.NaN()), point(math.NaN())), "both NaN match")
-}

pkg/querier/querier.go

@@ -24,7 +24,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/statsreporter"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
 	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
-	"github.com/SigNoz/signoz/pkg/types/featuretypes"
 	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
 	"github.com/SigNoz/signoz/pkg/types/metrictypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -47,19 +46,11 @@ type Querier interface {
 }
 
 type querier struct {
-	logger         *slog.Logger
-	fl             flagger.Flagger
-	telemetryStore telemetrystore.TelemetryStore
-	metadataStore  telemetrytypes.MetadataStore
-	promEngine     prometheus.Prometheus
-	// promV2 is the clickhousev2 prometheus provider, wired only when the
-	// serving provider is the default one (nil otherwise). It reads the same
-	// ClickHouse data through a different implementation; PromQL queries
-	// shadow-compare against it behind the use_prometheus_clickhouse_v2 flag
-	// and can be pinned to it for a response (see promqlOptions). It never
-	// serves by default — that cutover happens only after the shadow logs
-	// stay clean.
-	promV2                   prometheus.Prometheus
+	logger                   *slog.Logger
+	fl                       flagger.Flagger
+	telemetryStore           telemetrystore.TelemetryStore
+	metadataStore            telemetrytypes.MetadataStore
+	promEngine               prometheus.Prometheus
 	traceStmtBuilder         qbtypes.StatementBuilder[qbtypes.TraceAggregation]
 	logStmtBuilder           qbtypes.StatementBuilder[qbtypes.LogAggregation]
 	auditStmtBuilder         qbtypes.StatementBuilder[qbtypes.LogAggregation]
@@ -70,16 +61,8 @@ type querier struct {
 	liveDataRefresh          time.Duration
 	builderConfig            builderConfig
 	maxConcurrentQueries     int
-	// shadowSlots bounds concurrent shadow comparisons per process; shadows
-	// detach from their requests, so nothing else limits how many pile up.
-	shadowSlots chan struct{}
 }
 
-// maxConcurrentShadows is deliberately small: a shadow is a full extra
-// ClickHouse evaluation, and a sampled stream of comparisons is exactly as
-// useful for rollout evidence as an exhaustive one under load.
-const maxConcurrentShadows = 8
-
 var _ Querier = (*querier)(nil)
 
 func New(
@@ -87,7 +70,6 @@ func New(
 	telemetryStore telemetrystore.TelemetryStore,
 	metadataStore telemetrytypes.MetadataStore,
 	promEngine prometheus.Prometheus,
-	promV2 prometheus.Prometheus,
 	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
@@ -109,7 +91,6 @@ func New(
 		telemetryStore:           telemetryStore,
 		metadataStore:            metadataStore,
 		promEngine:               promEngine,
-		promV2:                   promV2,
 		traceStmtBuilder:         traceStmtBuilder,
 		logStmtBuilder:           logStmtBuilder,
 		auditStmtBuilder:         auditStmtBuilder,
@@ -122,7 +103,6 @@ func New(
 			logTraceIDWindowPaddingMS: uint64(logTraceIDWindowPadding.Milliseconds()),
 		},
 		maxConcurrentQueries: maxConcurrentQueries,
-		shadowSlots:          make(chan struct{}, maxConcurrentShadows),
 	}
 }
 
@@ -162,11 +142,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 		missingMetricQuerySet[name] = true
 	}
 
-	promqlOpts, err := q.promqlOptions(ctx, orgID, req)
-	if err != nil {
-		return nil, err
-	}
-	queries, steps, err := q.buildQueries(orgID, req, dependencyQueries, missingMetricQuerySet, event, promqlOpts)
+	queries, steps, err := q.buildQueries(orgID, req, dependencyQueries, missingMetricQuerySet, event)
 	if err != nil {
 		return nil, err
 	}
@@ -209,41 +185,12 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 	return qbResp, qbErr
 }
 
-// promqlOptions derives the PromQL execution options for a request. With the
-// org's use_prometheus_clickhouse_v2 flag on, queries are shadow-compared
-// against the clickhousev2 provider (serving unaffected, diffs logged; see
-// promql_shadow.go). The X-SigNoz-PromQL-Provider header may instead pin the
-// response to that provider — integration tests and support fetch both
-// results for comparison — so it is deliberately flag-gated too: without the
-// gate the header would be an unaudited switch onto a provider still under
-// validation.
-func (q *querier) promqlOptions(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (promqlOptions, error) {
-	enabled := q.fl.BooleanOrEmpty(ctx, flagger.FeatureUsePrometheusClickhouseV2, featuretypes.NewFlaggerEvaluationContext(orgID))
-	if req.PromQLProvider == "" {
-		if enabled && q.promV2 != nil {
-			return promqlOptions{shadow: q.promV2, shadowSlots: q.shadowSlots}, nil
-		}
-		return promqlOptions{}, nil
-	}
-	if req.PromQLProvider != prometheus.ProviderClickhouseV2 {
-		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "unknown promql provider %q", req.PromQLProvider)
-	}
-	if !enabled {
-		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "promql provider %q requires the use_prometheus_clickhouse_v2 flag", req.PromQLProvider)
-	}
-	if q.promV2 == nil {
-		return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "promql provider %q is not available", req.PromQLProvider)
-	}
-	return promqlOptions{serve: q.promV2}, nil
-}
-
 func (q *querier) buildQueries(
 	orgID valuer.UUID,
 	req *qbtypes.QueryRangeRequest,
 	dependencyQueries map[string]bool,
 	missingMetricQuerySet map[string]bool,
 	event *qbtypes.QBEvent,
-	promqlOpts promqlOptions,
 ) (map[string]qbtypes.Query, map[string]qbtypes.Step, error) {
 
 	tmplVars := req.Variables
@@ -268,7 +215,7 @@ func (q *querier) buildQueries(
 			if !ok {
 				return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid promql query spec %T", query.Spec)
 			}
-			promqlQuery := newPromqlQuery(q.logger, q.promEngine, promQuery, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars, promqlOpts)
+			promqlQuery := newPromqlQuery(q.logger, q.promEngine, promQuery, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars)
 			queries[promQuery.Name] = promqlQuery
 			steps[promQuery.Name] = promQuery.Step
 		case qbtypes.QueryTypeClickHouseSQL:
@@ -911,7 +858,7 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
 	switch qt := originalQuery.(type) {
 	case *promqlQuery:
 		queryCopy := qt.query.Copy()
-		return newPromqlQuery(q.logger, qt.promEngine, queryCopy, timeRange, qt.requestType, qt.vars, qt.opts)
+		return newPromqlQuery(q.logger, q.promEngine, queryCopy, timeRange, qt.requestType, qt.vars)
 
 	case *chSQLQuery:
 		queryCopy := qt.query.Copy()

pkg/querier/querier_test.go

@@ -48,7 +48,6 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
 		nil, // telemetryStore
 		metadataStore,
 		nil,                // prometheus
-		nil,                // promV2
 		nil,                // traceStmtBuilder
 		nil,                // logStmtBuilder
 		nil,                // auditStmtBuilder
@@ -121,7 +120,6 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
-		nil, // promV2
 		nil, // traceStmtBuilder
 		nil, // logStmtBuilder
 		nil, // auditStmtBuilder

pkg/querier/signozquerier/provider.go

@@ -18,7 +18,6 @@ import (
 func NewFactory(
 	telemetryStore telemetrystore.TelemetryStore,
 	prometheus prometheus.Prometheus,
-	promV2 prometheus.Prometheus,
 	metadataStore telemetrytypes.MetadataStore,
 	traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
 	logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
@@ -41,7 +40,6 @@ func NewFactory(
 				telemetryStore,
 				metadataStore,
 				prometheus,
-				promV2,
 				traceStmtBuilder,
 				logStmtBuilder,
 				auditStmtBuilder,

pkg/query-service/rules/managertestfactory.go

@@ -128,7 +128,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
 	meterStmtBuilder, err := meterstatementbuilder.NewFactory(metadataStore, flagger).New(ctx, providerSettings, cfg)
 	require.NoError(t, err)
 	bucketCache := querier.NewBucketCache(providerSettings, cache, 0, 0)
-	providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, nil, metadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger)
+	providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger)
 	mockQuerier, err := providerFactory.New(context.Background(), providerSettings, querier.Config{})
 	require.NoError(t, err)
 

pkg/query-service/rules/setups_test.go

@@ -40,7 +40,6 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
-		nil, // promV2
 		nil, // traceStmtBuilder
 		nil, // logStmtBuilder
 		nil, // auditStmtBuilder
@@ -75,7 +74,6 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
-		nil, // promV2
 		nil, // traceStmtBuilder
 		logStmtBuilder,
 		nil, // auditStmtBuilder
@@ -111,7 +109,6 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
 		telemetryStore,
 		metadataStore,
 		nil, // prometheus
-		nil, // promV2
 		traceStmtBuilder,
 		nil, // logStmtBuilder
 		nil, // auditStmtBuilder

pkg/querybuilder/clickhouse_sql.go

@@ -3,6 +3,8 @@ package querybuilder
 import (
 	"context"
 	"log/slog"
+	"maps"
+	"slices"
 	"strings"
 
 	chparser "github.com/AfterShip/clickhouse-sql-parser/parser"
@@ -25,7 +27,23 @@ var internalDatabases = map[string]struct{}{
 	"information_schema": {},
 }
 
-// The parser's grammar has gaps against SQL that ClickHouse itself accepts. See TestErrIfStatementIsNotValid_ShouldPassButFails.
+// generatorTableFunctions compute their rows from their arguments alone. They open no file or socket, reach no other host, and name no table, database or dictionary, so none of them can read through anything the rules here exist to protect. Can be used to build a dense axis to join a sparse series against. Every other table function is refused.
+//
+// Keyed by the lowercased name so that matching is case-insensitive, valued by the spelling to name it back to the caller.
+//
+// TODO(@therealpandey): take a deployment level allow list on top of this, so an operator can permit more without a release.
+var generatorTableFunctions = map[string]string{
+	"numbers":         "numbers",
+	"numbers_mt":      "numbers_mt",
+	"zeros":           "zeros",
+	"zeros_mt":        "zeros_mt",
+	"generateseries":  "generateSeries",
+	"generate_series": "generate_series",
+}
+
+var generatorTableFunctionsMessage = "allowed table functions are " + strings.Join(slices.Sorted(maps.Values(generatorTableFunctions)), ", ")
+
+// The parser's grammar has gaps against SQL that ClickHouse itself accepts.
 func ErrIfStatementIsNotValid(query string) (err error) {
 	defer func() {
 		// The parser has a history of panicking on malformed input rather than returning an error.
@@ -52,8 +70,17 @@ func ErrIfStatementIsNotValid(query string) (err error) {
 	visitor := &chparser.DefaultASTVisitor{Visit: func(node chparser.Expr) error {
 		switch expr := node.(type) {
 		case *chparser.TableFunctionExpr:
-			// Source table functions remain usable in ClickHouse read-only mode.
-			return errors.NewInvalidInputf(CodeClickHouseSQLTableFunction, "ClickHouse table functions are not allowed in SQL queries: %s", chparser.Format(expr.Name))
+			// Source table functions remain usable in ClickHouse read-only mode. Arguments are
+			// visited before this, so a read smuggled into one is already refused by the time
+			// an allowed generator gets here.
+			name := chparser.Format(expr.Name)
+			if _, ok := generatorTableFunctions[strings.ToLower(name)]; ok {
+				return nil
+			}
+
+			return errors.
+				NewInvalidInputf(CodeClickHouseSQLTableFunction, "ClickHouse table functions are not allowed in SQL queries: %s", name).
+				WithAdditional(generatorTableFunctionsMessage)
 
 		case *chparser.TableIdentifier:
 			// Reading these is unaffected by ClickHouse read-only mode.

pkg/querybuilder/clickhouse_sql_test.go

@@ -2,12 +2,11 @@ package querybuilder
 
 import (
 	"testing"
+	"time"
 
 	"github.com/SigNoz/signoz/pkg/errors"
 
-	chparser "github.com/AfterShip/clickhouse-sql-parser/parser"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestErrIfStatementIsNotValid_Pass(t *testing.T) {
@@ -21,6 +20,9 @@ func TestErrIfStatementIsNotValid_Pass(t *testing.T) {
 		{"CommonTableExpression", "WITH t AS (SELECT fingerprint FROM signoz_metrics.time_series_v4) SELECT * FROM t"},
 		{"Join", "SELECT * FROM t1 LEFT JOIN t2 ON t1.a = t2.b"},
 		{"GlobalIn", "SELECT a FROM t WHERE a GLOBAL IN (SELECT b FROM t2)"},
+		// GLOBAL parsed only when the join type was omitted, and only before IN. https://github.com/AfterShip/clickhouse-sql-parser/pull/293
+		{"GlobalLeftJoin", "SELECT * FROM t1 GLOBAL LEFT JOIN t2 ON t1.a = t2.a"},
+		{"GlobalNotIn", "SELECT a FROM t WHERE a GLOBAL NOT IN (SELECT b FROM t2)"},
 		{"Union", "SELECT * FROM t UNION ALL SELECT * FROM t2"},
 		{"Intersect", "SELECT * FROM t INTERSECT SELECT * FROM t2"},
 		{"WindowFunction", "SELECT sum(v) OVER (PARTITION BY a ORDER BY t) FROM t"},
@@ -36,18 +38,50 @@ func TestErrIfStatementIsNotValid_Pass(t *testing.T) {
 		// order by interval
 		{"OrderByInterval", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval ORDER BY interval"},
 		{"OrderByIntervalAndDirection", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS `interval` ORDER BY `interval` ASC"},
-		// Unspaced, so rejected until the parser stopped lexing a signed literal after a
-		// closing bracket. The spaced form above no longer needs to be spaced.
-		// https://github.com/AfterShip/clickhouse-sql-parser/issues/286
+		// `interval` is a unit keyword, so unquoting it was rejected everywhere the parser
+		// expected a plain identifier. https://github.com/AfterShip/clickhouse-sql-parser/pull/296
+		{"OrderByUnquotedIntervalAsc", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval FROM t GROUP BY interval ORDER BY interval ASC"},
+		{"OrderByUnquotedIntervalDesc", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval FROM t GROUP BY interval ORDER BY interval DESC"},
+		{"UnquotedIntervalInGroupByTuple", "SELECT a FROM t GROUP BY (`service.name`, `service.version`, interval)"},
+		{"UnquotedIntervalProductionQuery", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, resource_string_service$$name AS `service.name`, attributes_string['http.route'] AS `http.route`, quantile(0.95)(duration_nano) / 1000000000 AS value FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_string_service$$name = 'svc-a' AND resources_string['deployment.environment'] = 'dev' AND attributes_string['http.route'] = '/v1' AND http_method = 'POST' AND timestamp BETWEEN toDateTime(1784601720) AND toDateTime(1784602620) AND ts_bucket_start BETWEEN 1784601720 - 1800 AND 1784602620 GROUP BY `service.name`, `http.route`, interval ORDER BY interval ASC"},
+		// Separating the two readings of INTERVAL needs backtracking as per the current implementation which could have performance regressions.
+		// https://github.com/AfterShip/clickhouse-sql-parser/pull/296#issuecomment-5150316367
+		{"UnquotedIntervalRepeatedThirtyTimes", "SELECT interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval + interval AS total FROM t WHERE interval > 0 ORDER BY interval ASC"},
 		{"SignedLiteralAfterClosingParenUnspaced", "SELECT now() AS ts, toFloat64(count()) AS value FROM ( SELECT attributes_string['TableName'] AS T, attributes_string['MissingId'] AS M, max(fromUnixTimestamp64Nano(timestamp)) AS last_seen, dateDiff('minute', min(fromUnixTimestamp64Nano(timestamp)), max(fromUnixTimestamp64Nano(timestamp))) AS age_min FROM signoz_logs.distributed_logs_v2 WHERE body='missing_map_record' AND timestamp >= (toUnixTimestamp(now())-3600)*1000000000 GROUP BY T, M ) WHERE age_min >= 20 AND last_seen >= now() - toIntervalMinute(8)"},
 		{"SignedLiteralAfterClosingParenMinimal", "SELECT (1)-1"},
 		{"TrimFunction", "SELECT trimBoth('/api/endpoint/', '/');"},
+		// The SQL-standard keyword-separated argument forms, which took commas only. https://github.com/AfterShip/clickhouse-sql-parser/pull/290
+		{"StandardTrimSyntax", "SELECT trim(BOTH ' ' FROM body) FROM t"},
+		{"StandardSubstringSyntax", "SELECT substring(body FROM 2 FOR 3) FROM t"},
+		{"StandardOverlaySyntax", "SELECT overlay(body PLACING 'x' FROM 2) FROM t"},
+		// Row generators compute their rows from their arguments, so they read through nothing. This is the shape they get used for: a dense interval axis to CROSS JOIN a sparse series against.
+		{"NumbersTableFunction", "SELECT intervals.interval AS interval, active.cluster AS cluster, toFloat64(if(ts_data.has_data = 0, 0, 1)) AS value FROM ( SELECT DISTINCT JSONExtractString(labels, 'k8s.cluster.name') AS cluster FROM signoz_metrics.distributed_time_series_v4 WHERE metric_name = 'my_metric' AND unix_milli >= toUnixTimestamp(now() - INTERVAL 30 DAY) * 1000 HAVING cluster != '' ) AS active CROSS JOIN ( SELECT toStartOfInterval( toDateTime(toUnixTimestamp(now() - INTERVAL 30 MINUTE) + number * 60), INTERVAL 1 MINUTE ) AS interval FROM numbers(31) ) AS intervals LEFT JOIN ( SELECT toStartOfInterval( toDateTime(intDiv(s.unix_milli, 1000)), INTERVAL 1 MINUTE ) AS interval, JSONExtractString(ts.labels, 'k8s.cluster.name') AS cluster, 1 AS has_data FROM signoz_metrics.distributed_samples_v4 s INNER JOIN ( SELECT DISTINCT fingerprint, labels FROM signoz_metrics.distributed_time_series_v4 WHERE metric_name = 'my_metric' ) AS ts ON s.fingerprint = ts.fingerprint WHERE s.metric_name = 'my_metric' AND s.unix_milli >= toUnixTimestamp(now() - INTERVAL 30 MINUTE) * 1000 GROUP BY interval, cluster ) AS ts_data ON active.cluster = ts_data.cluster AND intervals.interval = ts_data.interval ORDER BY interval ASC"},
+		{"NumbersMtTableFunction", "SELECT * FROM numbers_mt(31)"},
+		{"ZerosTableFunction", "SELECT * FROM zeros(31)"},
+		{"ZerosMtTableFunction", "SELECT * FROM zeros_mt(31)"},
+		{"GenerateSeriesTableFunction", "SELECT * FROM generateSeries(1, 10)"},
+		{"GenerateSeriesSnakeCaseTableFunction", "SELECT * FROM generate_series(1, 10)"},
+		{"GeneratorTableFunctionUppercase", "SELECT * FROM NUMBERS(31)"},
+		{"GeneratorTableFunctionParenthesisedArgument", "SELECT * FROM NUMBERS((31))"},
+		{"GeneratorTableFunctionInJoin", "SELECT * FROM signoz_logs.distributed_logs_v2 AS l CROSS JOIN numbers(31) AS n"},
+		{"GeneratorTableFunctionInCommonTableExpression", "WITH axis AS (SELECT number FROM numbers(31)) SELECT * FROM axis"},
+		{"GeneratorTableFunctionInWhereSubquery", "SELECT * FROM t WHERE a IN (SELECT number FROM numbers(31))"},
+		{"GeneratorTableFunctionInUnion", "SELECT number FROM numbers(31) UNION ALL SELECT number FROM zeros(31)"},
 	}
 
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			err := ErrIfStatementIsNotValid(testCase.query)
-			assert.NoError(t, err)
+			// Bounded rather than called directly: a parser that backtracks without memoising
+			// hangs instead of returning. Every case here parses in well under a millisecond.
+			errC := make(chan error, 1)
+			go func() { errC <- ErrIfStatementIsNotValid(testCase.query) }()
+
+			select {
+			case err := <-errC:
+				assert.NoError(t, err)
+			case <-time.After(10 * time.Second):
+				assert.Fail(t, "timed out, which means the parser is no longer bounding its backtracking")
+			}
 		})
 	}
 }
@@ -70,6 +104,8 @@ func TestErrIfStatementIsNotValid_Fail(t *testing.T) {
 		{"CreateTable", "CREATE TABLE evil (a Int) ENGINE = Memory", CodeClickHouseSQLNotSelect},
 		{"Grant", "GRANT ALL ON *.* TO admin", CodeClickHouseSQLNotSelect},
 		{"Set", "SET readonly = 0", CodeClickHouseSQLNotSelect},
+		// The parser still dereferences nil on a DEFAULT expression it cannot read, so the recover is what turns this into a rejection rather than a crash.
+		{"UnparseableDefaultExpression", "CREATE TABLE t (a String DEFAULT foo(b FROM 2)) ENGINE = Memory", CodeClickHouseSQLParserPanic},
 		// These the parser rejects outright rather than classifying.
 		{"ShowGrants", "SHOW GRANTS", CodeClickHouseSQLUnparseable},
 		{"IntoOutfile", "SELECT * FROM t INTO OUTFILE '/tmp/x.csv'", CodeClickHouseSQLUnparseable},
@@ -81,6 +117,20 @@ func TestErrIfStatementIsNotValid_Fail(t *testing.T) {
 		{"TableFunctionInCommonTableExpression", "WITH c AS (SELECT * FROM url('http://x', CSV, 'a String')) SELECT * FROM c", CodeClickHouseSQLTableFunction},
 		{"TableFunctionInWhereSubquery", "SELECT * FROM t WHERE a IN (SELECT * FROM file('/etc/passwd', CSV, 'a String'))", CodeClickHouseSQLTableFunction},
 		{"TableFunctionInUnion", "SELECT * FROM t UNION ALL SELECT * FROM url('http://x', CSV, 'a String')", CodeClickHouseSQLTableFunction},
+		// These reach the internal databases without ever naming one, so the table-function rule is the only thing that sees them.
+		{"MergeTableFunction", "SELECT * FROM merge('system', '.*')", CodeClickHouseSQLTableFunction},
+		{"RemoteTableFunction", "SELECT * FROM remote('other-host', 'system.users')", CodeClickHouseSQLTableFunction},
+		{"ClusterTableFunction", "SELECT * FROM cluster('c', 'system.users')", CodeClickHouseSQLTableFunction},
+		// Pure, but excluded: generateRandom streams rows the arguments do not bound, and values has no use here that an array literal does not already cover.
+		{"GenerateRandomTableFunction", "SELECT * FROM generateRandom('a UInt64')", CodeClickHouseSQLTableFunction},
+		{"ValuesTableFunction", "SELECT * FROM values('a UInt64', 1, 2)", CodeClickHouseSQLTableFunction},
+		// Arguments are visited before the table function itself, so allowing a generator does not give anyone a wrapper to smuggle a read through.
+		{"InternalDatabaseInsideAllowedTableFunction", "SELECT * FROM numbers((SELECT count() FROM system.users))", CodeClickHouseSQLInternalDatabase},
+		{"InternalDatabaseJoinedOntoAllowedTableFunction", "SELECT * FROM numbers(31) AS n JOIN system.users AS u ON 1 = 1", CodeClickHouseSQLInternalDatabase},
+		{"InternalDatabaseUnionedWithAllowedTableFunction", "SELECT number FROM numbers(31) UNION ALL SELECT name FROM system.users", CodeClickHouseSQLInternalDatabase},
+		{"RefusedTableFunctionJoinedOntoAllowedTableFunction", "SELECT * FROM numbers(31) AS n JOIN url('http://x', CSV, 'a String') AS u ON 1 = 1", CodeClickHouseSQLTableFunction},
+		{"RefusedTableFunctionInsideAllowedTableFunction", "SELECT * FROM numbers((SELECT count() FROM file('/etc/passwd', CSV, 'a String')))", CodeClickHouseSQLTableFunction},
+		{"InternalDatabaseInsideAllowedTableFunctionCommonTableExpression", "WITH axis AS (SELECT * FROM numbers((SELECT count() FROM system.users))) SELECT * FROM axis", CodeClickHouseSQLInternalDatabase},
 		// Internal databases, which hold grants and server metadata rather than telemetry.
 		{"SystemUsers", "SELECT * FROM system.users", CodeClickHouseSQLInternalDatabase},
 		{"SystemUppercase", "SELECT * FROM SYSTEM.USERS", CodeClickHouseSQLInternalDatabase},
@@ -103,50 +153,3 @@ func TestErrIfStatementIsNotValid_Fail(t *testing.T) {
 		})
 	}
 }
-
-// Queries the parser cannot read. ClickHouse runs all of them.
-func TestErrIfStatementIsNotValid_ShouldPassButFails(t *testing.T) {
-	testCases := []struct {
-		name  string
-		query string
-		// The construct the parser stops after, which is the one it cannot read.
-		expectedStopsAfter string
-		// The same construct written so the parser accepts it.
-		fix string
-	}{
-		{
-			name:               "IntervalAliasInOrderBy",
-			query:              "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, resource_string_service$$name AS `service.name`, attributes_string['http.route'] AS `http.route`, quantile(0.95)(duration_nano) / 1000000000 AS value FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_string_service$$name = 'svc-a' AND resources_string['deployment.environment'] = 'dev' AND attributes_string['http.route'] = '/v1' AND http_method = 'POST' AND timestamp BETWEEN toDateTime(1784601720) AND toDateTime(1784602620) AND ts_bucket_start BETWEEN 1784601720 - 1800 AND 1784602620 GROUP BY `service.name`, `http.route`, interval ORDER BY interval ASC",
-			expectedStopsAfter: "ORDER BY interval ASC",
-			fix:                "SELECT count() AS interval FROM t ORDER BY `interval` ASC",
-		},
-		{
-			name:               "IntervalAliasInOrderByDesc",
-			query:              "SELECT count() AS value, toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, serviceName, resourceTagsMap['deployment.environment'] AS environment, exceptionStacktrace FROM signoz_traces.distributed_signoz_error_index_v2 WHERE exceptionType != 'OSError' AND resourceTagsMap['deployment.environment'] = 'staging' AND timestamp BETWEEN toDateTime(1785186300) AND toDateTime(1785186600) GROUP BY serviceName, interval, environment, exceptionStacktrace ORDER BY interval DESC",
-			expectedStopsAfter: "ORDER BY interval DESC",
-			fix:                "SELECT count() AS interval FROM t ORDER BY `interval` DESC",
-		},
-		{
-			name:               "StandardTrimSyntax",
-			query:              "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 5 MINUTE) AS interval, resources_string['host.name'] as host_name, toFloat64(countIf( lower(trim(BOTH ' ' FROM replaceOne( JSONExtractString(body, 'Action'), 'health_status: ', '' ))) IN ('unhealthy','starting','failing') )) as value FROM signoz_logs.distributed_logs_v2 WHERE timestamp BETWEEN 1784602320000000000 AND 1784602620000000000 AND ts_bucket_start BETWEEN 1784602320 - 300 AND 1784602620 AND JSONExtractString(body, 'Type') = 'container' AND JSONExtractString(body, 'Actor', 'Attributes', 'name') IS NOT NULL AND resources_string['host.name'] IS NOT NULL AND resources_string['host.name'] = 'aihub-nightly' GROUP BY interval, host_name ORDER BY interval, host_name",
-			expectedStopsAfter: "trim(BOTH '",
-			fix:                "SELECT trimBoth(replaceOne( JSONExtractString(body, 'Action'), 'health_status: ', '' ), ' ')",
-		},
-	}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			err := ErrIfStatementIsNotValid(testCase.query)
-
-			var parseErr *chparser.ParseError
-			require.ErrorAs(t, err, &parseErr, "expected a parser failure rather than a rule violation")
-
-			// The parser reports the offset it stopped at, which sits just past the construct
-			// it choked on, so the text leading up to it is what needs looking at.
-			consumed := testCase.query[:parseErr.Pos]
-			assert.Equal(t, testCase.expectedStopsAfter, consumed[max(0, len(consumed)-len(testCase.expectedStopsAfter)):])
-
-			assert.NoError(t, ErrIfStatementIsNotValid(testCase.fix))
-		})
-	}
-}

pkg/signoz/provider.go

@@ -45,7 +45,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/pprof/nooppprof"
 	"github.com/SigNoz/signoz/pkg/prometheus"
 	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheus"
-	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/querier/signozquerier"
 	"github.com/SigNoz/signoz/pkg/sharder"
@@ -249,7 +248,6 @@ func NewTelemetryStoreProviderFactories() factory.NamedMap[factory.ProviderFacto
 func NewPrometheusProviderFactories(telemetryStore telemetrystore.TelemetryStore) factory.NamedMap[factory.ProviderFactory[prometheus.Prometheus, prometheus.Config]] {
 	return factory.MustNewNamedMap(
 		clickhouseprometheus.NewFactory(telemetryStore),
-		clickhouseprometheusv2.NewFactory(telemetryStore),
 	)
 }
 
@@ -291,9 +289,9 @@ func NewStatsReporterProviderFactories(aggregator statsreporter.Aggregator, orgG
 	)
 }
 
-func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, promV2 prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
+func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, metadataStore telemetrytypes.MetadataStore, traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation], logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation], metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation], traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder, bucketCache querier.BucketCache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
 	return factory.MustNewNamedMap(
-		signozquerier.NewFactory(telemetryStore, prometheus, promV2, metadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
+		signozquerier.NewFactory(telemetryStore, prometheus, metadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
 	)
 }
 

pkg/signoz/signoz.go

@@ -40,7 +40,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/modules/tag/impltag"
 	"github.com/SigNoz/signoz/pkg/modules/user/impluser"
 	"github.com/SigNoz/signoz/pkg/prometheus"
-	"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/queryparser"
 	"github.com/SigNoz/signoz/pkg/ruler"
@@ -300,11 +299,6 @@ func New(
 
 	retentionGetter := implretention.NewGetter(implretention.NewStore(sqlstore))
 
-	// promV2 is the clickhousev2 provider handed to the querier for shadow
-	// comparison and pinned serving (declared before the serving provider,
-	// whose variable shadows the package name below).
-	var promV2 prometheus.Prometheus
-
 	// Initialize prometheus from the available prometheus provider factories
 	prometheus, err := factory.NewProviderFromNamedMap(
 		ctx,
@@ -317,23 +311,6 @@ func New(
 		return nil, err
 	}
 
-	// With the default provider, also stand up the clickhousev2 provider for
-	// the querier: PromQL queries shadow-compare against it behind the
-	// use_prometheus_clickhouse_v2 flag (see pkg/querier/promql_shadow.go).
-	// It never serves by default. An explicit
-	// prometheus::provider: clickhousev2 makes v2 the serving provider
-	// outright, so there is nothing to compare against.
-	if config.Prometheus.Provider() == "clickhouse" {
-		v2Config := config.Prometheus
-		// The v2 engine only evaluates shadow and pinned queries; disable its
-		// active query tracker so two trackers never share a file.
-		v2Config.ActiveQueryTrackerConfig.Enabled = false
-		promV2, err = clickhouseprometheusv2.New(ctx, providerSettings, v2Config, telemetrystore)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	// Assemble the query stack (metadata store, statement builders, bucket cache) once,
 	// and reuse the single metadata store everywhere downstream.
 	telemetryMetadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, err := newQueryStack(ctx, providerSettings, config, telemetrystore, cache, flagger)
@@ -346,7 +323,7 @@ func New(
 		ctx,
 		providerSettings,
 		config.Querier,
-		NewQuerierProviderFactories(telemetrystore, prometheus, promV2, telemetryMetadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
+		NewQuerierProviderFactories(telemetrystore, prometheus, telemetryMetadataStore, traceStmtBuilder, logStmtBuilder, auditStmtBuilder, metricStmtBuilder, meterStmtBuilder, traceOperatorStmtBuilder, bucketCache, flagger),
 		config.Querier.Provider(),
 	)
 	if err != nil {

pkg/types/querybuildertypes/querybuildertypesv5/req.go

@@ -370,14 +370,6 @@ type QueryRangeRequest struct {
 	// NoCache is a flag to disable caching for the request.
 	NoCache bool `json:"noCache,omitempty"`
 
-	// PromQLProvider serves this request's PromQL queries via the named
-	// prometheus provider ("clickhousev2") instead of the default — the same
-	// data read through a different implementation. It is set from the
-	// X-SigNoz-PromQL-Provider header by the API handler, never from the
-	// body: a rollout-scoped comparison hook for integration tests and
-	// support should not become part of the public request schema.
-	PromQLProvider string `json:"-"`
-
 	FormatOptions *FormatOptions `json:"formatOptions,omitempty"`
 }
 

scripts/promqltestcorpus/go.mod

@@ -76,14 +76,14 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
-	golang.org/x/crypto v0.50.0 // indirect
+	golang.org/x/crypto v0.52.0 // indirect
 	golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
-	golang.org/x/net v0.53.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/sys v0.43.0 // indirect
-	golang.org/x/term v0.42.0 // indirect
-	golang.org/x/text v0.36.0 // indirect
+	golang.org/x/sys v0.45.0 // indirect
+	golang.org/x/term v0.43.0 // indirect
+	golang.org/x/text v0.37.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
 	google.golang.org/api v0.272.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect

scripts/promqltestcorpus/go.sum

@@ -377,29 +377,29 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
 go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
-golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
-golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
-golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
-golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
+golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
+golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
-golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
-golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
+golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
+golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
+golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
+golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
-golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
+golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/api v0.272.0 h1:eLUQZGnAS3OHn31URRf9sAmRk3w2JjMx37d2k8AjJmA=

tests/conftest.py

@@ -1,3 +1,5 @@
+import subprocess
+
 import pytest
 
 pytest_plugins = [
@@ -32,6 +34,22 @@ pytest_plugins = [
 ]
 
 
+def pytest_configure(config: pytest.Config):
+    if config.getoption("--rebuild"):
+        if not config.getoption("--reuse"):
+            raise pytest.UsageError("--rebuild requires --reuse: it replaces the signoz container within an environment that is being reused.")
+        if config.getoption("--teardown"):
+            raise pytest.UsageError("--rebuild cannot be combined with --teardown.")
+        if config.getoption("--clean"):
+            raise pytest.UsageError("--rebuild cannot be combined with --clean: --clean forces a cold build, which defeats the purpose of --rebuild.")
+
+
+def pytest_sessionstart(session: pytest.Session):
+    if session.config.getoption("--clean"):
+        # The type filter removes only cache mounts, leaving images and layer cache intact.
+        subprocess.run(["docker", "builder", "prune", "--force", "--filter", "type=exec.cachemount"], check=True)
+
+
 def pytest_addoption(parser: pytest.Parser):
     parser.addoption(
         "--reuse",
@@ -45,6 +63,18 @@ def pytest_addoption(parser: pytest.Parser):
         default=False,
         help="Teardown environment. Run pytest --basetemp=./tmp/ -vv --teardown src/bootstrap/setup::test_teardown to teardown your local dev environment.",
     )
+    parser.addoption(
+        "--rebuild",
+        action="store_true",
+        default=False,
+        help="Rebuild the signoz container from the current sources while reusing the rest of the stack (databases, mocks, migrations). Only meaningful together with --reuse: pytest --basetemp=./tmp/ -vv --reuse --rebuild integration/bootstrap/setup.py::test_setup.",
+    )
+    parser.addoption(
+        "--clean",
+        action="store_true",
+        default=False,
+        help="Prune the BuildKit cache mounts (go build and module caches) used by the signoz image build, forcing the next build to start cold. Combine with --teardown to reset everything: pytest --basetemp=./tmp/ -vv --teardown --clean integration/bootstrap/setup.py::test_teardown.",
+    )
     parser.addoption(
         "--with-web",
         action="store_true",

tests/fixtures/querier.py

@@ -168,7 +168,6 @@ def make_query_request(
     variables: dict | None = None,
     no_cache: bool = True,
     timeout: int = QUERY_TIMEOUT,
-    headers: dict | None = None,
 ) -> requests.Response:
     if format_options is None:
         format_options = {"formatTableResultForUI": False, "fillGaps": False}
@@ -188,7 +187,7 @@ def make_query_request(
     return requests.post(
         signoz.self.host_configs["8080"].get("/api/v5/query_range"),
         timeout=timeout,
-        headers={"authorization": f"Bearer {token}", **(headers or {})},
+        headers={"authorization": f"Bearer {token}"},
         json=payload,
     )
 

tests/fixtures/reuse.py

@@ -41,6 +41,7 @@ def wrap(  # pylint: disable=too-many-arguments,too-many-positional-arguments
     create: Callable[[], T],
     delete: Callable[[T], None],
     restore: Callable[[dict], T],
+    rebuild: bool = False,
 ) -> T:
     """
     Wraps a resource creation and cleanup process with reuse and teardown options.
@@ -51,6 +52,7 @@ def wrap(  # pylint: disable=too-many-arguments,too-many-positional-arguments
     - create: function to create the resource
     - delete: function to delete the resource
     - restore: function to restore resource from cache
+    - rebuild: under --reuse, delete the cached resource and recreate it instead of restoring it
     """
     resource = empty()
 
@@ -58,8 +60,13 @@ def wrap(  # pylint: disable=too-many-arguments,too-many-positional-arguments
         existing_resource = pytestconfig.cache.get(key, None)
         if existing_resource:
             assert isinstance(existing_resource, dict)
-            logger.info("Reusing existing %s(%s)", key, existing_resource)
-            return restore(existing_resource)
+            if rebuild:
+                logger.info("Rebuilding %s(%s), removing the existing one", key, existing_resource)
+                delete(restore(existing_resource))
+                pytestconfig.cache.set(key, None)
+            else:
+                logger.info("Reusing existing %s(%s)", key, existing_resource)
+                return restore(existing_resource)
 
     if not teardown(request):
         resource = create()

tests/fixtures/signoz.py

@@ -1,4 +1,6 @@
+import os
 import platform
+import subprocess
 import time
 from http import HTTPStatus
 from os import path
@@ -8,7 +10,6 @@ import docker.errors
 import pytest
 import requests
 from testcontainers.core.container import DockerContainer, Network
-from testcontainers.core.image import DockerImage
 
 from fixtures import reuse, types
 from fixtures.logger import setup_logger
@@ -50,17 +51,27 @@ def create_signoz(
 
         # Docker build context is the repo root — one up from pytest's
         # rootdir (tests/).
-        self = DockerImage(
-            path=str(pytestconfig.rootpath.parent),
-            dockerfile_path=dockerfile_path,
-            tag="signoz:integration",
-            buildargs={
-                "TARGETARCH": arch,
-                "ZEUSURL": zeus.container_configs["8080"].base(),
-            },
-        )
+        context = pytestconfig.rootpath.parent
 
-        self.build()
+        # The docker CLI is required: the Dockerfiles use BuildKit cache
+        # mounts, which docker-py does not support.
+        subprocess.run(
+            [
+                "docker",
+                "build",
+                "--file",
+                str(context / dockerfile_path),
+                "--tag",
+                "signoz:integration",
+                "--build-arg",
+                f"TARGETARCH={arch}",
+                "--build-arg",
+                f"ZEUSURL={zeus.container_configs['8080'].base()}",
+                str(context),
+            ],
+            check=True,
+            env=os.environ | {"DOCKER_BUILDKIT": "1"},
+        )
 
         env = (
             {
@@ -200,6 +211,7 @@ def create_signoz(
         create=create,
         delete=delete,
         restore=restore,
+        rebuild=pytestconfig.getoption("--rebuild"),
     )
 
 

tests/integration/testdata/promqltestcorpus/known_divergences_v2.json

@@ -1,4 +0,0 @@
-{
-  "note": "Divergences of the clickhousev2 provider (pinned via X-SigNoz-PromQL-Provider) from the upstream reference engine, enforced exactly by 01_upstream_corpus.py in both directions. This ledger is the rollout scorecard for the provider swap: the default provider cannot be replaced by clickhousev2 while anything is listed here. Entries must carry the defect's cause and be REMOVED as the provider is fixed.",
-  "divergences": {}
-}

tests/integration/tests/promqlconformance/01_upstream_corpus.py

@@ -3,26 +3,13 @@ Upstream promqltest conformance: replay the frozen corpus extracted from
 Prometheus' own promql/promqltest testdata and assert our API returns the
 reference engine's answers.
 
-Unlike live-vs-live parity suites, the oracle here is a committed file
+Unlike the parity suites, the oracle here is a committed file
 (tests/integration/testdata/promqltestcorpus/corpus.json), generated by
 scripts/promqltestcorpus from upstream's load scripts and the vendored
 reference engine. It therefore keeps working when the serving path itself is
 the thing being changed — the one situation where comparing two live paths
 against each other is blind.
 
-Every case replays on both serving paths — the default provider, and the
-clickhousev2 provider pinned via the flag-gated X-SigNoz-PromQL-Provider
-header (see conftest.py) — and each leg is asserted against the same frozen
-expectations, each leg against its own known-divergences ledger. The legs
-are deliberately never asserted against each other: both can sit within one
-rounding quantum of the expected value yet differ from each other by up to
-two quanta when a true value straddles a rounding boundary, so a leg-vs-leg
-equality check would reintroduce exactly the boundary noise the quantum
-tolerance exists to absorb. Because both legs anchor to the same oracle over
-the same ingested bytes, a case failing on one leg while passing on the
-other already localizes the defect to that provider — and the printed
-DIVERGED lines for both legs are the side-by-side view for triage.
-
 Datasets are placed on disjoint time windows (2h isolation gaps, far beyond
 the 5m lookback) so one bulk ingest serves every case without cross-talk.
 Expected values carry the API's 3-significant-decimal rounding, mirrored by
@@ -44,30 +31,12 @@ from fixtures.querier import get_all_series, make_query_request
 
 TESTDATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "testdata")
 CORPUS_FILE = os.path.join(TESTDATA_DIR, "promqltestcorpus", "corpus.json")
-
-# One ledger per leg, enforced exactly in both directions. The default leg's
-# ledger is empty and pinned there; the clickhousev2 ledger is the rollout
-# scorecard — the provider swap is measured by burning it down to empty.
-LEDGER_FILES = {
-    "default": os.path.join(TESTDATA_DIR, "promqltestcorpus", "known_divergences.json"),
-    "clickhousev2": os.path.join(TESTDATA_DIR, "promqltestcorpus", "known_divergences_v2.json"),
-}
-
-LEGS: list[tuple[str, dict | None]] = [
-    ("default", None),
-    ("clickhousev2", {"X-SigNoz-PromQL-Provider": "clickhousev2"}),
-]
+KNOWN_DIVERGENCES_FILE = os.path.join(TESTDATA_DIR, "promqltestcorpus", "known_divergences.json")
 
 ISOLATION_GAP_MS = 2 * 3600 * 1000
 SPECIALS = {"NaN": math.nan, "Inf": math.inf, "-Inf": -math.inf}
 
 
-def _decode(v: float | str) -> float:
-    if isinstance(v, str):
-        return SPECIALS[v]
-    return float(v)
-
-
 def _values_close(a: float, b: float) -> bool:
     if math.isnan(a) or math.isnan(b):
         return math.isnan(a) and math.isnan(b)
@@ -90,10 +59,6 @@ def _values_close(a: float, b: float) -> bool:
     return abs(a - b) <= quantum + 1e-12
 
 
-def _labelset(labels: dict[str, str]) -> tuple:
-    return tuple(sorted(labels.items()))
-
-
 def _response_series(data: dict) -> tuple[dict[tuple, dict[int, float]], list[tuple]]:
     """Returns (series map, duplicate labelsets). A response carrying several
     series with identical visible labels is itself a defect signal (e.g. a
@@ -104,70 +69,14 @@ def _response_series(data: dict) -> tuple[dict[tuple, dict[int, float]], list[tu
     # Empty results serialize with null aggregations/series/values fields.
     for series in get_all_series(data, "A") or []:
         lbls = {l["key"]["name"]: str(l["value"]) for l in series.get("labels") or []}
-        points = {int(v["timestamp"]): _decode(v["value"]) for v in series.get("values") or []}
-        key = _labelset(lbls)
+        points = {int(v["timestamp"]): SPECIALS[v["value"]] if isinstance(v["value"], str) else float(v["value"]) for v in series.get("values") or []}
+        key = tuple(sorted(lbls.items()))
         if key in out:
             duplicates.append(key)
         out[key] = points
     return out, duplicates
 
 
-def _case_failure(
-    signoz: types.SigNoz,
-    token: str,
-    case: dict,
-    base: int,
-    headers: dict | None,
-) -> str | None:
-    """Replays one corpus case on one leg; returns a failure line or None."""
-    start_ms = base + case["start_ms"]
-    end_ms = base + case["end_ms"]
-    step_s = max(1, case["step_ms"] // 1000)
-    req_start_ms = start_ms
-    if case["instant"]:
-        # The API rejects start == end; ask for one extra step backward
-        # and compare only at the instant timestamp. Nudging the start
-        # earlier instead of the end later keeps every window that the
-        # expected values were computed from untouched.
-        req_start_ms = start_ms - step_s * 1000
-    query = {
-        "type": "promql",
-        "spec": {"name": "A", "query": case["expr"], "step": step_s},
-    }
-
-    case_id = f"{case['source']}[{case['variant']}]"
-    response = make_query_request(signoz, token, req_start_ms, end_ms, [query], headers=headers)
-    if response.status_code != HTTPStatus.OK:
-        return f"{case_id}: HTTP {response.status_code} for {case['expr']!r}: {response.text[:200]}"
-
-    actual, duplicates = _response_series(response.json())
-    if duplicates:
-        return f"{case_id}: response carries multiple series with identical labels for {case['expr']!r}: {[dict(d) for d in duplicates[:3]]}"
-    if case["instant"]:
-        # Keep only the instant point; the extra grid step is a request
-        # encoding byproduct, not part of the assertion.
-        actual = {lset: {ts: v for ts, v in pts.items() if ts == end_ms} for lset, pts in actual.items()}
-        actual = {lset: pts for lset, pts in actual.items() if pts}
-    expected: dict[tuple, dict[int, float]] = {}
-    for res in case["expected"]:
-        points = {base + off_ms: _decode(v) for off_ms, v in res["points"]}
-        expected[_labelset(res["labels"])] = points
-
-    if set(actual) != set(expected):
-        missing = set(expected) - set(actual)
-        extra = set(actual) - set(expected)
-        return f"{case_id}: series mismatch for {case['expr']!r} (missing={sorted(missing)[:3]} extra={sorted(extra)[:3]}) actual={[(dict(k), {t - base: v for t, v in pts.items()}) for k, pts in actual.items()]}"
-
-    for lset, exp_points in expected.items():
-        act_points = actual[lset]
-        if set(act_points) != set(exp_points):
-            return f"{case_id}: timestamp mismatch for {case['expr']!r} series {dict(lset)} (expected {len(exp_points)} points, got {len(act_points)})"
-        for ts, exp_v in exp_points.items():
-            if not _values_close(act_points[ts], exp_v):
-                return f"{case_id}: value mismatch for {case['expr']!r} series {dict(lset)} at {ts}: expected {exp_v}, got {act_points[ts]}"
-    return None
-
-
 def test_upstream_promqltest_corpus(
     signoz: types.SigNoz,
     create_user_admin: None,  # pylint: disable=unused-argument
@@ -212,7 +121,7 @@ def test_upstream_promqltest_corpus(
                         metric_name=metric_name,
                         labels=labels,
                         timestamp=datetime.fromtimestamp((cursor + off_ms) / 1000, tz=UTC),
-                        value=0.0 if stale else _decode(raw),
+                        value=0.0 if stale else (SPECIALS[raw] if isinstance(raw, str) else float(raw)),
                         flags=1 if stale else 0,
                     )
                 )
@@ -221,36 +130,79 @@ def test_upstream_promqltest_corpus(
     insert_metrics(metrics)
     token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
 
-    failures: dict[str, list[str]] = {leg: [] for leg, _ in LEGS}
+    failures: list[str] = []
     for case in corpus["cases"]:
-        for leg, headers in LEGS:
-            f_line = _case_failure(signoz, token, case, bases[case["dataset"]], headers)
-            if f_line:
-                failures[leg].append(f_line)
+        base = bases[case["dataset"]]
+        start_ms = base + case["start_ms"]
+        end_ms = base + case["end_ms"]
+        step_s = max(1, case["step_ms"] // 1000)
+        req_start_ms = start_ms
+        if case["instant"]:
+            # The API rejects start == end; ask for one extra step backward
+            # and compare only at the instant timestamp. Nudging the start
+            # earlier instead of the end later keeps every window that the
+            # expected values were computed from untouched.
+            req_start_ms = start_ms - step_s * 1000
+        query = {
+            "type": "promql",
+            "spec": {"name": "A", "query": case["expr"], "step": step_s},
+        }
 
-    for leg, _ in LEGS:
-        for f_line in failures[leg]:
-            print("DIVERGED", f"[{leg}]", f_line)
+        case_id = f"{case['source']}[{case['variant']}]"
+        response = make_query_request(signoz, token, req_start_ms, end_ms, [query])
+        if response.status_code != HTTPStatus.OK:
+            failures.append(f"{case_id}: HTTP {response.status_code} for {case['expr']!r}: {response.text[:200]}")
+            continue
 
-    # Known divergences are defects of that leg's serving path, frozen with
-    # reasons. Each set is enforced exactly in both directions: a NEW
+        actual, duplicates = _response_series(response.json())
+        if duplicates:
+            failures.append(f"{case_id}: response carries multiple series with identical labels for {case['expr']!r}: {[dict(d) for d in duplicates[:3]]}")
+            continue
+        if case["instant"]:
+            # Keep only the instant point; the extra grid step is a request
+            # encoding byproduct, not part of the assertion.
+            actual = {lset: {ts: v for ts, v in pts.items() if ts == end_ms} for lset, pts in actual.items()}
+            actual = {lset: pts for lset, pts in actual.items() if pts}
+        expected: dict[tuple, dict[int, float]] = {}
+        for res in case["expected"]:
+            points = {base + off_ms: SPECIALS[v] if isinstance(v, str) else float(v) for off_ms, v in res["points"]}
+            expected[tuple(sorted(res["labels"].items()))] = points
+
+        if set(actual) != set(expected):
+            missing = set(expected) - set(actual)
+            extra = set(actual) - set(expected)
+            failures.append(f"{case_id}: series mismatch for {case['expr']!r} (missing={sorted(missing)[:3]} extra={sorted(extra)[:3]}) actual={[(dict(k), {t - base: v for t, v in pts.items()}) for k, pts in actual.items()]}")
+            continue
+
+        for lset, exp_points in expected.items():
+            act_points = actual[lset]
+            if set(act_points) != set(exp_points):
+                failures.append(f"{case_id}: timestamp mismatch for {case['expr']!r} series {dict(lset)} (expected {len(exp_points)} points, got {len(act_points)})")
+                break
+            for ts, exp_v in exp_points.items():
+                if not _values_close(act_points[ts], exp_v):
+                    failures.append(f"{case_id}: value mismatch for {case['expr']!r} series {dict(lset)} at {ts}: expected {exp_v}, got {act_points[ts]}")
+                    break
+            else:
+                continue
+            break
+
+    for f_line in failures:
+        print("DIVERGED", f_line)
+
+    # Known divergences are defects of the current serving path, frozen with
+    # reasons. The set is enforced exactly in both directions: a NEW
     # divergence is a regression, and a known divergence that starts passing
-    # must be removed from the file. Problems across both legs are collected
-    # before asserting so one leg's failure never hides the other's.
-    problems: list[str] = []
-    for leg, _ in LEGS:
-        known: dict[str, str] = {}
-        if os.path.exists(LEDGER_FILES[leg]):
-            with open(LEDGER_FILES[leg], encoding="utf-8") as f:
-                known = json.load(f)["divergences"]
+    # must be removed from the file — that is the ledger the serving-path
+    # swap is measured against.
+    known: dict[str, str] = {}
+    if os.path.exists(KNOWN_DIVERGENCES_FILE):
+        with open(KNOWN_DIVERGENCES_FILE, encoding="utf-8") as f:
+            known = json.load(f)["divergences"]
 
-        failed_ids = {f_line.split(": ", 1)[0] for f_line in failures[leg]}
-        unexpected = [f_line for f_line in failures[leg] if f_line.split(": ", 1)[0] not in known]
-        now_passing = sorted(set(known) - failed_ids)
+    failed_ids = {f_line.split(": ", 1)[0] for f_line in failures}
+    unexpected = [f_line for f_line in failures if f_line.split(": ", 1)[0] not in known]
+    now_passing = sorted(set(known) - failed_ids)
 
-        if unexpected:
-            problems.append(f"[{leg}] {len(unexpected)} corpus cases diverged beyond the known set:\n" + "\n".join(unexpected[:25]))
-        if now_passing:
-            problems.append(f"[{leg}] {len(now_passing)} known divergences now pass — remove them from {os.path.basename(LEDGER_FILES[leg])}: {now_passing[:25]}")
-
-    assert not problems, "\n\n".join(problems)
+    assert not unexpected, f"{len(unexpected)} corpus cases diverged beyond the known set:\n" + "\n".join(unexpected[:25])
+    assert not now_passing, f"{len(now_passing)} known divergences now pass — remove them from known_divergences.json: {now_passing[:25]}"

tests/integration/tests/promqlconformance/conftest.py

@@ -1,39 +0,0 @@
-import pytest
-from testcontainers.core.container import Network
-
-from fixtures import types
-from fixtures.signoz import create_signoz
-
-
-@pytest.fixture(name="signoz", scope="package")
-def signoz_promql_conformance(
-    network: Network,
-    migrator: types.Operation,  # pylint: disable=unused-argument
-    zeus: types.TestContainerDocker,
-    gateway: types.TestContainerDocker,
-    sqlstore: types.TestContainerSQL,
-    clickhouse: types.TestContainerClickhouse,
-    request: pytest.FixtureRequest,
-    pytestconfig: pytest.Config,
-) -> types.SigNoz:
-    """
-    Package-scoped SigNoz with use_prometheus_clickhouse_v2 on, so the corpus
-    can replay every case twice: once against the default provider and once
-    pinned to the clickhousev2 provider via the X-SigNoz-PromQL-Provider
-    header (which the flag gates). Each leg is asserted against the same
-    frozen expectations — see 01_upstream_corpus.py for why the legs are
-    never asserted against each other.
-    """
-    return create_signoz(
-        network=network,
-        zeus=zeus,
-        gateway=gateway,
-        sqlstore=sqlstore,
-        clickhouse=clickhouse,
-        request=request,
-        pytestconfig=pytestconfig,
-        cache_key="signoz-promql-conformance",
-        env_overrides={
-            "SIGNOZ_FLAGGER_CONFIG_BOOLEAN_USE__PROMETHEUS__CLICKHOUSE__V2": True,
-        },
-    )