fix(e2e): guard unmapped spec folders and share the binary cache key

.github/workflows/cacheci.yml

@@ -46,8 +46,34 @@ jobs:
           docker build -f "$RUNNER_TEMP/inject.Dockerfile" "$RUNNER_TEMP/cacheci"
       - name: build
         run: |
-          docker build -f cmd/enterprise/Dockerfile.integration --build-arg TARGETARCH=amd64 --build-arg ZEUSURL=http://zeus:8080 .
+          docker build -f cmd/enterprise/Dockerfile.integration --tag signoz:integration --build-arg TARGETARCH=amd64 --build-arg ZEUSURL=http://zeus:8080 .
           docker build -f cmd/enterprise/Dockerfile.with-web.integration --build-arg TARGETARCH=amd64 --build-arg ZEUSURL=http://zeus:8080 .
+      # e2eci runs this server binary for changes that leave the server as it
+      # is here, and builds only the web.
+      - name: e2e-signoz-key
+        id: e2e-signoz-key
+        run: |
+          echo "key=$(tests/e2e/scripts/signoz-key.sh)" >> "$GITHUB_OUTPUT"
+      - name: e2e-signoz
+        run: |
+          mkdir -p "$RUNNER_TEMP/e2e-signoz"
+          id=$(docker create signoz:integration)
+          docker cp "$id":/root/signoz "$RUNNER_TEMP/e2e-signoz/signoz"
+          docker rm "$id"
+      - name: save-e2e-signoz
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ runner.temp }}/e2e-signoz
+          key: ${{ steps.e2e-signoz-key.outputs.key }}
+      # Older keys only match commits main has moved past.
+      - name: prune-e2e-signoz
+        env:
+          GH_TOKEN: ${{ github.token }}
+          KEY: ${{ steps.e2e-signoz-key.outputs.key }}
+        run: |
+          gh cache list --repo "$GITHUB_REPOSITORY" --key e2e-signoz- --json key --jq '.[].key' \
+            | grep -vxF "$KEY" \
+            | xargs -r -n 1 gh cache delete --repo "$GITHUB_REPOSITORY"
       # docker cp instead of --output type=local (the local exporter stalls on
       # multi-GB outputs); tarballs instead of raw trees so the host never hits
       # the permission and symlink semantics that broke docker cp.

.github/workflows/e2eci.yaml

@@ -12,6 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       e2e: ${{ steps.changes.outputs.e2e }}
+      shards: ${{ steps.plan.outputs.shards }}
     steps:
       - name: changes
         id: changes
@@ -21,6 +22,7 @@ jobs:
           FORCE: ${{ contains(github.event.pull_request.labels.*.name, 'safe-to-e2e') }}
         run: |
           CHANGED_FILES=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename | select(startswith("tests/integration/") | not)')
+          printf '%s\n' "$CHANGED_FILES" > "$RUNNER_TEMP/changed.txt"
           PATTERN='^(frontend/|tests/|\.github/workflows/e2eci\.yaml$|cmd/enterprise/Dockerfile\.with-web\.integration$)'
 
           if [ "$FORCE" = "true" ]; then
@@ -30,6 +32,42 @@ jobs:
           else
             echo "e2e=false" >> "$GITHUB_OUTPUT"
           fi
+      - name: checkout
+        if: steps.changes.outputs.e2e == 'true'
+        uses: actions/checkout@v4
+      - name: install-pnpm
+        if: steps.changes.outputs.e2e == 'true'
+        uses: pnpm/action-setup@v6
+        with:
+          version: 10
+      - name: node
+        if: steps.changes.outputs.e2e == 'true'
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - name: pnpm-install
+        if: steps.changes.outputs.e2e == 'true'
+        run: |
+          cd tests/e2e && pnpm install --frozen-lockfile
+      - name: plan
+        id: plan
+        if: steps.changes.outputs.e2e == 'true'
+        env:
+          FORCE: ${{ contains(github.event.pull_request.labels.*.name, 'safe-to-e2e') }}
+        run: |
+          cd tests/e2e
+          if [ "$FORCE" = "true" ]; then
+            node scripts/plan.mjs --all < /dev/null
+          else
+            node scripts/plan.mjs < "$RUNNER_TEMP/changed.txt"
+          fi
+      - name: upload-plan
+        if: steps.changes.outputs.e2e == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-plan
+          path: tests/e2e/artifacts/plan/
+          retention-days: 5
   fmtlint:
     needs: changes
     if: needs.changes.outputs.e2e == 'true'
@@ -40,7 +78,7 @@ jobs:
       - name: node
         uses: actions/setup-node@v4
         with:
-          node-version: lts/*
+          node-version: 22
       - name: install-pnpm
         uses: pnpm/action-setup@v6
         with:
@@ -60,9 +98,9 @@ jobs:
       matrix:
         project:
           - chromium
-        shard: [1, 2, 3, 4]
+        shard: ${{ fromJSON(needs.changes.outputs.shards) }}
     needs: changes
-    if: needs.changes.outputs.e2e == 'true'
+    if: needs.changes.outputs.e2e == 'true' && needs.changes.outputs.shards != '[]'
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -95,10 +133,11 @@ jobs:
         background: true
         run: |
           cd tests && uv sync
+      # The frontend builds on Node 22, as in jsci and the with-web image.
       - name: node
         uses: actions/setup-node@v4
         with:
-          node-version: lts/*
+          node-version: 22
       - name: install-pnpm
         uses: pnpm/action-setup@v6
         with:
@@ -106,10 +145,44 @@ jobs:
       - name: pnpm-install
         run: |
           cd tests/e2e && pnpm install --frozen-lockfile
+      - name: signoz-key
+        id: signoz-key
+        run: |
+          echo "key=$(tests/e2e/scripts/signoz-key.sh)" >> "$GITHUB_OUTPUT"
+      # A hit means the change leaves the server as it is on main, so only the
+      # web build is new.
+      - name: signoz-binary
+        id: signoz-binary
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ runner.temp }}/e2e-signoz
+          key: ${{ steps.signoz-key.outputs.key }}
+      - name: signoz-image
+        id: signoz-image
+        if: steps.signoz-binary.outputs.cache-hit == 'true'
+        background: true
+        env:
+          NODE_OPTIONS: --max-old-space-size=8192
+        run: |
+          cd frontend && pnpm install --frozen-lockfile && pnpm build
+          cp -r build "$RUNNER_TEMP/e2e-signoz/web"
+          cp -r ../templates "$RUNNER_TEMP/e2e-signoz/templates"
+          cat > "$RUNNER_TEMP/e2e-signoz/Dockerfile" <<'EOF'
+          FROM golang:1.25-bookworm
+          ENV HOME=/root
+          WORKDIR /root
+          COPY signoz /root/signoz
+          COPY templates /root/templates
+          COPY web /root/web
+          RUN chmod 755 /root /root/signoz
+          ENTRYPOINT ["/root/signoz", "server"]
+          EOF
+          docker build --tag signoz:integration "$RUNNER_TEMP/e2e-signoz"
       # Restore-only: the cacheci workflow owns cache saves. Seeds the
       # BuildKit cache mounts so the in-test image build is incremental.
       - name: restore
         id: restore
+        if: steps.signoz-binary.outputs.cache-hit != 'true'
         uses: actions/cache/restore@v4
         with:
           path: ${{ runner.temp }}/cacheci
@@ -117,7 +190,7 @@ jobs:
           restore-keys: |
             tests-secondary
       - name: inject
-        if: steps.restore.outputs.cache-matched-key != ''
+        if: steps.signoz-binary.outputs.cache-hit != 'true' && steps.restore.outputs.cache-matched-key != ''
         run: |
           cat > "$RUNNER_TEMP/inject.Dockerfile" <<'EOF'
           FROM busybox:1.37
@@ -143,19 +216,30 @@ jobs:
           docker rm pw
           echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" >> "$GITHUB_ENV"
           cd tests/e2e && pnpm playwright install-deps ${{ matrix.project }}
+      # ponytail: bring-up waits for the whole image. Starting the databases
+      # first would overlap them with the web build, about 1 min more.
+      - name: wait-signoz-image
+        if: steps.signoz-binary.outputs.cache-hit == 'true'
+        wait: signoz-image
       - name: bring-up-stack
         run: |
           cd tests && \
             uv run pytest \
             --basetemp=./tmp/ \
             -vv --reuse --with-web \
+            ${{ steps.signoz-binary.outputs.cache-hit == 'true' && '--prebuilt-signoz' || '' }} \
             e2e/bootstrap/setup.py::test_setup
       - name: wait-playwright-browsers
         wait: playwright-browsers
+      - name: download-plan
+        uses: actions/download-artifact@v4
+        with:
+          name: e2e-plan
+          path: tests/e2e/artifacts/plan
       - name: playwright-test
         run: |
           cd tests/e2e && \
-            pnpm playwright test --project=${{ matrix.project }} --shard=${{ matrix.shard }}/4
+            pnpm playwright test --project=${{ matrix.project }} --test-list artifacts/plan/shard-${{ matrix.shard }}.txt
       - name: teardown-stack
         if: always()
         run: |

docs/contributing/tests/e2e.md

@@ -319,6 +319,24 @@ cp .env.example .env      # fill SIGNOZ_E2E_USERNAME / PASSWORD
 yarn test:staging
 ```
 
+### Which specs run on a PR
+
+`e2eci` runs the specs a change can reach plus the `@smoke` set, on as many shards as their recorded durations need.
+
+- A frontend change is traced up the import graph to the routes it reaches. `affected.json` maps each route's page to the spec folders that visit it. Pages under the `@smoke` key are covered by the smoke set alone.
+- A changed spec runs itself. A changed driver, helper or fixture runs every spec that imports it.
+- The full suite runs when a change reaches the app shell, sits outside the module graph (a lockfile, `package.json`, a public asset), touches the server or CI, or reaches a page `affected.json` doesn't name. It also runs while any spec folder outside `routing/` has no entry in `affected.json`.
+- `scripts/plan.mjs` packs the selected spec files into shards using the per-test durations in `durations.json`.
+
+When you add a spec folder, add it to `affected.json` with the pages its specs visit. Tag a describe with `{ tag: '@smoke' }` only for a core path every PR should check.
+
+```bash
+pnpm plan                 # what your branch would run, against origin/main
+pnpm test:smoke           # the smoke set
+pnpm durations            # refresh durations.json from the last 10 green e2eci runs
+pnpm test:scripts         # check the spec selection
+```
+
 ## How to configure different options for E2E tests?
 
 ### Environment variables
@@ -357,6 +375,7 @@ The same pytest flags integration tests expose work here, since E2E reuses the s
 - `--teardown` — tear everything down.
 - `--clean` — prune the docker build caches, forcing the next image build to start cold.
 - `--with-web` — build the frontend into the SigNoz container. **Required for E2E**; integration tests don't need it.
+- `--prebuilt-signoz` — run the `signoz:integration` image already in the local image store instead of building it. The image must be built with `ZEUSURL=http://zeus:8080`. CI uses it when main's server binary matches the change.
 - `--sqlstore-provider`, `--postgres-version`, `--clickhouse-version`, etc. — see `docs/contributing/tests/integration.md`.
 
 ## What should I remember?

tests/conftest.py

@@ -88,6 +88,12 @@ def pytest_addoption(parser: pytest.Parser):
         default=False,
         help="Build and run with web. Run pytest --basetemp=./tmp/ -vv --with-web src/bootstrap/setup::test_setup to setup your local dev environment with web.",
     )
+    parser.addoption(
+        "--prebuilt-signoz",
+        action="store_true",
+        default=False,
+        help="Run the signoz:integration image already in the local image store instead of building it. The image must be built with ZEUSURL=http://zeus:8080.",
+    )
     parser.addoption(
         "--sqlstore-provider",
         action="store",

tests/e2e/affected.json

@@ -0,0 +1,54 @@
+{
+	"alerts": [
+		"frontend/src/pages/AlertList",
+		"frontend/src/pages/CreateAlert",
+		"frontend/src/pages/EditRules",
+		"frontend/src/pages/LogsModulePage",
+		"frontend/src/pages/TracesModulePage"
+	],
+	"dashboards": [
+		"frontend/src/pages/DashboardsListPage",
+		"frontend/src/pages/DashboardPage",
+		"frontend/src/pages/HomePage",
+		"frontend/src/pages/LogsModulePage"
+	],
+	"llm-o11y": ["frontend/src/pages/LLMObservability"],
+	"logs-explorer": ["frontend/src/pages/LogsModulePage"],
+	"query-builder": ["frontend/src/pages/LogsModulePage"],
+	"quick-filters": ["frontend/src/pages/LogsModulePage"],
+	"sso": ["frontend/src/pages/Login", "frontend/src/pages/Settings"],
+	"trace-details": ["frontend/src/pages/TraceDetailsV3"],
+	"@smoke": [
+		"frontend/src/modules/Servicemap",
+		"frontend/src/modules/Usage",
+		"frontend/src/pages/AIAssistantPage",
+		"frontend/src/pages/AllErrors",
+		"frontend/src/pages/ApiMonitoring",
+		"frontend/src/pages/Celery",
+		"frontend/src/pages/ErrorBoundaryFallback",
+		"frontend/src/pages/ErrorDetails",
+		"frontend/src/pages/ForgotPassword",
+		"frontend/src/pages/InfrastructureMonitoring",
+		"frontend/src/pages/IntegrationsDetailsPage",
+		"frontend/src/pages/IntegrationsModulePage",
+		"frontend/src/pages/License",
+		"frontend/src/pages/LogsSettings",
+		"frontend/src/pages/MessagingQueues",
+		"frontend/src/pages/MeterExplorer",
+		"frontend/src/pages/MetricsApplication",
+		"frontend/src/pages/MetricsExplorer",
+		"frontend/src/pages/OnboardingPageV2",
+		"frontend/src/pages/OrgOnboarding",
+		"frontend/src/pages/PublicDashboard",
+		"frontend/src/pages/ResetPassword",
+		"frontend/src/pages/ServiceTopLevelOperations",
+		"frontend/src/pages/Services",
+		"frontend/src/pages/SignUp",
+		"frontend/src/pages/Status",
+		"frontend/src/pages/Support",
+		"frontend/src/pages/UnAuthorized",
+		"frontend/src/pages/WorkspaceAccessRestricted",
+		"frontend/src/pages/WorkspaceLocked",
+		"frontend/src/pages/WorkspaceSuspended"
+	]
+}

tests/e2e/package.json

@@ -12,6 +12,10 @@
     "env:clean": "rm -rf ../tmp ../.pytest_cache .env.local artifacts && echo 'Cleaned. Run docker container prune if needed.'",
     "test": "playwright test",
     "test:local": "pnpm env:start && pnpm test",
+    "test:smoke": "playwright test --project=chromium --grep @smoke",
+    "plan": "node scripts/plan.mjs",
+    "test:scripts": "node scripts/affected.test.mjs",
+    "durations": "node scripts/durations.mjs",
     "test:staging": "SIGNOZ_E2E_BASE_URL=https://app.us.staging.signoz.cloud playwright test",
     "test:ui": "playwright test --ui",
     "test:headed": "playwright test --headed",

tests/e2e/scripts/affected.mjs

@@ -0,0 +1,190 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import ts from 'typescript';
+
+import { E2E_DIR, REPO_DIR } from './utils.mjs';
+
+const FEATURES = Object.entries(
+	JSON.parse(fs.readFileSync(path.join(E2E_DIR, 'affected.json'), 'utf8')),
+);
+export const FEATURE_NAMES = FEATURES.map(([feature]) => feature);
+const ROUTER = path.join(REPO_DIR, 'frontend/src/AppRoutes/pageComponents.ts');
+const ENTRY = path.join(REPO_DIR, 'frontend/src/index.tsx');
+const SPECS_DIR = path.join(E2E_DIR, 'tests');
+const E2E_SOURCE = /^tests\/e2e\/(tests|drivers|helpers|fixtures)\/.+\.ts$/;
+const SOURCE = /\.[cm]?[jt]sx?$/;
+const IGNORED = [
+	/\.md$/,
+	/^docs\//,
+	/^tests\/integration\//,
+	/^frontend\/\.(storybook|husky)\//,
+	/^frontend\/src\/(storybook|mocks-server)\//,
+	/^frontend\/(.*\/)?(__tests__|__mocks__)\//,
+	/^frontend\/.*\.(test|spec|stories)\.[jt]sx?$/,
+];
+
+const isIgnored = (file) => IGNORED.some((pattern) => pattern.test(file));
+const relative = (file) => path.relative(REPO_DIR, file);
+const isUnder = (file, root) =>
+	file === root || file.startsWith(`${root}/`) || file.startsWith(`${root}.`);
+
+function importersOf(tsconfigPath) {
+	const { config } = ts.readConfigFile(tsconfigPath, (file) =>
+		ts.sys.readFile(file),
+	);
+	const { options, fileNames } = ts.parseJsonConfigFileContent(
+		config,
+		ts.sys,
+		path.dirname(tsconfigPath),
+	);
+	const cache = ts.createModuleResolutionCache(
+		path.dirname(tsconfigPath),
+		(name) => name,
+		options,
+	);
+	const aliases = Object.entries(options.paths ?? {});
+	const aliasBase = options.pathsBasePath ?? path.dirname(tsconfigPath);
+
+	const resolve = (specifier, from) => {
+		const { resolvedModule } = ts.resolveModuleName(
+			specifier,
+			from,
+			options,
+			ts.sys,
+			cache,
+		);
+		if (resolvedModule) {
+			return resolvedModule.isExternalLibraryImport
+				? undefined
+				: path.resolve(resolvedModule.resolvedFileName);
+		}
+		// TypeScript resolves only modules, so styles and assets are matched here.
+		const candidates = specifier.startsWith('.')
+			? [path.resolve(path.dirname(from), specifier)]
+			: aliases.flatMap(([pattern, targets]) => {
+					const prefix = pattern.replace(/\*$/, '');
+					const matches = pattern.endsWith('*')
+						? specifier.startsWith(prefix)
+						: specifier === pattern;
+					return matches
+						? targets.map((target) =>
+								path.resolve(
+									aliasBase,
+									target.replace('*', specifier.slice(prefix.length)),
+								),
+							)
+						: [];
+				});
+		return candidates.find((candidate) =>
+			fs.statSync(candidate, { throwIfNoEntry: false })?.isFile(),
+		);
+	};
+
+	const importers = new Map();
+	for (const file of fileNames) {
+		const { importedFiles } = ts.preProcessFile(
+			fs.readFileSync(file, 'utf8'),
+			true,
+			true,
+		);
+		for (const { fileName } of importedFiles) {
+			const target = resolve(fileName, file);
+			if (target) {
+				if (!importers.has(target)) {
+					importers.set(target, new Set());
+				}
+				importers.get(target).add(path.resolve(file));
+			}
+		}
+	}
+	return importers;
+}
+
+// Walks importers up from `start`. `tops` are files nothing imports; `routes`
+// are files imported straight by the router.
+function climb(start, importers, isRouter) {
+	const seen = new Set([start]);
+	const queue = [start];
+	const tops = [];
+	const routes = [];
+	while (queue.length) {
+		const file = queue.pop();
+		const parents = importers.get(file) ?? new Set();
+		if (!parents.size) {
+			tops.push(file);
+		}
+		for (const parent of parents) {
+			if (isRouter(parent)) {
+				routes.push(file);
+			} else if (!seen.has(parent)) {
+				seen.add(parent);
+				queue.push(parent);
+			}
+		}
+	}
+	return { tops, routes };
+}
+
+// Spec files and feature folders a change can affect. `all` is set when any
+// changed file can't be traced to them.
+export function affected(changedFiles) {
+	const reasons = [];
+	const features = new Set();
+	const specs = new Set();
+	let frontend;
+	let e2e;
+
+	for (const changed of changedFiles) {
+		const file = path.join(REPO_DIR, changed);
+		// A deleted module or style's importers are in the change as well.
+		const isDeletedImport =
+			!fs.existsSync(file) &&
+			(SOURCE.test(changed) || changed.startsWith('frontend/src/'));
+		if (isIgnored(changed) || isDeletedImport) {
+			continue;
+		}
+		if (changed.startsWith('frontend/')) {
+			frontend ??= importersOf(path.join(REPO_DIR, 'frontend/tsconfig.json'));
+			const { tops, routes } = climb(
+				file,
+				frontend,
+				(parent) => parent === ROUTER,
+			);
+			// Other tops are dead code, tests or stories. A file outside the module
+			// graph (config, lockfile, public asset, scss partial) is never traced.
+			const isModule = SOURCE.test(changed) && changed.startsWith('frontend/src/');
+			if (tops.includes(ENTRY)) {
+				reasons.push(`${changed} reaches the app shell`);
+			} else if (tops.includes(file) && !isModule) {
+				reasons.push(`${changed} has no importer`);
+			}
+			for (const route of routes) {
+				const matched = FEATURES.filter(([, roots]) =>
+					roots.some((root) => isUnder(relative(route), root)),
+				);
+				if (!matched.length) {
+					reasons.push(`${changed} reaches ${relative(route)}, unmapped`);
+				}
+				matched.forEach(([feature]) => features.add(feature));
+			}
+		} else if (E2E_SOURCE.test(changed)) {
+			e2e ??= importersOf(path.join(E2E_DIR, 'tsconfig.json'));
+			for (const top of climb(file, e2e, () => false).tops) {
+				if (top.startsWith(`${SPECS_DIR}/`)) {
+					specs.add(path.relative(SPECS_DIR, top));
+				} else {
+					reasons.push(`${changed} reaches ${relative(top)}`);
+				}
+			}
+		} else {
+			reasons.push(`${changed} is not traced`);
+		}
+	}
+
+	return {
+		all: reasons.length > 0,
+		reasons,
+		features: [...features].sort((a, b) => a.localeCompare(b)),
+		specs: [...specs].sort((a, b) => a.localeCompare(b)),
+	};
+}

tests/e2e/scripts/affected.test.mjs

@@ -0,0 +1,34 @@
+import assert from 'node:assert/strict';
+
+import { affected } from './affected.mjs';
+
+assert.deepEqual(
+	affected(['frontend/src/pages/AlertList/index.tsx']).features,
+	['alerts'],
+	'a page change selects the features mapped to its route',
+);
+
+for (const file of [
+	'frontend/.storybook/preview.tsx',
+	'docs/api/openapi.yml',
+]) {
+	assert.deepEqual(
+		affected([file]),
+		{ all: false, reasons: [], features: [], specs: [] },
+		`${file} selects nothing`,
+	);
+}
+
+assert.deepEqual(
+	affected(['tests/e2e/helpers/sso.ts']).specs,
+	['sso/auth-domains.spec.ts'],
+	'a helper change selects the specs that import it',
+);
+
+for (const file of [
+	'frontend/pnpm-lock.yaml',
+	'frontend/src/index.tsx',
+	'pkg/signoz/signoz.go',
+]) {
+	assert.equal(affected([file]).all, true, `${file} runs everything`);
+}

tests/e2e/scripts/durations.mjs

@@ -0,0 +1,125 @@
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { parseArgs } from 'node:util';
+
+import { E2E_DIR, listTests, median, REPO_DIR, testKey } from './utils.mjs';
+
+// Reads reports already downloaded into `dir` when given; otherwise downloads
+// the Playwright artifacts of the last green e2eci runs.
+const { values: options, positionals } = parseArgs({
+	options: { runs: { type: 'string', default: '10' } },
+	allowPositionals: true,
+});
+
+const gh = (...args) =>
+	execFileSync('gh', args, {
+		cwd: REPO_DIR,
+		stdio: ['ignore', 'pipe', 'inherit'],
+	})
+		.toString()
+		.trim();
+
+const dir =
+	positionals[0] ?? fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-durations-'));
+if (!positionals[0]) {
+	const runIds = gh(
+		'run',
+		'list',
+		'--workflow',
+		'e2eci.yaml',
+		'--status',
+		'success',
+		'--limit',
+		options.runs,
+		'--json',
+		'databaseId',
+		'--jq',
+		'.[].databaseId',
+	).split('\n');
+	for (const runId of runIds) {
+		try {
+			gh(
+				'run',
+				'download',
+				runId,
+				'--pattern',
+				'playwright-artifacts-*',
+				'--dir',
+				path.join(dir, runId),
+			);
+		} catch {
+			console.warn(`Skipped run ${runId}: its artifacts did not download.`);
+		}
+	}
+}
+
+const samples = new Map();
+const reports = fs
+	.readdirSync(dir, { recursive: true })
+	.filter((file) => file.endsWith('results.json'));
+for (const report of reports) {
+	const walk = (suite, titles) => {
+		for (const spec of suite.specs ?? []) {
+			for (const test of spec.tests) {
+				if (test.projectName !== 'chromium' || !test.results.length) {
+					continue;
+				}
+				const key = testKey({ file: spec.file, titles: [...titles, spec.title] });
+				const sample = samples.get(key) ?? { first: [], flaky: 0 };
+				sample.first.push(test.results[0].duration);
+				sample.flaky += test.status === 'flaky' ? 1 : 0;
+				samples.set(key, sample);
+			}
+		}
+		for (const child of suite.suites ?? []) {
+			walk(child, [...titles, child.title]);
+		}
+	};
+	for (const fileSuite of JSON.parse(fs.readFileSync(path.join(dir, report)))
+		.suites) {
+		walk(fileSuite, []);
+	}
+}
+if (!positionals[0]) {
+	fs.rmSync(dir, { recursive: true, force: true });
+}
+
+const durations = {};
+for (const key of listTests()
+	.map(testKey)
+	.sort((a, b) => a.localeCompare(b))) {
+	const sample = samples.get(key);
+	if (sample) {
+		durations[key] = {
+			ms: Math.round(median(sample.first)),
+			runs: sample.first.length,
+			flaky: sample.flaky,
+		};
+	}
+}
+fs.writeFileSync(
+	path.join(E2E_DIR, 'durations.json'),
+	`${JSON.stringify(durations, null, '\t')}\n`,
+);
+
+const entries = Object.entries(durations);
+const describe = ([key, { ms, runs, flaky }]) =>
+	`  ${(ms / 1000).toFixed(1).padStart(5)} s  ${flaky}/${runs} flaky  ${key}`;
+process.stdout.write(
+	[
+		`${entries.length} tests from ${reports.length} reports.`,
+		'Slowest:',
+		...entries
+			.sort((a, b) => b[1].ms - a[1].ms)
+			.slice(0, 10)
+			.map(describe),
+		'Flaky:',
+		...entries
+			.filter(([, { flaky }]) => flaky)
+			.sort((a, b) => b[1].flaky / b[1].runs - a[1].flaky / a[1].runs)
+			.map(describe),
+		'',
+	].join('\n'),
+);

tests/e2e/scripts/plan.mjs

@@ -0,0 +1,161 @@
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseArgs } from 'node:util';
+
+import { affected, FEATURE_NAMES } from './affected.mjs';
+import { E2E_DIR, listTests, median, REPO_DIR, testKey } from './utils.mjs';
+
+// Matches `workers` in playwright.config.ts on CI.
+const WORKERS = 2;
+
+const { values: options } = parseArgs({
+	options: {
+		all: { type: 'boolean', default: false },
+		out: { type: 'string', default: 'artifacts/plan' },
+		'max-shards': { type: 'string', default: '8' },
+		'shard-minutes': { type: 'string', default: '4' },
+	},
+});
+
+function changedFiles() {
+	if (!process.stdin.isTTY) {
+		return fs.readFileSync(0, 'utf8').split('\n').filter(Boolean);
+	}
+	const git = (...args) =>
+		execFileSync('git', args, { cwd: REPO_DIR })
+			.toString()
+			.split('\n')
+			.filter(Boolean);
+	const [base] = git('merge-base', 'origin/main', 'HEAD');
+	return [
+		...git('diff', '--name-only', base),
+		...git('ls-files', '--others', '--exclude-standard'),
+	];
+}
+
+const traced = options.all
+	? { all: true, reasons: ['--all'], features: [], specs: [] }
+	: affected(changedFiles());
+const tests = listTests();
+// A spec folder the map does not name would never run for a page change.
+const unmapped = [
+	...new Set(
+		tests
+			.filter((test) => !test.tags.includes('@smoke'))
+			.map((test) => test.file.split('/')[0]),
+	),
+].filter((folder) => !FEATURE_NAMES.includes(folder));
+const reasons = [
+	...traced.reasons,
+	...(traced.features.length || traced.specs.length
+		? unmapped.map(
+				(folder) => `tests/e2e/tests/${folder}/ has no entry in affected.json`,
+			)
+		: []),
+];
+const selection = { ...traced, all: reasons.length > 0, reasons };
+const chosen = new Set(
+	tests.filter(
+		(test) =>
+			selection.all ||
+			selection.specs.includes(test.file) ||
+			selection.features.includes(test.file.split('/')[0]),
+	),
+);
+if (chosen.size || selection.features.length || selection.specs.length) {
+	tests
+		.filter((test) => test.tags.includes('@smoke'))
+		.forEach((test) => chosen.add(test));
+}
+
+// A flaky test is expected to cost one extra attempt per flaky run.
+const durations = JSON.parse(
+	fs.readFileSync(path.join(E2E_DIR, 'durations.json'), 'utf8'),
+);
+const fallbackMs =
+	median(Object.values(durations).map((duration) => duration.ms)) || 30_000;
+const cost = (test) => {
+	const duration = durations[testKey(test)];
+	return duration
+		? duration.ms * (1 + duration.flaky / duration.runs)
+		: fallbackMs;
+};
+
+// Files stay whole so serial groups and worker-scoped seeds stay on one shard.
+const files = new Map();
+for (const test of chosen) {
+	const file = files.get(test.file) ?? { tests: [], ms: 0 };
+	file.tests.push(test);
+	file.ms += cost(test);
+	files.set(test.file, file);
+}
+const totalMs = [...files.values()].reduce((sum, file) => sum + file.ms, 0);
+const shardMs = Number(options['shard-minutes']) * 60_000 * WORKERS;
+const shardCount = files.size
+	? Math.min(Number(options['max-shards']), Math.ceil(totalMs / shardMs))
+	: 0;
+
+const shards = Array.from({ length: shardCount }, () => ({ ms: 0, tests: [] }));
+for (const file of [...files.values()].sort((a, b) => b.ms - a.ms)) {
+	const shard = shards.reduce((least, next) =>
+		next.ms < least.ms ? next : least,
+	);
+	shard.ms += file.ms;
+	shard.tests.push(...file.tests);
+}
+
+const out = path.resolve(E2E_DIR, options.out);
+fs.rmSync(out, { recursive: true, force: true });
+fs.mkdirSync(out, { recursive: true });
+shards.forEach((shard, index) => {
+	fs.writeFileSync(
+		path.join(out, `shard-${index + 1}.txt`),
+		shard.tests
+			.map((test) => `[chromium] › ${[test.file, ...test.titles].join(' › ')}`)
+			.join('\n') + '\n',
+	);
+});
+
+const minutes = (ms) => (ms / WORKERS / 60_000).toFixed(1);
+const scope = selection.all
+	? 'every spec'
+	: [...new Set([...selection.features, ...selection.specs, '@smoke'])].join(
+			', ',
+		);
+const summary = [
+	shardCount
+		? `**E2E plan:** ${chosen.size} of ${tests.length} tests on ${shardCount} shard${shardCount > 1 ? 's' : ''}, at most ${minutes(Math.max(...shards.map((shard) => shard.ms)))} min of tests per shard.`
+		: '**E2E plan:** no test reaches this change.',
+	shardCount ? `**Runs:** ${scope}` : '',
+	...selection.reasons
+		.slice(0, 5)
+		.map(
+			(reason, index) => `${index ? '' : '**Full suite because:** '}${reason}`,
+		),
+]
+	.filter(Boolean)
+	.join('\n\n');
+
+fs.writeFileSync(
+	path.join(out, 'plan.json'),
+	JSON.stringify(
+		{
+			...selection,
+			tests: chosen.size,
+			shards: shards.map((shard) => minutes(shard.ms)),
+		},
+		null,
+		'\t',
+	),
+);
+process.stdout.write(`${summary.replaceAll('**', '')}\n`);
+if (process.env.GITHUB_OUTPUT) {
+	fs.appendFileSync(
+		process.env.GITHUB_OUTPUT,
+		`shards=${JSON.stringify(shards.map((_, index) => index + 1))}\n`,
+	);
+}
+if (process.env.GITHUB_STEP_SUMMARY) {
+	fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`);
+}

tests/e2e/scripts/signoz-key.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "e2e-signoz-$(git rev-parse HEAD:cmd HEAD:ee HEAD:pkg HEAD:go.mod HEAD:go.sum HEAD:.dockerignore HEAD:.github/workflows/cacheci.yml | sha256sum | cut -c1-32)"

tests/e2e/scripts/utils.mjs

@@ -0,0 +1,46 @@
+import { execFileSync } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export const E2E_DIR = path.resolve(
+	path.dirname(fileURLToPath(import.meta.url)),
+	'..',
+);
+export const REPO_DIR = path.resolve(E2E_DIR, '../..');
+
+export const testKey = ({ file, titles }) => [file, ...titles].join(' › ');
+
+export const median = (values) =>
+	values.length ? [...values].sort((a, b) => a - b)[values.length >> 1] : 0;
+
+export function listTests() {
+	const output = execFileSync(
+		path.join(E2E_DIR, 'node_modules/.bin/playwright'),
+		['test', '--list', '--project=chromium', '--reporter=json'],
+		{ cwd: E2E_DIR, maxBuffer: 64 * 1024 * 1024 },
+	);
+	const report = JSON.parse(output.toString());
+	if (report.errors?.length) {
+		throw new Error(report.errors.map((error) => error.message).join('\n'));
+	}
+	const tests = [];
+	const walk = (suite, titles) => {
+		for (const spec of suite.specs ?? []) {
+			if (spec.tests.some((test) => test.projectName === 'chromium')) {
+				tests.push({
+					file: spec.file,
+					titles: [...titles, spec.title],
+					// The JSON reporter drops the leading @.
+					tags: spec.tags.map((tag) => `@${tag}`),
+				});
+			}
+		}
+		for (const child of suite.suites ?? []) {
+			walk(child, [...titles, child.title]);
+		}
+	};
+	for (const fileSuite of report.suites) {
+		walk(fileSuite, []);
+	}
+	return tests;
+}

tests/e2e/tests/routing/guards.spec.ts

@@ -4,7 +4,7 @@ import { expect, PATH, test } from '../../drivers/routing';
 // told otherwise, which only shows when someone presses Back. Runs as `guest`:
 // an authenticated page would skip the chain under test.
 
-test.describe('Routing — auth guards', () => {
+test.describe('Routing — auth guards', { tag: '@smoke' }, () => {
 	test('TC-14 a deep-linked private route survives the login round trip', async ({
 		guest,
 	}) => {

tests/e2e/tests/routing/mount.spec.ts

@@ -2,7 +2,7 @@ import { expect, PATH, test } from '../../drivers/routing';
 
 // The shape of the router mount, which no other spec observes.
 
-test.describe('Routing: router mount', () => {
+test.describe('Routing: router mount', { tag: '@smoke' }, () => {
 	// Under a React transition the previous screen stays up instead of the
 	// Suspense fallback, so an uncached route renders no loader at all.
 	test('TC-18 an in-app navigation to an uncached route commits the Suspense fallback', async ({

tests/e2e/tests/routing/nav-blocker.spec.ts

@@ -16,47 +16,54 @@ test.describe.configure({ mode: 'serial' });
 const BLOCKED_ROLE_NAME = `routing-blocker-${uniqueAlphaSuffix()}`;
 const SAVED_ROLE_NAME = `routing-bypass-${uniqueAlphaSuffix()}`;
 
-test.describe('Routing — unsaved-changes navigation blocker', () => {
-	test.afterAll(async ({ browser }) => {
-		await onAdminPage(browser, (page) =>
-			routingDriver(page).roles.removeByName([BLOCKED_ROLE_NAME, SAVED_ROLE_NAME]),
-		);
-	});
+test.describe(
+	'Routing — unsaved-changes navigation blocker',
+	{ tag: '@smoke' },
+	() => {
+		test.afterAll(async ({ browser }) => {
+			await onAdminPage(browser, (page) =>
+				routingDriver(page).roles.removeByName([
+					BLOCKED_ROLE_NAME,
+					SAVED_ROLE_NAME,
+				]),
+			);
+		});
 
-	test('TC-16 a blocked navigation can be cancelled and then confirmed', async ({
-		routing,
-	}) => {
-		const { roles } = routing;
-		await roles.openCreateForm(BLOCKED_ROLE_NAME);
+		test('TC-16 a blocked navigation can be cancelled and then confirmed', async ({
+			routing,
+		}) => {
+			const { roles } = routing;
+			await roles.openCreateForm(BLOCKED_ROLE_NAME);
 
-		await roles.cancel();
-		await expect(roles.discardDialog()).toBeVisible();
+			await roles.cancel();
+			await expect(roles.discardDialog()).toBeVisible();
 
-		// Cancel drops the blocked transition rather than retrying it.
-		await roles
-			.discardDialog()
-			.getByRole('button', { name: 'Keep editing' })
-			.click();
-		await expect(roles.discardDialog()).toBeHidden();
-		expect(routing.url().pathname).toBe(PATH.roleCreate);
-		await expect(roles.nameInput()).toHaveValue(BLOCKED_ROLE_NAME);
+			// Cancel drops the blocked transition rather than retrying it.
+			await roles
+				.discardDialog()
+				.getByRole('button', { name: 'Keep editing' })
+				.click();
+			await expect(roles.discardDialog()).toBeHidden();
+			expect(routing.url().pathname).toBe(PATH.roleCreate);
+			await expect(roles.nameInput()).toHaveValue(BLOCKED_ROLE_NAME);
 
-		await roles.cancel();
-		await roles.discardDialog().getByRole('button', { name: 'Discard' }).click();
-		await routing.waitForPath(PATH.roles);
-	});
+			await roles.cancel();
+			await roles.discardDialog().getByRole('button', { name: 'Discard' }).click();
+			await routing.waitForPath(PATH.roles);
+		});
 
-	test('TC-17 saving bypasses the blocker for the next navigation', async ({
-		routing,
-	}) => {
-		const { roles } = routing;
-		await roles.openCreateForm(SAVED_ROLE_NAME);
+		test('TC-17 saving bypasses the blocker for the next navigation', async ({
+			routing,
+		}) => {
+			const { roles } = routing;
+			await roles.openCreateForm(SAVED_ROLE_NAME);
 
-		// `allowNextNavigation()` unarms the blocker for the push after a save.
-		await roles.save();
-		await routing.waitForPath(PATH.roles);
+			// `allowNextNavigation()` unarms the blocker for the push after a save.
+			await roles.save();
+			await routing.waitForPath(PATH.roles);
 
-		await expect(roles.discardDialog()).toHaveCount(0);
-		await expect(roles.settings()).toBeVisible();
-	});
-});
+			await expect(roles.discardDialog()).toHaveCount(0);
+			await expect(roles.settings()).toBeVisible();
+		});
+	},
+);

tests/e2e/tests/routing/navigation.spec.ts

@@ -3,7 +3,7 @@ import { dashboardListParams, expect, PATH, test } from '../../drivers/routing';
 // Imperative navigation: the `history.listen` counter behind `hasInAppHistory()`,
 // `useSafeNavigate`'s same-url suppression, and its `newTab` branch.
 
-test.describe('Routing — imperative navigation', () => {
+test.describe('Routing — imperative navigation', { tag: '@smoke' }, () => {
 	test('TC-05 Back from a detail page restores the list with its params', async ({
 		routing,
 		dashboard,

tests/e2e/tests/routing/route-matching.spec.ts

@@ -4,7 +4,7 @@ import { expect, PATH, test } from '../../drivers/routing';
 // `RouteTab`, `TopNav`'s per-route checks, and `SETTINGS`, a prefix route that
 // needs an explicit `/*` under v7.
 
-test.describe('Routing — route matching', () => {
+test.describe('Routing — route matching', { tag: '@smoke' }, () => {
 	test('TC-09 RouteTab switches the url and survives a reload', async ({
 		routing,
 	}) => {

tests/e2e/tests/routing/url-state.spec.ts

@@ -4,80 +4,84 @@ import { expect, PATH, test } from '../../drivers/routing';
 // to the History API with `history: 'push'`, while `relativeTime` is written by
 // the router through `useSafeNavigate`.
 
-test.describe('Routing — nuqs and router param coexistence', () => {
-	test('TC-01 a router write keeps the nuqs params it did not author', async ({
-		routing,
-		pods,
-	}) => {
-		await routing.k8s.goto({ relativeTime: '30m' });
+test.describe(
+	'Routing — nuqs and router param coexistence',
+	{ tag: '@smoke' },
+	() => {
+		test('TC-01 a router write keeps the nuqs params it did not author', async ({
+			routing,
+			pods,
+		}) => {
+			await routing.k8s.goto({ relativeTime: '30m' });
 
-		await routing.k8s.groupBy(pods.groupBy);
-		await routing.selectRelativeTime('1h');
-		await routing.waitForParam('relativeTime', '1h');
+			await routing.k8s.groupBy(pods.groupBy);
+			await routing.selectRelativeTime('1h');
+			await routing.waitForParam('relativeTime', '1h');
 
-		expect(routing.url().pathname).toBe(PATH.k8s);
-		expect(routing.params()).toMatchObject({
-			relativeTime: '1h',
-			category: 'pods',
-		});
-		expect(routing.params().groupBy).toContain(pods.groupBy);
-		await expect(routing.k8s.groupBySelect()).toContainText(pods.groupBy);
-	});
-
-	test('TC-02 a nuqs write keeps the relativeTime the router authored', async ({
-		routing,
-		pods,
-	}) => {
-		await routing.k8s.goto({ relativeTime: '30m' });
-
-		await routing.selectRelativeTime('6h');
-		await routing.waitForParam('relativeTime', '6h');
-
-		await routing.k8s.groupBy(pods.groupBy);
-
-		expect(routing.url().pathname).toBe(PATH.k8s);
-		expect(routing.params().relativeTime).toBe('6h');
-		expect(routing.params().groupBy).toContain(pods.groupBy);
-		await routing.expectRelativeTime('6h');
-	});
-
-	test('TC-03 POP across a nuqs push reverts both url and list', async ({
-		routing,
-		pods,
-	}) => {
-		await routing.k8s.goto({ relativeTime: '30m' });
-
-		// One flush, one entry: otherwise one Back lands on an intermediate url.
-		const depthBeforeGroupBy = await routing.historyDepth();
-		await routing.k8s.groupBy(pods.groupBy);
-		expect(await routing.historyDepth()).toBe(depthBeforeGroupBy + 1);
-		await expect(routing.k8s.groupBySelect()).toContainText(pods.groupBy);
-
-		await routing.goBack();
-		await routing.k8s.waitUntilGrouped(pods.groupBy, false);
-		expect(routing.url().pathname).toBe(PATH.k8s);
-		await expect(routing.k8s.groupBySelect()).not.toContainText(pods.groupBy);
-
-		await routing.goForward();
-		await routing.k8s.waitUntilGrouped(pods.groupBy);
-		await expect(routing.k8s.groupBySelect()).toContainText(pods.groupBy);
-	});
-
-	test('TC-04 a deep link carrying both families applies both on first paint', async ({
-		routing,
-		pods,
-	}) => {
-		await routing.k8s.goto({
-			relativeTime: '6h',
-			groupBy: JSON.stringify([pods.groupBy]),
+			expect(routing.url().pathname).toBe(PATH.k8s);
+			expect(routing.params()).toMatchObject({
+				relativeTime: '1h',
+				category: 'pods',
+			});
+			expect(routing.params().groupBy).toContain(pods.groupBy);
+			await expect(routing.k8s.groupBySelect()).toContainText(pods.groupBy);
 		});
 
-		// `k8s.goto` waited for the mount-time republish, so anything it dropped is
-		// missing by now.
-		expect(routing.url().pathname).toBe(PATH.k8s);
-		expect(routing.params().relativeTime).toBe('6h');
-		expect(routing.params().groupBy).toContain(pods.groupBy);
-		await routing.expectRelativeTime('6h');
-		await expect(routing.k8s.groupBySelect()).toContainText(pods.groupBy);
-	});
-});
+		test('TC-02 a nuqs write keeps the relativeTime the router authored', async ({
+			routing,
+			pods,
+		}) => {
+			await routing.k8s.goto({ relativeTime: '30m' });
+
+			await routing.selectRelativeTime('6h');
+			await routing.waitForParam('relativeTime', '6h');
+
+			await routing.k8s.groupBy(pods.groupBy);
+
+			expect(routing.url().pathname).toBe(PATH.k8s);
+			expect(routing.params().relativeTime).toBe('6h');
+			expect(routing.params().groupBy).toContain(pods.groupBy);
+			await routing.expectRelativeTime('6h');
+		});
+
+		test('TC-03 POP across a nuqs push reverts both url and list', async ({
+			routing,
+			pods,
+		}) => {
+			await routing.k8s.goto({ relativeTime: '30m' });
+
+			// One flush, one entry: otherwise one Back lands on an intermediate url.
+			const depthBeforeGroupBy = await routing.historyDepth();
+			await routing.k8s.groupBy(pods.groupBy);
+			expect(await routing.historyDepth()).toBe(depthBeforeGroupBy + 1);
+			await expect(routing.k8s.groupBySelect()).toContainText(pods.groupBy);
+
+			await routing.goBack();
+			await routing.k8s.waitUntilGrouped(pods.groupBy, false);
+			expect(routing.url().pathname).toBe(PATH.k8s);
+			await expect(routing.k8s.groupBySelect()).not.toContainText(pods.groupBy);
+
+			await routing.goForward();
+			await routing.k8s.waitUntilGrouped(pods.groupBy);
+			await expect(routing.k8s.groupBySelect()).toContainText(pods.groupBy);
+		});
+
+		test('TC-04 a deep link carrying both families applies both on first paint', async ({
+			routing,
+			pods,
+		}) => {
+			await routing.k8s.goto({
+				relativeTime: '6h',
+				groupBy: JSON.stringify([pods.groupBy]),
+			});
+
+			// `k8s.goto` waited for the mount-time republish, so anything it dropped is
+			// missing by now.
+			expect(routing.url().pathname).toBe(PATH.k8s);
+			expect(routing.params().relativeTime).toBe('6h');
+			expect(routing.params().groupBy).toContain(pods.groupBy);
+			await routing.expectRelativeTime('6h');
+			await expect(routing.k8s.groupBySelect()).toContainText(pods.groupBy);
+		});
+	},
+);

tests/fixtures/http.py

@@ -28,6 +28,8 @@ def zeus(
     def create() -> types.TestContainerDocker:
         container = WireMockContainer(image="wiremock/wiremock:2.35.1-1", secure=False)
         container.with_network(network)
+        # The zeus address in images built for --prebuilt-signoz.
+        container.with_network_aliases("zeus")
         container.start()
 
         return types.TestContainerDocker(

tests/fixtures/signoz.py

@@ -58,23 +58,24 @@ def create_signoz(
 
         # The docker CLI is required: the Dockerfiles use BuildKit cache
         # mounts, which docker-py does not support.
-        subprocess.run(
-            [
-                "docker",
-                "build",
-                "--file",
-                str(context / dockerfile_path),
-                "--tag",
-                "signoz:integration",
-                "--build-arg",
-                f"TARGETARCH={arch}",
-                "--build-arg",
-                f"ZEUSURL={zeus.container_configs['8080'].base()}",
-                str(context),
-            ],
-            check=True,
-            env=os.environ | {"DOCKER_BUILDKIT": "1"},
-        )
+        if not pytestconfig.getoption("--prebuilt-signoz"):
+            subprocess.run(
+                [
+                    "docker",
+                    "build",
+                    "--file",
+                    str(context / dockerfile_path),
+                    "--tag",
+                    "signoz:integration",
+                    "--build-arg",
+                    f"TARGETARCH={arch}",
+                    "--build-arg",
+                    f"ZEUSURL={zeus.container_configs['8080'].base()}",
+                    str(context),
+                ],
+                check=True,
+                env=os.environ | {"DOCKER_BUILDKIT": "1"},
+            )
 
         env = (
             {