fix: expose gha runtime

.github/workflows/e2eci.yaml

@@ -45,9 +45,15 @@ jobs:
       (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))) && contains(github.event.pull_request.labels.*.name, 'safe-to-e2e')
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      SIGNOZ_BUILDX_GHA_SCOPE: signoz-e2e
     steps:
       - name: checkout
         uses: actions/checkout@v4
+      - name: setup-buildx
+        uses: docker/setup-buildx-action@v3
+      - name: expose-gha-runtime
+        uses: crazy-max/ghaction-github-runtime@v3
       - name: python
         uses: actions/setup-python@v5
         with:

.github/workflows/integrationci.yaml

@@ -69,9 +69,15 @@ jobs:
       ((github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
       (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))) && contains(github.event.pull_request.labels.*.name, 'safe-to-integrate')
     runs-on: ubuntu-latest
+    env:
+      SIGNOZ_BUILDX_GHA_SCOPE: signoz-integration
     steps:
       - name: checkout
         uses: actions/checkout@v4
+      - name: setup-buildx
+        uses: docker/setup-buildx-action@v3
+      - name: expose-gha-runtime
+        uses: crazy-max/ghaction-github-runtime@v3
       - name: python
         uses: actions/setup-python@v5
         with:

Makefile

@@ -34,6 +34,7 @@ DOCKER_BUILD_ARCHS_ENTERPRISE 	= $(addprefix docker-build-enterprise-,$(ARCHS))
 DOCKERFILE_ENTERPRISE 			= $(SRC)/cmd/enterprise/Dockerfile
 DOCKER_REGISTRY_ENTERPRISE 		?= docker.io/signoz/signoz
 JS_BUILD_CONTEXT 				= $(SRC)/frontend
+DOCKER_BUILDX_PRUNE_FLAGS 		?= --force
 
 ##############################################################
 # directories
@@ -229,6 +230,21 @@ py-clean: ## Clear all pycache and pytest cache from tests directory recursively
 	@find tests -type f -name "*.pyo" -delete 2>/dev/null || true
 	@echo ">> python cache cleaned"
 
+.PHONY: py-docker-clean
+py-docker-clean: ## Remove Docker image and build caches used by python integration tests
+	@echo ">> removing SigNoz integration test image"
+	@docker image rm -f signoz:integration 2>/dev/null || true
+	@echo ">> removing local integration buildx cache directories"
+	@rm -rf /tmp/signoz-integration-buildx-cache /tmp/signoz-integration-buildx-cache-next /tmp/signoz-e2e-buildx-cache /tmp/signoz-e2e-buildx-cache-next
+	@echo ">> pruning docker buildx cache with flags: $(DOCKER_BUILDX_PRUNE_FLAGS)"
+	@docker buildx prune $(DOCKER_BUILDX_PRUNE_FLAGS)
+
+.PHONY: py-test-clean
+py-test-clean: ## Tear down python test stack and remove python/Docker test caches
+	@$(MAKE) py-test-teardown || true
+	@$(MAKE) py-clean
+	@$(MAKE) py-docker-clean
+
 
 ##############################################################
 # generate commands

cmd/enterprise/Dockerfile.integration

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.13
+
 FROM golang:1.25-bookworm
 
 ARG OS="linux"
@@ -21,7 +23,8 @@ RUN set -eux; \
 
 COPY go.mod go.sum ./
 
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod \
+	go mod download
 
 COPY ./cmd/ ./cmd/
 COPY ./ee/ ./ee/
@@ -29,7 +32,9 @@ COPY ./pkg/ ./pkg/
 COPY ./templates/email /root/templates
 
 COPY Makefile Makefile
-RUN TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
+RUN --mount=type=cache,target=/go/pkg/mod \
+	--mount=type=cache,target=/root/.cache/go-build \
+	TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
 RUN mv /root/linux-${TARGETARCH}/signoz /root/signoz
 
 RUN chmod 755 /root /root/signoz

cmd/enterprise/Dockerfile.with-web.integration

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.13
+
 FROM node:22-bookworm AS build
 
 WORKDIR /opt/
@@ -30,7 +32,8 @@ RUN set -eux; \
 
 COPY go.mod go.sum ./
 
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod \
+	go mod download
 
 COPY ./cmd/ ./cmd/
 COPY ./ee/ ./ee/
@@ -38,7 +41,9 @@ COPY ./pkg/ ./pkg/
 COPY ./templates/email /root/templates
 
 COPY Makefile Makefile
-RUN TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
+RUN --mount=type=cache,target=/go/pkg/mod \
+	--mount=type=cache,target=/root/.cache/go-build \
+	TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
 RUN mv /root/linux-${TARGETARCH}/signoz /root/signoz
 
 COPY --from=build /opt/build ./web/

tests/fixtures/http.py

@@ -17,6 +17,8 @@ from fixtures.logger import setup_logger
 
 logger = setup_logger(__name__)
 
+ZEUS_NETWORK_ALIAS = "signoz-zeus"
+
 
 @pytest.fixture(name="zeus", scope="package")
 def zeus(
@@ -31,6 +33,7 @@ def zeus(
     def create() -> types.TestContainerDocker:
         container = WireMockContainer(image="wiremock/wiremock:2.35.1-1", secure=False)
         container.with_network(network)
+        container.with_network_aliases(ZEUS_NETWORK_ALIAS)
         container.start()
 
         return types.TestContainerDocker(
@@ -42,7 +45,7 @@ def zeus(
                     container.get_exposed_port(8080),
                 )
             },
-            container_configs={"8080": types.TestContainerUrlConfig("http", container.get_wrapped_container().name, 8080)},
+            container_configs={"8080": types.TestContainerUrlConfig("http", ZEUS_NETWORK_ALIAS, 8080)},
         )
 
     def delete(container: types.TestContainerDocker):

tests/fixtures/signoz.py

@@ -1,14 +1,19 @@
+import os
 import platform
+import shutil
+import subprocess
+import threading
 import time
+from dataclasses import dataclass
 from http import HTTPStatus
 from os import path
+from pathlib import Path
 
 import docker
 import docker.errors
 import pytest
 import requests
 from testcontainers.core.container import DockerContainer, Network
-from testcontainers.core.image import DockerImage
 
 from fixtures import reuse, types
 from fixtures.logger import setup_logger
@@ -16,6 +21,110 @@ from fixtures.logger import setup_logger
 logger = setup_logger(__name__)
 
 
+@dataclass
+class SigNozImageBuild:
+    process: subprocess.Popen
+    command: list[str]
+    cache_path: Path | None = None
+    next_cache_path: Path | None = None
+    reader: threading.Thread | None = None
+
+
+def _stream_build_output(pipe, log) -> None:
+    try:
+        for line in iter(pipe.readline, ""):
+            log.info("buildx: %s", line.rstrip())
+    finally:
+        pipe.close()
+
+
+def start_signoz_image_build(pytestconfig: pytest.Config, dockerfile_path: str, arch: str, zeus_url: str) -> SigNozImageBuild:
+    root = pytestconfig.rootpath.parent
+    command = [
+        "docker",
+        "buildx",
+        "build",
+        "--load",
+        "--progress",
+        "plain",
+        "--tag",
+        "signoz:integration",
+        "--file",
+        dockerfile_path,
+        "--build-arg",
+        f"TARGETARCH={arch}",
+        "--build-arg",
+        f"ZEUSURL={zeus_url}",
+        str(root),
+    ]
+
+    cache_path = None
+    next_cache_path = None
+    if os.environ.get("ACTIONS_RUNTIME_TOKEN"):
+        # Running in GitHub Actions — use BuildKit's native GHA cache backend.
+        # Avoids the local-write races and partial exports seen with type=local.
+        scope = os.environ.get("SIGNOZ_BUILDX_GHA_SCOPE", "signoz-integration")
+        command.extend(["--cache-from", f"type=gha,scope={scope}"])
+        command.extend(["--cache-to", f"type=gha,scope={scope},mode=max"])
+    elif build_cache_dir := os.environ.get("SIGNOZ_INTEGRATION_BUILD_CACHE_DIR"):
+        # Local cache for developer machines / non-GHA CI.
+        cache_path = Path(build_cache_dir)
+        next_cache_path = Path(f"{build_cache_dir}-next")
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.rmtree(next_cache_path, ignore_errors=True)
+
+        if cache_path.exists():
+            command.extend(["--cache-from", f"type=local,src={cache_path}"])
+        command.extend(["--cache-to", f"type=local,dest={next_cache_path},mode=max"])
+
+    logger.info("Building SigNoz integration image with %s", " ".join(command))
+    process = subprocess.Popen(
+        command,
+        cwd=root,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1,
+    )
+    reader = threading.Thread(target=_stream_build_output, args=(process.stdout, logger), daemon=True)
+    reader.start()
+
+    return SigNozImageBuild(
+        process=process,
+        command=command,
+        cache_path=cache_path,
+        next_cache_path=next_cache_path,
+        reader=reader,
+    )
+
+
+def wait_for_signoz_image_build(build: SigNozImageBuild) -> None:
+    returncode = build.process.wait()
+    if build.reader is not None:
+        build.reader.join(timeout=5)
+    if returncode != 0:
+        raise subprocess.CalledProcessError(returncode, build.command)
+
+    if build.cache_path and build.next_cache_path and build.next_cache_path.exists():
+        shutil.rmtree(build.cache_path, ignore_errors=True)
+        shutil.move(build.next_cache_path, build.cache_path)
+
+
+def stop_signoz_image_build(build: SigNozImageBuild) -> None:
+    if build.process.poll() is not None:
+        return
+
+    build.process.terminate()
+    try:
+        build.process.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        build.process.kill()
+        build.process.wait()
+
+    if build.reader is not None:
+        build.reader.join(timeout=5)
+
+
 def create_signoz(
     network: Network,
     zeus: types.TestContainerDocker,
@@ -33,9 +142,6 @@ def create_signoz(
     """
 
     def create() -> types.SigNoz:
-        # Run the migrations for clickhouse
-        request.getfixturevalue("migrator")
-
         # Get the no-web flag
         with_web = pytestconfig.getoption("--with-web")
 
@@ -48,19 +154,15 @@ def create_signoz(
         if with_web:
             dockerfile_path = "cmd/enterprise/Dockerfile.with-web.integration"
 
-        # Docker build context is the repo root — one up from pytest's
-        # rootdir (tests/).
-        self = DockerImage(
-            path=str(pytestconfig.rootpath.parent),
-            dockerfile_path=dockerfile_path,
-            tag="signoz:integration",
-            buildargs={
-                "TARGETARCH": arch,
-                "ZEUSURL": zeus.container_configs["8080"].base(),
-            },
-        )
-
-        self.build()
+        # The SigNoz image build does not depend on ClickHouse migrations, so
+        # build it while the migrator container runs.
+        image_build = start_signoz_image_build(pytestconfig, dockerfile_path, arch, zeus.container_configs["8080"].base())
+        try:
+            request.getfixturevalue("migrator")
+            wait_for_signoz_image_build(image_build)
+        except Exception:  # pylint: disable=broad-exception-caught
+            stop_signoz_image_build(image_build)
+            raise
 
         env = (
             {