chore: speed up python integration test setup builds

.github/workflows/e2eci.yaml

@@ -45,9 +45,20 @@ jobs:
       (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))) && contains(github.event.pull_request.labels.*.name, 'safe-to-e2e')
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      SIGNOZ_INTEGRATION_BUILD_CACHE_DIR: /tmp/signoz-e2e-buildx-cache
     steps:
       - name: checkout
         uses: actions/checkout@v4
+      - name: setup-buildx
+        uses: docker/setup-buildx-action@v3
+      - name: restore-buildx-cache
+        uses: actions/cache@v4
+        with:
+          path: /tmp/signoz-e2e-buildx-cache
+          key: ${{ runner.os }}-signoz-e2e-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-signoz-e2e-buildx-
       - name: python
         uses: actions/setup-python@v5
         with:

.github/workflows/integrationci.yaml

@@ -69,9 +69,20 @@ jobs:
       ((github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
       (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))) && contains(github.event.pull_request.labels.*.name, 'safe-to-integrate')
     runs-on: ubuntu-latest
+    env:
+      SIGNOZ_INTEGRATION_BUILD_CACHE_DIR: /tmp/signoz-integration-buildx-cache
     steps:
       - name: checkout
         uses: actions/checkout@v4
+      - name: setup-buildx
+        uses: docker/setup-buildx-action@v3
+      - name: restore-buildx-cache
+        uses: actions/cache@v4
+        with:
+          path: /tmp/signoz-integration-buildx-cache
+          key: ${{ runner.os }}-signoz-integration-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-signoz-integration-buildx-
       - name: python
         uses: actions/setup-python@v5
         with:

Makefile

@@ -34,6 +34,7 @@ DOCKER_BUILD_ARCHS_ENTERPRISE 	= $(addprefix docker-build-enterprise-,$(ARCHS))
 DOCKERFILE_ENTERPRISE 			= $(SRC)/cmd/enterprise/Dockerfile
 DOCKER_REGISTRY_ENTERPRISE 		?= docker.io/signoz/signoz
 JS_BUILD_CONTEXT 				= $(SRC)/frontend
+DOCKER_BUILDX_PRUNE_FLAGS 		?= --force
 
 ##############################################################
 # directories
@@ -229,6 +230,21 @@ py-clean: ## Clear all pycache and pytest cache from tests directory recursively
 	@find tests -type f -name "*.pyo" -delete 2>/dev/null || true
 	@echo ">> python cache cleaned"
 
+.PHONY: py-docker-clean
+py-docker-clean: ## Remove Docker image and build caches used by python integration tests
+	@echo ">> removing SigNoz integration test image"
+	@docker image rm -f signoz:integration 2>/dev/null || true
+	@echo ">> removing local integration buildx cache directories"
+	@rm -rf /tmp/signoz-integration-buildx-cache /tmp/signoz-integration-buildx-cache-next /tmp/signoz-e2e-buildx-cache /tmp/signoz-e2e-buildx-cache-next
+	@echo ">> pruning docker buildx cache with flags: $(DOCKER_BUILDX_PRUNE_FLAGS)"
+	@docker buildx prune $(DOCKER_BUILDX_PRUNE_FLAGS)
+
+.PHONY: py-test-clean
+py-test-clean: ## Tear down python test stack and remove python/Docker test caches
+	@$(MAKE) py-test-teardown || true
+	@$(MAKE) py-clean
+	@$(MAKE) py-docker-clean
+
 
 ##############################################################
 # generate commands

cmd/enterprise/Dockerfile.integration

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1
+
 FROM golang:1.25-bookworm
 
 ARG OS="linux"
@@ -21,7 +23,8 @@ RUN set -eux; \
 
 COPY go.mod go.sum ./
 
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod \
+	go mod download
 
 COPY ./cmd/ ./cmd/
 COPY ./ee/ ./ee/
@@ -29,7 +32,9 @@ COPY ./pkg/ ./pkg/
 COPY ./templates/email /root/templates
 
 COPY Makefile Makefile
-RUN TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
+RUN --mount=type=cache,target=/go/pkg/mod \
+	--mount=type=cache,target=/root/.cache/go-build \
+	TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
 RUN mv /root/linux-${TARGETARCH}/signoz /root/signoz
 
 RUN chmod 755 /root /root/signoz

cmd/enterprise/Dockerfile.with-web.integration

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1
+
 FROM node:22-bookworm AS build
 
 WORKDIR /opt/
@@ -30,7 +32,8 @@ RUN set -eux; \
 
 COPY go.mod go.sum ./
 
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod \
+	go mod download
 
 COPY ./cmd/ ./cmd/
 COPY ./ee/ ./ee/
@@ -38,7 +41,9 @@ COPY ./pkg/ ./pkg/
 COPY ./templates/email /root/templates
 
 COPY Makefile Makefile
-RUN TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
+RUN --mount=type=cache,target=/go/pkg/mod \
+	--mount=type=cache,target=/root/.cache/go-build \
+	TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race
 RUN mv /root/linux-${TARGETARCH}/signoz /root/signoz
 
 COPY --from=build /opt/build ./web/

tests/fixtures/signoz.py

@@ -1,14 +1,18 @@
+import os
 import platform
+import shutil
+import subprocess
 import time
+from dataclasses import dataclass
 from http import HTTPStatus
 from os import path
+from pathlib import Path
 
 import docker
 import docker.errors
 import pytest
 import requests
 from testcontainers.core.container import DockerContainer, Network
-from testcontainers.core.image import DockerImage
 
 from fixtures import reuse, types
 from fixtures.logger import setup_logger
@@ -16,6 +20,77 @@ from fixtures.logger import setup_logger
 logger = setup_logger(__name__)
 
 
+@dataclass
+class SigNozImageBuild:
+    process: subprocess.Popen
+    command: list[str]
+    cache_path: Path | None = None
+    next_cache_path: Path | None = None
+
+
+def start_signoz_image_build(pytestconfig: pytest.Config, dockerfile_path: str, arch: str, zeus_url: str) -> SigNozImageBuild:
+    root = pytestconfig.rootpath.parent
+    command = [
+        "docker",
+        "buildx",
+        "build",
+        "--load",
+        "--progress",
+        "plain",
+        "--tag",
+        "signoz:integration",
+        "--file",
+        dockerfile_path,
+        "--build-arg",
+        f"TARGETARCH={arch}",
+        "--build-arg",
+        f"ZEUSURL={zeus_url}",
+        str(root),
+    ]
+
+    cache_path = None
+    next_cache_path = None
+    if build_cache_dir := os.environ.get("SIGNOZ_INTEGRATION_BUILD_CACHE_DIR"):
+        cache_path = Path(build_cache_dir)
+        next_cache_path = Path(f"{build_cache_dir}-next")
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.rmtree(next_cache_path, ignore_errors=True)
+
+        if cache_path.exists():
+            command.extend(["--cache-from", f"type=local,src={cache_path}"])
+        command.extend(["--cache-to", f"type=local,dest={next_cache_path},mode=max,ignore-error=true"])
+
+    logger.info("Building SigNoz integration image with %s", " ".join(command))
+    return SigNozImageBuild(
+        process=subprocess.Popen(command, cwd=root),
+        command=command,
+        cache_path=cache_path,
+        next_cache_path=next_cache_path,
+    )
+
+
+def wait_for_signoz_image_build(build: SigNozImageBuild) -> None:
+    returncode = build.process.wait()
+    if returncode != 0:
+        raise subprocess.CalledProcessError(returncode, build.command)
+
+    if build.cache_path and build.next_cache_path and build.next_cache_path.exists():
+        shutil.rmtree(build.cache_path, ignore_errors=True)
+        shutil.move(build.next_cache_path, build.cache_path)
+
+
+def stop_signoz_image_build(build: SigNozImageBuild) -> None:
+    if build.process.poll() is not None:
+        return
+
+    build.process.terminate()
+    try:
+        build.process.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        build.process.kill()
+        build.process.wait()
+
+
 def create_signoz(
     network: Network,
     zeus: types.TestContainerDocker,
@@ -33,9 +108,6 @@ def create_signoz(
     """
 
     def create() -> types.SigNoz:
-        # Run the migrations for clickhouse
-        request.getfixturevalue("migrator")
-
         # Get the no-web flag
         with_web = pytestconfig.getoption("--with-web")
 
@@ -48,19 +120,15 @@ def create_signoz(
         if with_web:
             dockerfile_path = "cmd/enterprise/Dockerfile.with-web.integration"
 
-        # Docker build context is the repo root — one up from pytest's
-        # rootdir (tests/).
-        self = DockerImage(
-            path=str(pytestconfig.rootpath.parent),
-            dockerfile_path=dockerfile_path,
-            tag="signoz:integration",
-            buildargs={
-                "TARGETARCH": arch,
-                "ZEUSURL": zeus.container_configs["8080"].base(),
-            },
-        )
-
-        self.build()
+        image_build = start_signoz_image_build(pytestconfig, dockerfile_path, arch, zeus.container_configs["8080"].base())
+        try:
+            # The SigNoz image does not depend on ClickHouse migrations, so
+            # build it while the migrator container runs.
+            request.getfixturevalue("migrator")
+            wait_for_signoz_image_build(image_build)
+        except Exception:  # pylint: disable=broad-exception-caught
+            stop_signoz_image_build(image_build)
+            raise
 
         env = (
             {