Compare commits

...

5 Commits

Author SHA1 Message Date
Vinícius Lourenço
6970483877 test(e2e): add a model pattern before saving a pricing rule 2026-10-08 00:03:29 -03:00
Vinícius Lourenço
72f9a0b922 ci(e2e): start playwright-browsers after the image pulls 2026-10-07 23:46:10 -03:00
Vinícius Lourenço
b695c62c33 chore(tests): note e2eci pull-images sync on version defaults 2026-10-07 23:06:52 -03:00
Vinícius Lourenço
8ccb621320 ci(e2e): pull stack images and install python deps in background 2026-10-07 23:06:04 -03:00
Vinícius Lourenço
ca9d20d132 ci(e2e): shard playwright across 4 runners and install browsers in background 2026-10-07 22:57:58 -03:00
3 changed files with 42 additions and 12 deletions

View File

@@ -40,6 +40,7 @@ jobs:
matrix:
project:
- chromium
shard: [1, 2, 3, 4]
if: |
((github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
(github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))) && contains(github.event.pull_request.labels.*.name, 'safe-to-e2e')
@@ -48,12 +49,33 @@ jobs:
steps:
- name: checkout
uses: actions/checkout@v4
# Images bring-up-stack would otherwise pull, at the tests/conftest.py
# defaults. A stale tag only loses the head start.
- name: pull-images
id: pull-images
background: true
run: |
printf '%s\n' \
node:22-bookworm \
golang:1.25-bookworm \
python:3.13-slim \
postgres:15 \
clickhouse/clickhouse-server:25.12.5 \
clickhouse/clickhouse-keeper:25.12.5 \
signoz/signoz-schema-migrator:v0.144.9 \
wiremock/wiremock:2.35.1-1 \
| xargs -P 0 -n 1 docker pull -q
- name: python
uses: actions/setup-python@v5
with:
python-version: 3.13
- name: uv
uses: astral-sh/setup-uv@v4
- name: python-install
id: python-install
background: true
run: |
cd tests && uv sync
- name: node
uses: actions/setup-node@v4
with:
@@ -62,19 +84,9 @@ jobs:
uses: pnpm/action-setup@v6
with:
version: 10
- name: python-install
run: |
cd tests && uv sync
- name: pnpm-install
run: |
cd tests/e2e && pnpm install --frozen-lockfile
- name: playwright-browsers
run: |
docker create --name pw mcr.microsoft.com/playwright:v1.57.0-noble
docker cp pw:/ms-playwright "$RUNNER_TEMP/ms-playwright"
docker rm pw
echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" >> "$GITHUB_ENV"
cd tests/e2e && pnpm playwright install-deps ${{ matrix.project }}
# Restore-only: the cacheci workflow owns cache saves. Seeds the
# BuildKit cache mounts so the in-test image build is incremental.
- name: restore
@@ -99,6 +111,19 @@ jobs:
tar -xf /restored/pnpm-store.tar -C /pnpm/store
EOF
docker build -f "$RUNNER_TEMP/inject.Dockerfile" "$RUNNER_TEMP/cacheci"
- name: wait-stack-deps
wait: [pull-images, python-install]
# Started after the pulls so its download overlaps the CPU-bound
# image build instead of competing for the network.
- name: playwright-browsers
id: playwright-browsers
background: true
run: |
docker create --name pw mcr.microsoft.com/playwright:v1.57.0-noble
docker cp pw:/ms-playwright "$RUNNER_TEMP/ms-playwright"
docker rm pw
echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" >> "$GITHUB_ENV"
cd tests/e2e && pnpm playwright install-deps ${{ matrix.project }}
- name: bring-up-stack
run: |
cd tests && \
@@ -106,10 +131,12 @@ jobs:
--basetemp=./tmp/ \
-vv --reuse --with-web \
e2e/bootstrap/setup.py::test_setup
- name: wait-playwright-browsers
wait: playwright-browsers
- name: playwright-test
run: |
cd tests/e2e && \
pnpm playwright test --project=${{ matrix.project }}
pnpm playwright test --project=${{ matrix.project }} --shard=${{ matrix.shard }}/4
- name: teardown-stack
if: always()
run: |
@@ -122,6 +149,6 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: playwright-artifacts-${{ matrix.project }}
name: playwright-artifacts-${{ matrix.project }}-${{ matrix.shard }}
path: tests/e2e/artifacts/
retention-days: 5

View File

@@ -99,6 +99,7 @@ def pytest_addoption(parser: pytest.Parser):
default="delete",
help="sqlite mode",
)
# Keep the version defaults below in sync with pull-images in .github/workflows/e2eci.yaml.
parser.addoption(
"--postgres-version",
action="store",

View File

@@ -33,6 +33,8 @@ test.describe('LLM Observability — Model Pricing', () => {
await expect(page.getByTestId('drawer-model-id-input')).toBeVisible();
await page.getByTestId('drawer-model-id-input').fill(MODEL_NAME);
await page.getByTestId('drawer-pattern-input').fill(MODEL_NAME);
await page.getByTestId('drawer-pattern-add-btn').click();
await page.getByTestId('drawer-input-cost').fill(INPUT_COST);
await page.getByTestId('drawer-output-cost').fill(OUTPUT_COST);