Compare commits

..

1 Commits

Author SHA1 Message Date
Vinicius Lourenço
0e2adf3329 ci(e2e): shard playwright across 4 runners and install browsers in background (#13077)
Some checks are pending
build-staging / prepare (push) Waiting to run
build-staging / js-build (push) Blocked by required conditions
build-staging / go-build (push) Blocked by required conditions
build-staging / staging (push) Blocked by required conditions
cacheci / tests (push) Waiting to run
Release Drafter / update_release_draft (push) Waiting to run
<!--A few plain bullets saying what changed and why, for a reviewer
skimming it - not a wall of text, not a restatement of the diff, not
generated boilerplate.-->
#### Description

Two little optimizations:

- run the e2e CI shards since it's getting a little bigger (and slower)
- run parts of the steps in background
- this is something that was introduced recently by
https://github.blog/changelog/2026-06-25-actions-steps-can-now-be-run-in-parallel/

Also adds a model pattern in `llm-o11y/llm-pricing.spec.ts`: the backend
rejects an empty `modelPattern` since #12995.

#### Additional Information

Latest run
([37721605759](https://github.com/SigNoz/signoz/actions/runs/37721605759))
against the last 10 successful single-job runs (Sep 7 to Oct 1):

| | Before (median) | After (4 shards) |
|---|---|---|
| Wall time | 26.5 min | 12.6 min |
| Playwright tests | 19.7 min | 3.7 to 5.9 min per shard |
| Job start to tests | 7.3 min | 6.3 to 7.8 min |
| Runner time | 26.5 min | 46.9 min |

- Sharding gives almost all of the gain. The previous run
([37717653047](https://github.com/SigNoz/signoz/actions/runs/37717653047))
took 12.3 min wall and 46.6 min runner time.
- Starting `playwright-browsers` after the image pulls cut its download
from 1.7 to 3.1 min down to about 50 s, but it now overlaps
`bring-up-stack`, which got about 30 s slower. Job start to tests stays
at about 7 min in both runs, so the background steps save about 20 s per
job.
2026-10-08 17:39:32 +00:00
5 changed files with 44 additions and 18 deletions

View File

@@ -40,6 +40,7 @@ jobs:
matrix:
project:
- chromium
shard: [1, 2, 3, 4]
if: |
((github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
(github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))) && contains(github.event.pull_request.labels.*.name, 'safe-to-e2e')
@@ -48,12 +49,33 @@ jobs:
steps:
- name: checkout
uses: actions/checkout@v4
# Images bring-up-stack would otherwise pull, at the tests/conftest.py
# defaults. A stale tag only loses the head start.
- name: pull-images
id: pull-images
background: true
run: |
printf '%s\n' \
node:22-bookworm \
golang:1.25-bookworm \
python:3.13-slim \
postgres:15 \
clickhouse/clickhouse-server:25.12.5 \
clickhouse/clickhouse-keeper:25.12.5 \
signoz/signoz-schema-migrator:v0.144.9 \
wiremock/wiremock:2.35.1-1 \
| xargs -P 0 -n 1 docker pull -q
- name: python
uses: actions/setup-python@v5
with:
python-version: 3.13
- name: uv
uses: astral-sh/setup-uv@v4
- name: python-install
id: python-install
background: true
run: |
cd tests && uv sync
- name: node
uses: actions/setup-node@v4
with:
@@ -62,19 +84,9 @@ jobs:
uses: pnpm/action-setup@v6
with:
version: 10
- name: python-install
run: |
cd tests && uv sync
- name: pnpm-install
run: |
cd tests/e2e && pnpm install --frozen-lockfile
- name: playwright-browsers
run: |
docker create --name pw mcr.microsoft.com/playwright:v1.57.0-noble
docker cp pw:/ms-playwright "$RUNNER_TEMP/ms-playwright"
docker rm pw
echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" >> "$GITHUB_ENV"
cd tests/e2e && pnpm playwright install-deps ${{ matrix.project }}
# Restore-only: the cacheci workflow owns cache saves. Seeds the
# BuildKit cache mounts so the in-test image build is incremental.
- name: restore
@@ -99,6 +111,19 @@ jobs:
tar -xf /restored/pnpm-store.tar -C /pnpm/store
EOF
docker build -f "$RUNNER_TEMP/inject.Dockerfile" "$RUNNER_TEMP/cacheci"
- name: wait-stack-deps
wait: [pull-images, python-install]
# Started after the pulls so its download overlaps the CPU-bound
# image build instead of competing for the network.
- name: playwright-browsers
id: playwright-browsers
background: true
run: |
docker create --name pw mcr.microsoft.com/playwright:v1.57.0-noble
docker cp pw:/ms-playwright "$RUNNER_TEMP/ms-playwright"
docker rm pw
echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" >> "$GITHUB_ENV"
cd tests/e2e && pnpm playwright install-deps ${{ matrix.project }}
- name: bring-up-stack
run: |
cd tests && \
@@ -106,10 +131,12 @@ jobs:
--basetemp=./tmp/ \
-vv --reuse --with-web \
e2e/bootstrap/setup.py::test_setup
- name: wait-playwright-browsers
wait: playwright-browsers
- name: playwright-test
run: |
cd tests/e2e && \
pnpm playwright test --project=${{ matrix.project }}
pnpm playwright test --project=${{ matrix.project }} --shard=${{ matrix.shard }}/4
- name: teardown-stack
if: always()
run: |
@@ -122,6 +149,6 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: playwright-artifacts-${{ matrix.project }}
name: playwright-artifacts-${{ matrix.project }}-${{ matrix.shard }}
path: tests/e2e/artifacts/
retention-days: 5

View File

@@ -918,10 +918,6 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr
sb.OrderBy("__result_0 DESC")
}
if b.operator.Limit > 0 {
sb.Limit(b.operator.Limit)
}
combinedArgs := allAggChArgs
// Add HAVING clause if specified

View File

@@ -343,8 +343,8 @@ func TestTraceOperatorStatementBuilder(t *testing.T) {
},
},
expected: qbtypes.Statement{
Query: "WITH toDateTime64(1747947419000000000, 9) AS t_from, toDateTime64(1747983448000000000, 9) AS t_to, 1747945619 AS bucket_from, 1747983448 AS bucket_to, all_spans AS (SELECT *, resource_string_service$$name AS `service.name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_A AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), A AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_A) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), B AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND toFloat64(response_status_code) < ?), A_AND_B AS (SELECT l.* FROM A AS l INNER JOIN B AS r ON l.trace_id = r.trace_id) SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `service.name`, avg(toFloat64(duration_nano)) AS __result_0 FROM A_AND_B GROUP BY `service.name` ORDER BY __result_0 desc LIMIT ? SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "frontend", "%service.name%", "%service.name\":\"frontend%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), float64(400), 10},
Query: "WITH toDateTime64(1747947419000000000, 9) AS t_from, toDateTime64(1747983448000000000, 9) AS t_to, 1747945619 AS bucket_from, 1747983448 AS bucket_to, all_spans AS (SELECT *, resource_string_service$$name AS `service.name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_A AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), A AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_A) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), B AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND toFloat64(response_status_code) < ?), A_AND_B AS (SELECT l.* FROM A AS l INNER JOIN B AS r ON l.trace_id = r.trace_id) SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `service.name`, avg(toFloat64(duration_nano)) AS __result_0 FROM A_AND_B GROUP BY `service.name` ORDER BY __result_0 desc SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "frontend", "%service.name%", "%service.name\":\"frontend%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), float64(400)},
},
expectedErr: nil,
},

View File

@@ -99,6 +99,7 @@ def pytest_addoption(parser: pytest.Parser):
default="delete",
help="sqlite mode",
)
# Keep the version defaults below in sync with pull-images in .github/workflows/e2eci.yaml.
parser.addoption(
"--postgres-version",
action="store",

View File

@@ -33,6 +33,8 @@ test.describe('LLM Observability — Model Pricing', () => {
await expect(page.getByTestId('drawer-model-id-input')).toBeVisible();
await page.getByTestId('drawer-model-id-input').fill(MODEL_NAME);
await page.getByTestId('drawer-pattern-input').fill(MODEL_NAME);
await page.getByTestId('drawer-pattern-add-btn').click();
await page.getByTestId('drawer-input-cost').fill(INPUT_COST);
await page.getByTestId('drawer-output-cost').fill(OUTPUT_COST);