From 76b8cfbb6a4cd568bc5f6a5bd9357f92e94f3ce3 Mon Sep 17 00:00:00 2001 From: Estrella Pan Date: Fri, 10 Jul 2026 13:51:35 +0200 Subject: [PATCH] ci: add hermetic e2e lanes --- .github/workflows/build.yml | 2 +- .github/workflows/e2e-nightly.yml | 174 ++++++++ .github/workflows/e2e.yml | 393 ++++++++++++++++++ .../src/test/test_e2e_workflow_contract.py | 217 ++++++++++ e2e/README.md | 193 +++++++++ 5 files changed, 978 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e-nightly.yml create mode 100644 .github/workflows/e2e.yml create mode 100644 backend/src/test/test_e2e_workflow_contract.py create mode 100644 e2e/README.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 84e6b785..856bfd20 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: - name: Test run: | mkdir -p backend/config - cd backend && uv run pytest src/test -v + cd backend && uv run pytest src/test -v -m "not e2e" webui-test: runs-on: ubuntu-latest diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml new file mode 100644 index 00000000..e6033a0e --- /dev/null +++ b/.github/workflows/e2e-nightly.yml @@ -0,0 +1,174 @@ +name: Hermetic E2E Nightly + +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: e2e-nightly-${{ github.ref }} + cancel-in-progress: true + +env: + E2E_VERSION: 3.3.999-e2e.1 + PNPM_VERSION: 9.11.0 + PLAYWRIGHT_VERSION: 1.61.1 + +jobs: + e2e-nightly: + name: e2e-nightly (${{ matrix.lane }}-${{ matrix.pass }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - lane: firefox + pass: 1 + profile: browser + browser: firefox + playwright-project: firefox-nightly + - lane: browser + pass: 1 + profile: browser + browser: chromium + playwright-project: chromium-desktop + - lane: browser + pass: 2 + profile: browser + browser: chromium + playwright-project: chromium-desktop + - lane: downloader + pass: 1 + profile: downloader + browser: chromium + playwright-project: chromium-desktop + - lane: downloader + pass: 2 + profile: downloader + browser: chromium + playwright-project: chromium-desktop + env: + AB_E2E_PROJECT: ab-e2e-nightly-${{ matrix.lane }}-${{ matrix.pass }}-${{ github.run_id }}-${{ github.run_attempt }} + AB_E2E_APP_IMAGE: auto-bangumi:e2e-nightly-${{ matrix.lane }}-${{ matrix.pass }}-${{ github.run_id }}-${{ github.run_attempt }} + AB_E2E_CACHE_IMAGE: auto-bangumi:e2e-cache + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Prepare isolated work directory + id: paths + run: | + work_dir="$RUNNER_TEMP/$AB_E2E_PROJECT" + mkdir -p "$work_dir/artifacts" + { + echo "AB_E2E_WORK_DIR=$work_dir" + echo "AB_E2E_IMAGE_ARCHIVE=$RUNNER_TEMP/auto-bangumi-e2e-image.tar" + echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" + } >> "$GITHUB_ENV" + { + echo "work-dir=$work_dir" + echo "image-archive=$RUNNER_TEMP/auto-bangumi-e2e-image.tar" + echo "playwright-path=$RUNNER_TEMP/ms-playwright" + } >> "$GITHUB_OUTPUT" + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + version: ${{ env.PNPM_VERSION }} + run_install: false + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: pnpm + cache-dependency-path: webui/pnpm-lock.yaml + + - name: Install frontend dependencies + run: pnpm --dir webui install --frozen-lockfile + + - name: Cache Playwright browser + uses: actions/cache@v4 + with: + path: ${{ steps.paths.outputs.playwright-path }} + key: playwright-v1-${{ runner.os }}-${{ env.PLAYWRIGHT_VERSION }}-${{ matrix.browser }}-${{ hashFiles('webui/pnpm-lock.yaml') }} + + - name: Install Playwright browser and system dependencies + run: pnpm --dir webui exec playwright install --with-deps "${{ matrix.browser }}" + + - name: Type-check Playwright tests + run: pnpm --dir webui run test:e2e:typecheck + + - name: Cache production WebUI + id: webui-dist + uses: actions/cache@v4 + with: + path: webui/dist + key: webui-dist-v1-${{ runner.os }}-${{ hashFiles('webui/package.json', 'webui/pnpm-lock.yaml', 'webui/index.html', 'webui/vite.config.*', 'webui/tsconfig*.json', 'webui/src/**', 'webui/public/**') }} + + - name: Build production WebUI + if: steps.webui-dist.outputs.cache-hit != 'true' + run: pnpm --dir webui run build + + - name: Cache native E2E image + id: app-image + uses: actions/cache@v4 + with: + path: ${{ steps.paths.outputs.image-archive }} + key: app-image-v1-3.3.999-e2e.1-${{ runner.os }}-${{ hashFiles('Dockerfile', '.dockerignore', 'entrypoint.sh', 'backend/pyproject.toml', 'backend/uv.lock', 'backend/src/**', 'webui/package.json', 'webui/pnpm-lock.yaml', 'webui/index.html', 'webui/vite.config.*', 'webui/tsconfig*.json', 'webui/src/**', 'webui/public/**') }} + + - name: Load cached native E2E image + if: steps.app-image.outputs.cache-hit == 'true' + run: | + docker load --input "$AB_E2E_IMAGE_ARCHIVE" + docker tag "$AB_E2E_CACHE_IMAGE" "$AB_E2E_APP_IMAGE" + + - name: Build and verify native E2E image + if: steps.app-image.outputs.cache-hit != 'true' + run: | + python3 e2e/scripts/build_test_image.py --version "$E2E_VERSION" --image "$AB_E2E_APP_IMAGE" --dist webui/dist + docker tag "$AB_E2E_APP_IMAGE" "$AB_E2E_CACHE_IMAGE" + docker save "$AB_E2E_CACHE_IMAGE" --output "$AB_E2E_IMAGE_ARCHIVE" + + - name: Run nightly E2E pass + run: | + python3 e2e/scripts/stack.py run --profile "${{ matrix.profile }}" --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" -- pnpm --dir webui run test:e2e --project="${{ matrix.playwright-project }}" --retries=0 + + - name: Collect startup diagnostics + if: failure() + continue-on-error: true + shell: bash + run: | + compose_files=(--file e2e/compose/browser.yml) + if [[ "${{ matrix.profile }}" == "downloader" ]]; then + compose_files+=(--file e2e/compose/downloader.yml) + fi + python3 e2e/scripts/collect_artifacts.py --artifact-dir "$AB_E2E_WORK_DIR/artifacts/startup-diagnostics" --data-dir "$AB_E2E_WORK_DIR/data" --log-file "$AB_E2E_WORK_DIR/logs/log.txt" -- docker compose --project-name "$AB_E2E_PROJECT" --env-file "$AB_E2E_WORK_DIR/stack.env" "${compose_files[@]}" + + - name: Ensure stack teardown + if: always() + continue-on-error: true + run: | + python3 e2e/scripts/stack.py stop --profile "${{ matrix.profile }}" --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" + + - name: Upload nightly diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-nightly-${{ matrix.lane }}-${{ matrix.pass }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ steps.paths.outputs.work-dir }}/artifacts + webui/playwright-report + webui/test-results + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..571e8b4c --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,393 @@ +name: Hermetic E2E + +on: + pull_request: {} + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: e2e-pr-${{ github.event.pull_request.number || github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + E2E_VERSION: 3.3.999-e2e.1 + PNPM_VERSION: 9.11.0 + PLAYWRIGHT_VERSION: 1.61.1 + +jobs: + e2e-runtime: + name: e2e-runtime + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + AB_E2E_PROJECT: ab-e2e-pr-runtime-${{ github.run_id }}-${{ github.run_attempt }} + AB_E2E_APP_IMAGE: auto-bangumi:e2e-pr-runtime-${{ github.run_id }}-${{ github.run_attempt }} + AB_E2E_CACHE_IMAGE: auto-bangumi:e2e-cache + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Prepare isolated work directory + id: paths + run: | + work_dir="$RUNNER_TEMP/$AB_E2E_PROJECT" + mkdir -p "$work_dir/artifacts" + { + echo "AB_E2E_WORK_DIR=$work_dir" + echo "AB_E2E_IMAGE_ARCHIVE=$RUNNER_TEMP/auto-bangumi-e2e-image.tar" + echo "UV_CACHE_DIR=$RUNNER_TEMP/uv-cache" + } >> "$GITHUB_ENV" + { + echo "work-dir=$work_dir" + echo "image-archive=$RUNNER_TEMP/auto-bangumi-e2e-image.tar" + } >> "$GITHUB_OUTPUT" + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + with: + version: "0.11.26" + enable-cache: true + cache-dependency-glob: backend/uv.lock + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + version: ${{ env.PNPM_VERSION }} + run_install: false + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: pnpm + cache-dependency-path: webui/pnpm-lock.yaml + + - name: Install dependencies + run: | + uv sync --directory backend --locked --group dev + pnpm --dir webui install --frozen-lockfile + + - name: Cache production WebUI + id: webui-dist + uses: actions/cache@v4 + with: + path: webui/dist + key: webui-dist-v1-${{ runner.os }}-${{ hashFiles('webui/package.json', 'webui/pnpm-lock.yaml', 'webui/index.html', 'webui/vite.config.*', 'webui/tsconfig*.json', 'webui/src/**', 'webui/public/**') }} + + - name: Build production WebUI + if: steps.webui-dist.outputs.cache-hit != 'true' + run: pnpm --dir webui run build + + - name: Cache native E2E image + id: app-image + uses: actions/cache@v4 + with: + path: ${{ steps.paths.outputs.image-archive }} + key: app-image-v1-3.3.999-e2e.1-${{ runner.os }}-${{ hashFiles('Dockerfile', '.dockerignore', 'entrypoint.sh', 'backend/pyproject.toml', 'backend/uv.lock', 'backend/src/**', 'webui/package.json', 'webui/pnpm-lock.yaml', 'webui/index.html', 'webui/vite.config.*', 'webui/tsconfig*.json', 'webui/src/**', 'webui/public/**') }} + + - name: Load cached native E2E image + if: steps.app-image.outputs.cache-hit == 'true' + run: | + docker load --input "$AB_E2E_IMAGE_ARCHIVE" + docker tag "$AB_E2E_CACHE_IMAGE" "$AB_E2E_APP_IMAGE" + + - name: Build and verify native E2E image + if: steps.app-image.outputs.cache-hit != 'true' + run: | + python3 e2e/scripts/build_test_image.py --version "$E2E_VERSION" --image "$AB_E2E_APP_IMAGE" --dist webui/dist + docker tag "$AB_E2E_APP_IMAGE" "$AB_E2E_CACHE_IMAGE" + docker save "$AB_E2E_CACHE_IMAGE" --output "$AB_E2E_IMAGE_ARCHIVE" + + - name: Audit hermetic sources + run: python3 e2e/scripts/audit_sources.py + + - name: Run runtime E2E + run: | + python3 e2e/scripts/stack.py run --profile browser --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" -- uv run --directory backend pytest src/test/e2e/runtime -m e2e -q --tb=long --junitxml="$AB_E2E_WORK_DIR/artifacts/runtime-junit.xml" + + - name: Collect startup diagnostics + if: failure() + continue-on-error: true + run: | + python3 e2e/scripts/collect_artifacts.py --artifact-dir "$AB_E2E_WORK_DIR/artifacts/startup-diagnostics" --data-dir "$AB_E2E_WORK_DIR/data" --log-file "$AB_E2E_WORK_DIR/logs/log.txt" -- docker compose --project-name "$AB_E2E_PROJECT" --env-file "$AB_E2E_WORK_DIR/stack.env" --file e2e/compose/browser.yml + + - name: Ensure stack teardown + if: always() + continue-on-error: true + run: | + python3 e2e/scripts/stack.py stop --profile browser --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" + + - name: Upload runtime diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-runtime-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.paths.outputs.work-dir }}/artifacts + if-no-files-found: warn + retention-days: 14 + + e2e-browser: + name: e2e-browser (${{ matrix.project }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - project: chromium + browser: chromium + script: test:e2e:chromium + - project: webkit-mobile + browser: webkit + script: test:e2e:webkit + env: + AB_E2E_PROJECT: ab-e2e-pr-browser-${{ matrix.project }}-${{ github.run_id }}-${{ github.run_attempt }} + AB_E2E_APP_IMAGE: auto-bangumi:e2e-pr-browser-${{ matrix.project }}-${{ github.run_id }}-${{ github.run_attempt }} + AB_E2E_CACHE_IMAGE: auto-bangumi:e2e-cache + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Prepare isolated work directory + id: paths + run: | + work_dir="$RUNNER_TEMP/$AB_E2E_PROJECT" + mkdir -p "$work_dir/artifacts" + { + echo "AB_E2E_WORK_DIR=$work_dir" + echo "AB_E2E_IMAGE_ARCHIVE=$RUNNER_TEMP/auto-bangumi-e2e-image.tar" + echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" + } >> "$GITHUB_ENV" + { + echo "work-dir=$work_dir" + echo "image-archive=$RUNNER_TEMP/auto-bangumi-e2e-image.tar" + echo "playwright-path=$RUNNER_TEMP/ms-playwright" + } >> "$GITHUB_OUTPUT" + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + version: ${{ env.PNPM_VERSION }} + run_install: false + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: pnpm + cache-dependency-path: webui/pnpm-lock.yaml + + - name: Install frontend dependencies + run: pnpm --dir webui install --frozen-lockfile + + - name: Cache Playwright browser + uses: actions/cache@v4 + with: + path: ${{ steps.paths.outputs.playwright-path }} + key: playwright-v1-${{ runner.os }}-${{ env.PLAYWRIGHT_VERSION }}-${{ matrix.browser }}-${{ hashFiles('webui/pnpm-lock.yaml') }} + + - name: Install Playwright browser and system dependencies + run: pnpm --dir webui exec playwright install --with-deps "${{ matrix.browser }}" + + - name: Type-check Playwright tests + run: pnpm --dir webui run test:e2e:typecheck + + - name: Cache production WebUI + id: webui-dist + uses: actions/cache@v4 + with: + path: webui/dist + key: webui-dist-v1-${{ runner.os }}-${{ hashFiles('webui/package.json', 'webui/pnpm-lock.yaml', 'webui/index.html', 'webui/vite.config.*', 'webui/tsconfig*.json', 'webui/src/**', 'webui/public/**') }} + + - name: Build production WebUI + if: steps.webui-dist.outputs.cache-hit != 'true' + run: pnpm --dir webui run build + + - name: Cache native E2E image + id: app-image + uses: actions/cache@v4 + with: + path: ${{ steps.paths.outputs.image-archive }} + key: app-image-v1-3.3.999-e2e.1-${{ runner.os }}-${{ hashFiles('Dockerfile', '.dockerignore', 'entrypoint.sh', 'backend/pyproject.toml', 'backend/uv.lock', 'backend/src/**', 'webui/package.json', 'webui/pnpm-lock.yaml', 'webui/index.html', 'webui/vite.config.*', 'webui/tsconfig*.json', 'webui/src/**', 'webui/public/**') }} + + - name: Load cached native E2E image + if: steps.app-image.outputs.cache-hit == 'true' + run: | + docker load --input "$AB_E2E_IMAGE_ARCHIVE" + docker tag "$AB_E2E_CACHE_IMAGE" "$AB_E2E_APP_IMAGE" + + - name: Build and verify native E2E image + if: steps.app-image.outputs.cache-hit != 'true' + run: | + python3 e2e/scripts/build_test_image.py --version "$E2E_VERSION" --image "$AB_E2E_APP_IMAGE" --dist webui/dist + docker tag "$AB_E2E_APP_IMAGE" "$AB_E2E_CACHE_IMAGE" + docker save "$AB_E2E_CACHE_IMAGE" --output "$AB_E2E_IMAGE_ARCHIVE" + + - name: Run browser E2E + run: | + python3 e2e/scripts/stack.py run --profile browser --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" -- pnpm --dir webui run "${{ matrix.script }}" --retries=0 + + - name: Collect startup diagnostics + if: failure() + continue-on-error: true + run: | + python3 e2e/scripts/collect_artifacts.py --artifact-dir "$AB_E2E_WORK_DIR/artifacts/startup-diagnostics" --data-dir "$AB_E2E_WORK_DIR/data" --log-file "$AB_E2E_WORK_DIR/logs/log.txt" -- docker compose --project-name "$AB_E2E_PROJECT" --env-file "$AB_E2E_WORK_DIR/stack.env" --file e2e/compose/browser.yml + + - name: Ensure stack teardown + if: always() + continue-on-error: true + run: | + python3 e2e/scripts/stack.py stop --profile browser --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" + + - name: Upload browser diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-browser-${{ matrix.project }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ steps.paths.outputs.work-dir }}/artifacts + webui/playwright-report + webui/test-results + if-no-files-found: warn + retention-days: 14 + + e2e-downloader-image: + name: e2e-downloader-image + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + AB_E2E_PROJECT: ab-e2e-pr-downloader-image-${{ github.run_id }}-${{ github.run_attempt }} + AB_E2E_APP_IMAGE: auto-bangumi:e2e-pr-downloader-${{ github.run_id }}-${{ github.run_attempt }} + AB_E2E_CACHE_IMAGE: auto-bangumi:e2e-cache + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Prepare isolated work directory + id: paths + run: | + work_dir="$RUNNER_TEMP/$AB_E2E_PROJECT" + mkdir -p "$work_dir/artifacts" + { + echo "AB_E2E_WORK_DIR=$work_dir" + echo "AB_E2E_IMAGE_ARCHIVE=$RUNNER_TEMP/auto-bangumi-e2e-image.tar" + echo "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/ms-playwright" + } >> "$GITHUB_ENV" + { + echo "work-dir=$work_dir" + echo "image-archive=$RUNNER_TEMP/auto-bangumi-e2e-image.tar" + echo "playwright-path=$RUNNER_TEMP/ms-playwright" + } >> "$GITHUB_OUTPUT" + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + with: + version: "0.11.26" + enable-cache: true + cache-dependency-glob: backend/uv.lock + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + with: + version: ${{ env.PNPM_VERSION }} + run_install: false + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: pnpm + cache-dependency-path: webui/pnpm-lock.yaml + + - name: Install downloader-lane dependencies + run: | + uv sync --directory backend --locked --group dev + pnpm --dir webui install --frozen-lockfile + + - name: Cache Playwright Chromium + uses: actions/cache@v4 + with: + path: ${{ steps.paths.outputs.playwright-path }} + key: playwright-v1-${{ runner.os }}-${{ env.PLAYWRIGHT_VERSION }}-chromium-${{ hashFiles('webui/pnpm-lock.yaml') }} + + - name: Install Chromium and system dependencies + run: pnpm --dir webui exec playwright install --with-deps chromium + + - name: Type-check Playwright tests + run: pnpm --dir webui run test:e2e:typecheck + + - name: Cache production WebUI + id: webui-dist + uses: actions/cache@v4 + with: + path: webui/dist + key: webui-dist-v1-${{ runner.os }}-${{ hashFiles('webui/package.json', 'webui/pnpm-lock.yaml', 'webui/index.html', 'webui/vite.config.*', 'webui/tsconfig*.json', 'webui/src/**', 'webui/public/**') }} + + - name: Build production WebUI + if: steps.webui-dist.outputs.cache-hit != 'true' + run: pnpm --dir webui run build + + - name: Cache native E2E image + id: app-image + uses: actions/cache@v4 + with: + path: ${{ steps.paths.outputs.image-archive }} + key: app-image-v1-3.3.999-e2e.1-${{ runner.os }}-${{ hashFiles('Dockerfile', '.dockerignore', 'entrypoint.sh', 'backend/pyproject.toml', 'backend/uv.lock', 'backend/src/**', 'webui/package.json', 'webui/pnpm-lock.yaml', 'webui/index.html', 'webui/vite.config.*', 'webui/tsconfig*.json', 'webui/src/**', 'webui/public/**') }} + + - name: Load cached native E2E image + if: steps.app-image.outputs.cache-hit == 'true' + run: | + docker load --input "$AB_E2E_IMAGE_ARCHIVE" + docker tag "$AB_E2E_CACHE_IMAGE" "$AB_E2E_APP_IMAGE" + + - name: Build and verify native E2E image + if: steps.app-image.outputs.cache-hit != 'true' + run: | + python3 e2e/scripts/build_test_image.py --version "$E2E_VERSION" --image "$AB_E2E_APP_IMAGE" --dist webui/dist + docker tag "$AB_E2E_APP_IMAGE" "$AB_E2E_CACHE_IMAGE" + docker save "$AB_E2E_CACHE_IMAGE" --output "$AB_E2E_IMAGE_ARCHIVE" + + - name: Run real downloader and packaged-image E2E + run: | + python3 e2e/scripts/stack.py run --profile downloader --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" -- python3 e2e/scripts/run_downloader_lane.py + + - name: Collect startup diagnostics + if: failure() + continue-on-error: true + run: | + python3 e2e/scripts/collect_artifacts.py --artifact-dir "$AB_E2E_WORK_DIR/artifacts/startup-diagnostics" --data-dir "$AB_E2E_WORK_DIR/data" --log-file "$AB_E2E_WORK_DIR/logs/log.txt" -- docker compose --project-name "$AB_E2E_PROJECT" --env-file "$AB_E2E_WORK_DIR/stack.env" --file e2e/compose/browser.yml --file e2e/compose/downloader.yml + + - name: Ensure stack teardown + if: always() + continue-on-error: true + run: | + python3 e2e/scripts/stack.py stop --profile downloader --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" + + - name: Upload downloader diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-downloader-image-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ steps.paths.outputs.work-dir }}/artifacts + webui/playwright-report + webui/test-results + if-no-files-found: warn + retention-days: 14 diff --git a/backend/src/test/test_e2e_workflow_contract.py b/backend/src/test/test_e2e_workflow_contract.py new file mode 100644 index 00000000..f3c756d1 --- /dev/null +++ b/backend/src/test/test_e2e_workflow_contract.py @@ -0,0 +1,217 @@ +"""Pure source contracts for the hermetic E2E GitHub workflows.""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +PR_WORKFLOW = REPO_ROOT / ".github/workflows/e2e.yml" +NIGHTLY_WORKFLOW = REPO_ROOT / ".github/workflows/e2e-nightly.yml" +BUILD_WORKFLOW = REPO_ROOT / ".github/workflows/build.yml" +PLAYWRIGHT_CONFIG = REPO_ROOT / "webui/playwright.config.ts" +DOWNLOADER_RUNNER = REPO_ROOT / "e2e/scripts/run_downloader_lane.py" +E2E_README = REPO_ROOT / "e2e/README.md" + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _workflow_header(source: str) -> str: + header, separator, _ = source.partition("\njobs:\n") + assert separator, "workflow must contain a top-level jobs mapping" + return header + + +def _job(source: str, job_id: str) -> str: + match = re.search( + rf"(?ms)^ {re.escape(job_id)}:\n" + rf"(?P.*?)(?=^ [a-z0-9][a-z0-9-]*:\n|\Z)", + source, + ) + assert match, f"missing workflow job: {job_id}" + return match.group("body") + + +def test_pr_workflow_runs_for_every_pull_request() -> None: + source = _read(PR_WORKFLOW) + header = _workflow_header(source) + + assert re.search(r"(?m)^ pull_request: \{\}$", header) + assert "schedule:" not in header + assert not re.search( + r"(?m)^\s+(?:branches|branches-ignore|paths|paths-ignore):", + header, + ) + + +def test_pr_workflow_expands_to_four_stable_fifteen_minute_checks() -> None: + source = _read(PR_WORKFLOW) + runtime = _job(source, "e2e-runtime") + browser = _job(source, "e2e-browser") + downloader = _job(source, "e2e-downloader-image") + + assert "name: e2e-runtime" in runtime + assert "name: e2e-browser (${{ matrix.project }})" in browser + assert "project: chromium" in browser + assert "project: webkit-mobile" in browser + assert "name: e2e-downloader-image" in downloader + + for job in (runtime, browser, downloader): + assert "timeout-minutes: 15" in job + assert not re.search(r"(?m)^ continue-on-error:", job) + + +def test_pr_jobs_use_unique_stack_and_work_directory_names() -> None: + source = _read(PR_WORKFLOW) + + for job_id in ("e2e-runtime", "e2e-browser", "e2e-downloader-image"): + job = _job(source, job_id) + assert "AB_E2E_PROJECT:" in job + assert "AB_E2E_WORK_DIR" in job + assert "github.run_id" in job + assert "github.run_attempt" in job + assert "RUNNER_TEMP" in job + assert '--project-name "$AB_E2E_PROJECT"' in job + assert '--work-dir "$AB_E2E_WORK_DIR"' in job + assert "Ensure stack teardown" in job + assert "if: always()" in job + + browser = _job(source, "e2e-browser") + assert "matrix.project" in browser + + +def test_pr_commands_match_the_repository_entry_points() -> None: + source = _read(PR_WORKFLOW) + runtime = _job(source, "e2e-runtime") + browser = _job(source, "e2e-browser") + downloader = _job(source, "e2e-downloader-image") + + for job in (runtime, browser, downloader): + assert "e2e/scripts/build_test_image.py" in job + assert "e2e/scripts/stack.py run" in job + assert "e2e/scripts/collect_artifacts.py" in job + + assert "uv run --directory backend pytest src/test/e2e/runtime -m e2e" in runtime + assert "script: test:e2e:chromium" in browser + assert "script: test:e2e:webkit" in browser + assert 'pnpm --dir webui run "${{ matrix.script }}"' in browser + assert "--profile downloader" in downloader + assert "python3 e2e/scripts/run_downloader_lane.py" in downloader + + runner = _read(DOWNLOADER_RUNNER) + assert '"rss-download-rename.spec.ts"' in runner + assert '"--project=chromium-desktop"' in runner + assert '"src/test/e2e/downloader"' in runner + assert '"--junitxml=' in runner + + +def test_pr_dependencies_and_generated_outputs_are_cached() -> None: + source = _read(PR_WORKFLOW) + runtime = _job(source, "e2e-runtime") + browser = _job(source, "e2e-browser") + downloader = _job(source, "e2e-downloader-image") + + for job in (runtime, downloader): + assert "astral-sh/setup-uv@v4" in job + assert "enable-cache: true" in job + assert "cache-dependency-glob: backend/uv.lock" in job + assert "uv sync --directory backend --locked --group dev" in job + + for job in (runtime, browser, downloader): + assert "pnpm/action-setup@v4" in job + assert "cache: pnpm" in job + assert "cache-dependency-path: webui/pnpm-lock.yaml" in job + assert "pnpm --dir webui install --frozen-lockfile" in job + assert "path: webui/dist" in job + assert "AB_E2E_IMAGE_ARCHIVE" in job + assert "backend/src/**" in job + assert "webui/src/**" in job + + for job in (browser, downloader): + assert "PLAYWRIGHT_BROWSERS_PATH" in job + assert "playwright install --with-deps" in job + assert "hashFiles('webui/pnpm-lock.yaml')" in job + + +def test_pr_browser_runs_have_zero_retries_and_fourteen_day_artifacts() -> None: + source = _read(PR_WORKFLOW) + browser = _job(source, "e2e-browser") + downloader = _job(source, "e2e-downloader-image") + downloader_runner = _read(DOWNLOADER_RUNNER) + + assert source.count("--retries=0") == 1 + assert downloader_runner.count('"--retries=0"') == 1 + for command_source in (source, downloader_runner): + assert "-- --retries" not in command_source + assert not re.search(r"--retries(?:=|\s+)[1-9][0-9]*", command_source) + for job in (_job(source, "e2e-runtime"), browser, downloader): + assert "uses: actions/upload-artifact@v4" in job + assert "retention-days: 14" in job + assert "if-no-files-found: warn" in job + assert "if: always()" in job + + playwright = _read(PLAYWRIGHT_CONFIG) + assert re.search(r"\bretries:\s*0\b", playwright) + assert re.search(r"\bworkers:\s*1\b", playwright) + + +def test_nightly_is_non_pr_non_blocking_and_repeats_fresh_stacks() -> None: + source = _read(NIGHTLY_WORKFLOW) + header = _workflow_header(source) + job = _job(source, "e2e-nightly") + + assert "pull_request:" not in header + assert "schedule:" in header + assert "workflow_dispatch:" in header + assert "continue-on-error: true" in job + assert "timeout-minutes: 15" in job + assert job.count("lane: firefox") == 1 + assert job.count("lane: browser") == 2 + assert job.count("lane: downloader") == 2 + assert job.count("pass: 2") == 2 + assert "playwright-project: firefox-nightly" in job + assert "playwright-project: chromium-desktop" in job + assert "--retries=0" in job + assert "run test:e2e -- --project" not in job + assert "AB_E2E_PROJECT:" in job + assert "matrix.lane" in job + assert "matrix.pass" in job + assert "github.run_id" in job + assert "github.run_attempt" in job + assert "retention-days: 14" in job + assert "if: always()" in job + assert not re.search(r"--retries(?:=|\s+)[1-9][0-9]*", source) + + +def test_normal_backend_workflow_excludes_e2e_tests() -> None: + source = _read(BUILD_WORKFLOW) + assert re.search( + r'uv run pytest src/test -v -m ["\']not e2e["\']', + source, + ) + + +def test_readme_documents_local_operation_and_maintenance() -> None: + source = _read(E2E_README) + + for command in ( + "e2e/scripts/build_test_image.py", + "e2e/scripts/stack.py run", + "pytest src/test/e2e/runtime -m e2e", + "test:e2e:chromium", + "test:e2e:webkit", + "--project=firefox-nightly", + "e2e/scripts/generate_fixtures.py --check", + "e2e/scripts/audit_sources.py", + "docker buildx imagetools inspect", + ): + assert command in source + + assert "$AB_E2E_WORK_DIR/artifacts" in source + assert "webui/playwright-report" in source + assert "14 days" in source + assert "15-minute" in source + assert "sha256" in source + assert "admin/adminadmin" in source diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 00000000..b166a282 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,193 @@ +# Hermetic E2E tests + +The E2E suites run a production-built AutoBangumi image against local, +deterministic upstream services. Docker Compose publishes random loopback ports +and uses an internal network for service-to-service traffic. Tests must not use +public RSS, TMDB, image, player, or downloader endpoints. + +## Prerequisites + +- Docker Engine with Compose v2 +- Python 3.13 and `uv` +- Node.js 20 and pnpm 9.11 +- enough disk space for the AutoBangumi, Python, Playwright, and qBittorrent + images + +Install dependencies and browser binaries before starting a stack: + +```bash +uv sync --directory backend --locked --group dev +pnpm --dir webui install --frozen-lockfile +pnpm --dir webui exec playwright install --with-deps chromium webkit firefox +``` + +## Build the production test image + +The build helper places the compiled WebUI and generated version module in a +temporary Docker context. It does not write `dist` or +`module/__version__.py` into the source tree. + +```bash +pnpm --dir webui run build +python3 e2e/scripts/build_test_image.py \ + --version 3.3.999-e2e.1 \ + --image auto-bangumi:e2e \ + --dist webui/dist +``` + +## Run a lane locally + +Create a fresh project and work directory for every invocation. Reusing a work +directory also reuses setup state, which defeats first-run coverage. + +```bash +export LANE=runtime +export AB_E2E_PROJECT="ab-e2e-local-$LANE-$$" +export AB_E2E_WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/ab-e2e-$LANE.XXXXXX")" +export AB_E2E_APP_IMAGE=auto-bangumi:e2e +``` + +Runtime API and process coverage: + +```bash +python3 e2e/scripts/stack.py run \ + --profile browser \ + --project-name "$AB_E2E_PROJECT" \ + --work-dir "$AB_E2E_WORK_DIR" \ + -- uv run --directory backend pytest src/test/e2e/runtime -m e2e -q +``` + +Chromium desktop: + +```bash +python3 e2e/scripts/stack.py run \ + --profile browser \ + --project-name "$AB_E2E_PROJECT" \ + --work-dir "$AB_E2E_WORK_DIR" \ + -- pnpm --dir webui run test:e2e:chromium --retries=0 +``` + +WebKit at the 390×844 mobile viewport: + +```bash +python3 e2e/scripts/stack.py run \ + --profile browser \ + --project-name "$AB_E2E_PROJECT" \ + --work-dir "$AB_E2E_WORK_DIR" \ + -- pnpm --dir webui run test:e2e:webkit --retries=0 +``` + +Real qBittorrent plus the packaged-image workflow: + +```bash +python3 e2e/scripts/stack.py run \ + --profile downloader \ + --project-name "$AB_E2E_PROJECT" \ + --work-dir "$AB_E2E_WORK_DIR" \ + -- python3 e2e/scripts/run_downloader_lane.py +``` + +The downloader runner executes the focused Chromium download/rename spec and +`backend/src/test/e2e/downloader` inside that same fresh stack. + +Firefox is a nightly, non-required lane: + +```bash +python3 e2e/scripts/stack.py run \ + --profile browser \ + --project-name "$AB_E2E_PROJECT" \ + --work-dir "$AB_E2E_WORK_DIR" \ + -- pnpm --dir webui run test:e2e --project=firefox-nightly --retries=0 +``` + +The nightly workflow runs browser and downloader twice as separate matrix +passes. Each pass receives a fresh project and work directory; locally, repeat +the corresponding command after recreating both variables above. + +Useful lifecycle checks: + +```bash +python3 e2e/scripts/stack.py validate --profile browser +python3 e2e/scripts/stack.py validate --profile downloader +python3 e2e/scripts/stack.py smoke --profile browser +python3 e2e/scripts/stack.py ports --profile browser \ + --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" +python3 e2e/scripts/stack.py env --profile browser \ + --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" +python3 e2e/scripts/stack.py restart --profile browser \ + --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" app +python3 e2e/scripts/stack.py stop --profile browser \ + --project-name "$AB_E2E_PROJECT" --work-dir "$AB_E2E_WORK_DIR" +``` + +## Duration and artifacts + +The pull-request checks have a hard 15-minute limit. Typical targets are 5–8 +minutes for runtime, 8–12 minutes for each browser, and 10–15 minutes for the +real-downloader/image lane. + +`stack.py run` collects diagnostics before Compose teardown, including +Compose state and logs, mock-service request journals, qBittorrent state, +backend logs, and SQLite files. Local stack artifacts are written to +`$AB_E2E_WORK_DIR/artifacts`. Playwright writes screenshots and videos to +`webui/test-results`, with its HTML report in `webui/playwright-report`. +Raw Playwright traces are disabled because they serialize authenticated +storage state and response bodies; token-reveal tests also disable visual +recordings. Browser console errors and failed responses remain available in +the redacted `browser-diagnostics.json` attachment. CI uploads these paths +with `if: always()` and retains them for 14 days. Never print credentials or +session material from a test. + +## Legacy workflow migration + +The former numbered, session-scoped workflow and its fixed ports were removed. +Setup/auth/config/restart persistence now live in isolated runtime tests; +parser, offset-review, and Windows-path regressions have focused runtime and +browser scenarios; the real qB lane owns download, rename, idempotency, and +file deletion. Endpoint shape, validation, CRUD, empty-list, log, search, and +notification contracts remain in the faster `backend/src/test/test_api_*.py` +unit/integration suites. No test depends on state created by an earlier test. + +## Add or change a mock scenario + +1. Put static RSS, TMDB, player, torrent, image, or file data under + `e2e/fixtures`. Fixture URLs must use only Compose service names or + loopback addresses. +2. Add named TMDB behavior to + `e2e/fixtures/tmdb/scenarios.json`. +3. Select or reset scenarios through `/__admin/scenario/` and + `/__admin/reset`; assert requests through + `/__admin/requests` or `/__admin/state`. +4. Keep responses deterministic and add a contract test for any new endpoint, + error mode, or request field. +5. Run `python3 e2e/scripts/audit_sources.py` before opening a PR. + +Regenerate and verify the deterministic media and torrent assets with: + +```bash +python3 e2e/scripts/generate_fixtures.py +python3 e2e/scripts/generate_fixtures.py --check +``` + +Commit the generated media, torrent, and checksum changes together. + +## Update the qBittorrent image digest + +The downloader overlay must retain both an exact patch tag and an immutable +manifest digest. To update it: + +1. Choose the exact linuxserver/qbittorrent patch release and inspect its + published manifest: + + ```bash + docker buildx imagetools inspect linuxserver/qbittorrent:5.2.3 + ``` + +2. Replace only the tag and `sha256` digest in + `e2e/compose/downloader.yml`. Never use `latest` or a + moving major/minor tag. +3. Pull that exact reference, then run the downloader validation and smoke + checks. Startup verifies the fixed `admin/adminadmin` credentials + and rejects enabled DHT, PeX, LSD, UPnP, RSS processing, or a changed save + path. +4. Run the downloader Playwright lane and inspect its artifacts before merging.