mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-01 04:47:03 +08:00
refactor: move site collector releases and sources to independent repository
This commit is contained in:
2
.github/ISSUE_TEMPLATE/feature_request.yml
vendored
2
.github/ISSUE_TEMPLATE/feature_request.yml
vendored
@@ -8,7 +8,7 @@ body:
|
||||
value: |
|
||||
请说明你希望添加的功能。
|
||||
|
||||
站点适配请求请从 [采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 下载最新采集器,再按 [站点适配采集说明](https://github.com/jxxghp/MoviePilot/blob/v3/docs/site-adapter-capture.md) 生成脱敏 ZIP,并在下方附加。Issue 及附件是公开内容,提交前必须解压预览四个文件。不要上传 Cookie、Authorization、通行密钥、会话字段或任何原始数据。
|
||||
站点适配请求请从 [采集器独立 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest) 下载最新采集器,再按 [站点适配采集说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md) 生成脱敏 ZIP,并在下方附加。Issue 及附件是公开内容,提交前必须解压预览四个文件。不要上传 Cookie、Authorization、通行密钥、会话字段或任何原始数据。
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
|
||||
257
.github/workflows/site-adapter-collector.yml
vendored
257
.github/workflows/site-adapter-collector.yml
vendored
@@ -1,257 +0,0 @@
|
||||
name: Site Adapter Collector
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
paths:
|
||||
- scripts/site_adapter_collector.py
|
||||
- scripts/site_adapter_collector.spec
|
||||
- scripts/site_adapter_collector_requirements.txt
|
||||
- scripts/start-site-adapter-collector.command
|
||||
- scripts/collect-site-adapter.sh
|
||||
- scripts/collector/**
|
||||
- .github/workflows/site-adapter-collector.yml
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
publish:
|
||||
description: Publish an independent release only when collector inputs changed
|
||||
required: true
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
# 串行选择版本和发布,避免多个采集器提交争抢同一个版本。
|
||||
concurrency:
|
||||
group: site-adapter-collector-release
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Check collector changes and select independent version
|
||||
if: github.ref == 'refs/heads/v3'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changed: ${{ steps.version.outputs.changed }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
previous_tag: ${{ steps.version.outputs.previous_tag }}
|
||||
source_sha: ${{ steps.version.outputs.source_sha }}
|
||||
steps:
|
||||
- name: Checkout current collector source
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
# 排队期间可能已有新提交;统一构建当前 v3,避免旧任务回退采集器。
|
||||
ref: v3
|
||||
fetch-depth: 0
|
||||
fetch-tags: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Select collector release version
|
||||
id: version
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --paginate \
|
||||
--jq '.[] | select(.draft == false and .prerelease == false) | .tag_name' \
|
||||
> collector-release-tags.txt
|
||||
python scripts/collector/release.py plan \
|
||||
--tags-file collector-release-tags.txt --output "$GITHUB_OUTPUT"
|
||||
|
||||
build:
|
||||
name: Build ${{ matrix.platform_name }} collector
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.changed == 'true' || (github.event_name == 'workflow_dispatch' && !inputs.publish)
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform_name: Windows
|
||||
platform_id: windows
|
||||
runner: windows-latest
|
||||
source_name: moviepilot-site-collector.exe
|
||||
asset_name: moviepilot-site-collector-windows.exe
|
||||
artifact_name: site-adapter-collector-windows
|
||||
- platform_name: macOS
|
||||
platform_id: macos
|
||||
runner: macos-latest
|
||||
source_name: moviepilot-site-collector
|
||||
asset_name: MoviePilot-Site-Collector-macOS.zip
|
||||
artifact_name: site-adapter-collector-macos
|
||||
- platform_name: Linux
|
||||
platform_id: linux
|
||||
runner: ubuntu-latest
|
||||
source_name: moviepilot-site-collector
|
||||
asset_name: moviepilot-site-collector-linux
|
||||
artifact_name: site-adapter-collector-linux
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ needs.prepare.outputs.source_sha }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@v10.0.1
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: scripts/site_adapter_collector_requirements.txt
|
||||
|
||||
- name: Install build dependencies
|
||||
run: uv pip install --system --requirement scripts/site_adapter_collector_requirements.txt
|
||||
|
||||
- name: Build single-file collector
|
||||
shell: bash
|
||||
env:
|
||||
COLLECTOR_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
python scripts/collector/release.py stamp --version "$COLLECTOR_VERSION"
|
||||
pyinstaller --clean --noconfirm scripts/site_adapter_collector.spec
|
||||
|
||||
- name: Smoke-test collector
|
||||
env:
|
||||
SOURCE_NAME: ${{ matrix.source_name }}
|
||||
COLLECTOR_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
python -c "import os, subprocess; from pathlib import Path; subprocess.run([str((Path('dist') / os.environ['SOURCE_NAME']).resolve()), '--help'], check=True)"
|
||||
python -c "import os, subprocess; from pathlib import Path; result = subprocess.run([str((Path('dist') / os.environ['SOURCE_NAME']).resolve()), '--version'], check=True, capture_output=True, text=True); assert result.stdout.strip() == os.environ['COLLECTOR_VERSION'], result.stdout"
|
||||
|
||||
- name: Package macOS double-click archive
|
||||
if: matrix.platform_id == 'macos'
|
||||
shell: bash
|
||||
env:
|
||||
ASSET_NAME: ${{ matrix.asset_name }}
|
||||
SOURCE_NAME: ${{ matrix.source_name }}
|
||||
run: |
|
||||
package_dir="dist/MoviePilot-Collector"
|
||||
mkdir -p "$package_dir"
|
||||
cp "dist/$SOURCE_NAME" "$package_dir/moviepilot-site-collector-macos"
|
||||
cp scripts/start-site-adapter-collector.command "$package_dir/start-site-adapter-collector.command"
|
||||
chmod +x "$package_dir/moviepilot-site-collector-macos"
|
||||
chmod +x "$package_dir/start-site-adapter-collector.command"
|
||||
cd dist
|
||||
COPYFILE_DISABLE=1 zip -q -r -X "$ASSET_NAME" MoviePilot-Collector
|
||||
|
||||
- name: Rename Windows and Linux collector
|
||||
if: matrix.platform_id != 'macos'
|
||||
env:
|
||||
ASSET_NAME: ${{ matrix.asset_name }}
|
||||
SOURCE_NAME: ${{ matrix.source_name }}
|
||||
run: |
|
||||
python -c "import os; from pathlib import Path; (Path('dist') / os.environ['SOURCE_NAME']).replace(Path('dist') / os.environ['ASSET_NAME'])"
|
||||
|
||||
- name: Generate SHA-256 checksum
|
||||
env:
|
||||
ASSET_NAME: ${{ matrix.asset_name }}
|
||||
run: |
|
||||
python -c "import hashlib, os; from pathlib import Path; path = Path('dist') / os.environ['ASSET_NAME']; path.with_name(path.name + '.sha256').write_text(f'{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n', encoding='utf-8', newline='\n')"
|
||||
|
||||
- name: Upload collector artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ${{ matrix.artifact_name }}
|
||||
path: |
|
||||
dist/${{ matrix.asset_name }}
|
||||
dist/${{ matrix.asset_name }}.sha256
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
|
||||
publish:
|
||||
name: Publish independent collector release
|
||||
if: needs.prepare.outputs.changed == 'true' && (github.event_name == 'push' || inputs.publish)
|
||||
needs:
|
||||
- prepare
|
||||
- build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Checkout release source
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ needs.prepare.outputs.source_sha }}
|
||||
fetch-depth: 0
|
||||
fetch-tags: false
|
||||
|
||||
- name: Download collector artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: site-adapter-collector-*
|
||||
path: release-assets
|
||||
merge-multiple: true
|
||||
|
||||
- name: Generate collector release notes
|
||||
env:
|
||||
COLLECTOR_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
PREVIOUS_TAG: ${{ needs.prepare.outputs.previous_tag }}
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
if [[ -n "$PREVIOUS_TAG" ]]; then
|
||||
git fetch --no-tags origin "refs/tags/$PREVIOUS_TAG:refs/tags/$PREVIOUS_TAG"
|
||||
fi
|
||||
python - <<'PY'
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from scripts.collector.release import RELEASE_PATHS
|
||||
|
||||
version = os.environ['COLLECTOR_VERSION']
|
||||
previous = os.environ['PREVIOUS_TAG']
|
||||
revision = f"{previous}..HEAD" if previous else "HEAD"
|
||||
records = subprocess.check_output(
|
||||
['git', 'log', revision, '--no-merges', '--format=- %s (%h) — %an', '--', *RELEASE_PATHS],
|
||||
text=True,
|
||||
).strip()
|
||||
notes = (
|
||||
f"MoviePilot 站点适配采集器 {version},独立于后端版本发布。\n\n"
|
||||
"下载与系统对应的程序及同名 `.sha256` 校验文件。\n\n"
|
||||
"[使用说明](https://github.com/jxxghp/MoviePilot/blob/v3/docs/site-adapter-collector-release.md)\n\n"
|
||||
f"源码提交:`{os.environ['SOURCE_SHA']}`\n\n## 变更记录\n\n{records}\n"
|
||||
)
|
||||
Path('collector-release-notes.md').write_text(notes, encoding='utf-8')
|
||||
PY
|
||||
|
||||
- name: Create and publish independent collector release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ needs.prepare.outputs.tag }}
|
||||
COLLECTOR_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
if gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
|
||||
--json isDraft,targetCommitish --jq '[.isDraft, .targetCommitish] | @tsv' \
|
||||
> collector-draft-state.txt; then
|
||||
# 草稿尚未发布时标签可能不存在,应核对创建草稿时绑定的源码提交。
|
||||
IFS=$'\t' read -r is_draft draft_source < collector-draft-state.txt
|
||||
if [[ "$is_draft" != "true" || "$draft_source" != "$SOURCE_SHA" ]]; then
|
||||
echo "Refusing to overwrite a published release or a draft from another source commit" >&2
|
||||
exit 1
|
||||
fi
|
||||
gh release upload "$RELEASE_TAG" release-assets/* \
|
||||
--clobber --repo "$GITHUB_REPOSITORY"
|
||||
gh release edit "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
|
||||
--title "Site Adapter Collector v$COLLECTOR_VERSION" \
|
||||
--notes-file collector-release-notes.md
|
||||
else
|
||||
gh release create "$RELEASE_TAG" release-assets/* \
|
||||
--repo "$GITHUB_REPOSITORY" --target "$SOURCE_SHA" --draft \
|
||||
--title "Site Adapter Collector v$COLLECTOR_VERSION" \
|
||||
--notes-file collector-release-notes.md --latest=false
|
||||
fi
|
||||
# 全部附件上传完成才转正式发布,上传失败不会消耗正式版本。
|
||||
gh release edit "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
|
||||
--draft=false --latest=false
|
||||
@@ -61,8 +61,8 @@ curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v3/scripts/bootst
|
||||
- 文档规则入口:[docs/rules/README.md](docs/rules/README.md)
|
||||
- 开发环境与本地源码运行:[docs/development-setup.md](docs/development-setup.md)
|
||||
- 测试说明:[docs/testing.md](docs/testing.md)
|
||||
- 新站点适配采集与 Feature Request 提交:[docs/site-adapter-capture.md](docs/site-adapter-capture.md)
|
||||
- 站点采集器下载:[独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true),仅在采集器变化时递增版本并发布。
|
||||
- 新站点适配采集与 Feature Request 提交:[MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md)
|
||||
- 站点采集器下载:[独立 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest),仅在采集器变化时递增版本并发布。
|
||||
- REST API 文档:https://api.movie-pilot.org
|
||||
- 插件开发说明:https://wiki.movie-pilot.org/zh/plugindev
|
||||
|
||||
|
||||
@@ -59,7 +59,8 @@ Before contributing, read the repository rules and local environment guide, keep
|
||||
- Rule index: [docs/rules/README.md](docs/rules/README.md)
|
||||
- Development setup and local source run: [docs/development-setup.md](docs/development-setup.md)
|
||||
- Testing guide: [docs/testing.md](docs/testing.md)
|
||||
- New site adapter capture and Feature Request submission: [docs/site-adapter-capture.md](docs/site-adapter-capture.md)
|
||||
- New site adapter capture and Feature Request submission: [MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md)
|
||||
- Site collector downloads: [independent repository Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest), published only when collector inputs change.
|
||||
- REST API documentation: https://api.movie-pilot.org
|
||||
- Plugin development guide: https://wiki.movie-pilot.org/zh/plugindev
|
||||
|
||||
|
||||
@@ -234,7 +234,7 @@ Docker 镜像直接基于官方基础镜像构建,只安装应用运行所需
|
||||
报告,不自动更新基线。普通主仓改动不要求额外检出插件仓或运行观察任务。宿主架构门禁和 changed-file
|
||||
Pylint 由 `v3` PR/push 的 GitHub Actions 执行,`app/` 全量 Pylint 是建议性报告。
|
||||
|
||||
站点采集器使用独立的 `site-adapter-collector-v1.0.x` Release,仅在采集器发布输入变化时递增版本并构建,不跟随后端 Release。下载入口为 [采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true),自动版本和手动测试构建规则见 [采集器发布说明](site-adapter-collector-release.md#维护者发布流程)。
|
||||
站点采集器源码与构建工作流位于 [MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector),使用独立的 `site-adapter-collector-v1.0.x` Release,仅在采集器发布输入变化时递增版本并构建,不跟随后端 Release。下载入口为 [采集器独立 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest),自动版本和手动测试构建规则见 [采集器发布说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-collector-release.md#维护者发布流程)。
|
||||
|
||||
Ruff/Mypy 基线只允许收紧,不接受新增诊断或类型错误增长;受治零错误文件由 `mypy.ini` 的
|
||||
`files=` 维护。Mypy 完整 ratchet 固定按 Linux/Python 3.14 分析。Coverage job 在 `v3` PR/push
|
||||
|
||||
@@ -286,14 +286,15 @@ moviepilot help scheduler
|
||||
## Site Adapter Capture — macOS / Linux
|
||||
|
||||
```bash
|
||||
# Run from a MoviePilot source checkout and reuse its virtual environment
|
||||
# Run from the independent MoviePilot-Site-Collector checkout and reuse its virtual environment
|
||||
cd /path/to/MoviePilot-Site-Collector
|
||||
bash scripts/collect-site-adapter.sh
|
||||
```
|
||||
|
||||
**Rules:**
|
||||
- The default collector asks only for the site HTTPS address, opens an isolated local Chrome/Edge profile, and reads the completed search page after the user confirms.
|
||||
- Users must not be asked to inspect HTML or copy Cookie/User-Agent values in the default flow. `--manual-cookie` is an advanced fallback only.
|
||||
- Run only the collector shipped with a trusted local MoviePilot source checkout or installation package. Do not pipe a remote branch script into a shell.
|
||||
- Run only the collector from a trusted local [MoviePilot-Site-Collector checkout](https://github.com/jxxghp/MoviePilot-Site-Collector) or its [official Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest). Do not pipe a remote branch script into a shell.
|
||||
- Never put a Cookie or other credential in command arguments or shell history.
|
||||
- Feature Request attachments are public. Review all four files in the generated ZIP before attaching it, and never attach raw HTML, HAR, or browser network archives.
|
||||
|
||||
|
||||
@@ -1,60 +1,11 @@
|
||||
# 站点适配采集
|
||||
|
||||
当开发者没有目标站点账号时,可以由已有账号的用户在本地采集一份经过裁剪和脱敏的搜索页结构,并把采集 ZIP 附加到站点适配 Feature Request。开发者和自动化流程只处理脱敏包,不需要获取用户账号。
|
||||
站点采集器已迁入独立仓库 [MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector)。普通用户无需安装 Python、Git 或 MoviePilot 后端:
|
||||
|
||||
## 普通用户一键采集
|
||||
1. 从 [采集器最新 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest) 下载对应系统的程序。
|
||||
2. 按 [采集说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md) 在临时浏览器中登录、搜索并生成脱敏 ZIP。
|
||||
3. 在本仓库创建“功能改进” Issue,选择“站点适配”,附加生成的 `moviepilot-site-capture-*.zip`。
|
||||
|
||||
普通用户只需准备两样东西:目标站点账号,以及已安装的 Chrome、Edge 或 Chromium。无需安装 Python、Git、MoviePilot、Docker,也不需要查看 HTML、复制 Cookie 或填写 User-Agent。
|
||||
Issue 及附件是公开内容。提交前请解压预览 `manifest.json`、`request.json`、`search.html`、`redaction-report.json`,确认没有账号或隐私信息。不要上传 Cookie、Authorization、通行密钥、原始 HTML、HAR 或浏览器网络归档。
|
||||
|
||||
1. 从官方 [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 选择最新采集器版本,下载与 Windows、macOS 或 Linux 对应的程序;各系统文件名见下方下载说明。
|
||||
2. 运行采集器,只输入站点首页地址,例如 `https://tracker.example.com`。
|
||||
3. 程序会打开一个临时浏览器窗口。在这个窗口里正常登录站点,搜索一个能返回至少 3 条结果的常见关键词,并保持搜索结果页打开。
|
||||
4. 回到采集器按回车。程序会自动识别搜索地址、关键词、Cookie 和 User-Agent,完成本地裁剪与脱敏后生成 ZIP。
|
||||
|
||||
临时浏览器使用独立的一次性用户目录,不会读取日常浏览器的历史登录状态。采集完成后程序会关闭临时浏览器并清理这次登录数据。原始页面和 Cookie 只在内存中处理,不会写入采集包。
|
||||
|
||||
各系统下载文件和首次运行方式见 [站点适配采集器下载说明](site-adapter-collector-release.md)。
|
||||
|
||||
## 开发者源码入口
|
||||
|
||||
已经有 MoviePilot 源码和 Python 环境的开发者,也可以在项目目录执行:
|
||||
|
||||
```bash
|
||||
bash scripts/collect-site-adapter.sh
|
||||
```
|
||||
|
||||
源码入口和独立程序使用同一套浏览器采集流程。只有排查兼容问题时才使用 `--manual-cookie` 高级模式;普通用户不需要接触 Cookie。
|
||||
|
||||
## 第一版限制
|
||||
|
||||
当前采集器会读取浏览器渲染后的页面,因此可由用户在临时窗口中完成验证码、Cloudflare 检查和普通登录。但自动适配协议仍要求搜索结果地址能够表示为 HTTPS GET URL。以下场景第一版不做自动适配:
|
||||
|
||||
- 必须 POST 表单才能搜索的站点。
|
||||
- 搜索完成后地址栏完全没有关键词或可复用搜索参数的站点。
|
||||
- 需要专用 API、复杂签名或无法从一次搜索结果页观察出 FREE/HR 规则的站点。
|
||||
|
||||
遇到这些场景时,请在 Feature Request 中说明失败步骤和终端错误文字,等待人工确认采集方案。不要用原始 HAR、原始 HTML 或包含账号信息的截图替代脱敏 ZIP。
|
||||
|
||||
## 脱敏范围
|
||||
|
||||
采集器只接受 HTTPS 地址。默认模式从本机临时浏览器只读当前搜索结果页;高级手动模式设置 30 秒超时和 5 MiB 响应上限,并禁止携带 Cookie 跨 origin 重定向。读取页面后会在本机完成以下处理:
|
||||
|
||||
- 只保留种子列表、表头和最多 25 条结果相关 DOM,丢弃账号导航、页脚、脚本、样式、隐藏表单和其他页面内容。
|
||||
- 替换种子标题、用户名、邮箱、IP、时间、大小、统计值和长随机标识,只保留适配所需的标签、class、字段与链接结构。
|
||||
- URL 转为同源相对路径,查询值统一替换为占位符,凭据语义的字段直接移除。
|
||||
- Cookie 和浏览器 UA 仅用于本次请求,不写入采集包;写入前还会使用 Cookie 原值执行二次泄露检查。
|
||||
|
||||
输出 ZIP 根目录固定包含:
|
||||
|
||||
- `manifest.json`:包版本、站点标识、采集时间、结果行数、HTML 摘要和隐私声明。
|
||||
- `request.json`:仅包含 GET、origin、相对路径和脱敏后的查询参数,搜索值固定为 `{keyword}`。
|
||||
- `search.html`:本地裁剪并脱敏后的种子列表结构。
|
||||
- `redaction-report.json`:脱敏状态和各类处理计数。
|
||||
|
||||
## 提交 Feature Request
|
||||
|
||||
在 GitHub 创建“功能改进” Issue,类型选择“站点适配”,然后把生成的 `moviepilot-site-capture-*.zip` 直接拖入“站点适配采集文件”输入框。
|
||||
|
||||
Feature Request 及其附件是公开内容。提交前请先在本地解压 ZIP,确认根目录只有上述四个文件,并逐一预览确认没有站点账号、搜索隐私或其他不希望公开的信息。
|
||||
|
||||
站点适配请求必须附加采集器生成并人工复核过的 ZIP。严禁手工上传 Cookie、Authorization、通行密钥、会话字段、原始 HTML、原始 HAR、浏览器网络归档或截图中的账号信息。如果采集失败,请只提交终端错误文字,不要用任何原始数据代替脱敏包。
|
||||
源码运行、采集限制与脱敏协议均见 [独立仓库完整说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md)。本页面保留以兼容旧文档链接。
|
||||
|
||||
@@ -1,42 +1,12 @@
|
||||
# 站点适配采集器下载说明
|
||||
|
||||
普通用户优先使用 [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 提供的单文件采集器,选择列表中最新版本。采集器使用 `site-adapter-collector-v1.0.x` 独立标签,与 MoviePilot 后端版本分开发布。单文件已经包含 Python 和采集器依赖,不需要安装 Python、pip、Git、MoviePilot 后端,也不需要下载源码。电脑只需已安装 Chrome、Edge 或 Chromium 浏览器。
|
||||
站点采集器的源码、版本、构建工作流和历史 Release 已迁入独立仓库 [MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector),与 MoviePilot 主程序发布完全分开。
|
||||
|
||||
## 选择下载文件
|
||||
请从 [采集器最新 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest) 下载 Windows、macOS 或 Linux 程序及同名 `.sha256` 文件。
|
||||
|
||||
请只从官方 [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 下载与系统匹配的文件;MoviePilot 主程序 Release 不再提供采集器附件:
|
||||
|
||||
| 系统 | 下载文件 | 用户侧运行环境 |
|
||||
|---|---|---|
|
||||
| Windows | `moviepilot-site-collector-windows.exe` | Chrome、Edge 或 Chromium |
|
||||
| macOS | `MoviePilot-Site-Collector-macOS.zip` | Chrome、Edge 或 Chromium |
|
||||
| Linux | `moviepilot-site-collector-linux` | Chrome、Edge 或 Chromium |
|
||||
|
||||
每个程序旁边还有同名的 `.sha256` 文件,可用于核对下载文件是否完整。程序启动时显示版本,也可执行 `--version` 查看。GitHub Actions 的手动构建产物主要用于维护者测试;普通用户应使用采集器独立 Release 资产。
|
||||
|
||||
## 运行采集器
|
||||
|
||||
Windows 用户下载后双击 `.exe`,按窗口提示操作即可。macOS 用户解压 ZIP 后双击 `start-site-adapter-collector.command`,不要打开构建目录中的 `.pkg` 文件。Linux 用户在下载目录打开终端,只需首次赋予执行权限后运行:
|
||||
|
||||
```bash
|
||||
chmod +x moviepilot-site-collector-linux
|
||||
./moviepilot-site-collector-linux
|
||||
```
|
||||
|
||||
运行后只需输入站点首页地址,随后在弹出的临时浏览器中登录并搜索,最后回到采集器按回车。采集器会在当前目录生成 `moviepilot-site-capture-*.zip`,用户只需把这个 ZIP 附加到站点适配 Feature Request,不需要提交任何源码、Cookie 或 HTML。
|
||||
|
||||
## 系统安全提示
|
||||
|
||||
当前自动构建产物尚未接入 Windows 或 Apple 代码签名。Windows SmartScreen 或 macOS Gatekeeper 可能因此显示安全提示。仅在文件来自官方 [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true),且校验摘要一致时运行;不要从聊天、网盘或第三方站点接收采集器。
|
||||
|
||||
如果系统阻止运行,可改用随 MoviePilot 源码提供的本地采集脚本;该方式需要 Python 3.14+ 及完整后端依赖,不适合作为普通用户的首选路径。
|
||||
- [完整下载与运行说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-collector-release.md)
|
||||
- [采集流程和源码入口](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md)
|
||||
|
||||
## 维护者发布流程
|
||||
|
||||
`.github/workflows/site-adapter-collector.yml` 在 `v3` 分支的采集器源码、依赖、启动脚本或打包工作流变化时触发,支持手动运行。后端 Release 发布及普通后端代码、文档、测试改动不会触发采集器构建。
|
||||
|
||||
构建前会与上一个已正式发布的采集器标签比较输入文件。没有变化就跳过构建和发版;首次发布沿用源码中的 `COLLECTOR_VERSION`(当前为 `1.0.1`),后续有变化时自动递增补丁版本。工作流只在 Runner 构建副本中写入版本,使程序、`--version` 和脱敏包中的 `collector_version` 一致,不向 `v3` 分支回写版本提交。采集包的 `format_version` 协议版本保持独立。
|
||||
|
||||
每个平台先执行 `--help` 和 `--version` 启动检查,再上传程序及 SHA-256 摘要。三个平台全部成功后,工作流创建独立版本 Release,自动生成仅包含采集器相关提交的变更记录,并设置 `latest=false`,使主程序的最新 Release 入口继续指向后端版本。
|
||||
|
||||
手动触发时 `publish` 默认为 `true`,仍遵守“输入变化才发版”。设为 `false` 时可重复构建测试,只生成保留 3 天的 Artifact,不创建 Release、不消耗版本号。历史主程序 Release 中的采集器附件会迁移移除,下载请始终使用上面的独立 Release 入口。
|
||||
采集器仅在自身发布输入变化时自动递增版本并发布,不跟随 MoviePilot 发版。维护者请使用 [独立仓库发布流程](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-collector-release.md#维护者发布流程)。本页面保留以兼容旧文档链接。
|
||||
|
||||
@@ -399,7 +399,7 @@ MoviePilot/app/application/site
|
||||
- 后端用户数据解析器可以自动探测 Gazelle、NexusPHP 等站点变种。
|
||||
- 解析模板支持更完整的标题、文件名、标签和可选字段回退。
|
||||
- 构建产物覆盖 Linux、Windows、macOS、x86_64、Arm64,以及 V3t 所需的 Python 3.14t ABI。
|
||||
- [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 提供生成脱敏样本的程序,维护者可通过自动化流程匹配模板并创建待审核的适配 PR。
|
||||
- [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest) 提供生成脱敏样本的程序,维护者可通过自动化流程匹配模板并创建待审核的适配 PR。
|
||||
|
||||
### 7.4 用户遇到站点问题时应如何判断
|
||||
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ORIGINAL_DIR="$PWD"
|
||||
SCRIPT_DIR=""
|
||||
PROJECT_ROOT=""
|
||||
PYTHON_BIN=""
|
||||
COLLECTOR_PATH=""
|
||||
|
||||
# 判断候选 Python 是否满足 3.14 最低版本。
|
||||
python_version_ok() {
|
||||
"$1" - <<'PY' >/dev/null 2>&1
|
||||
import sys
|
||||
raise SystemExit(0 if sys.version_info >= (3, 14) else 1)
|
||||
PY
|
||||
}
|
||||
|
||||
# 从当前源码目录、环境变量或脚本位置查找 MoviePilot 根目录。
|
||||
find_project_root() {
|
||||
local candidate=""
|
||||
local source_path="${BASH_SOURCE[0]:-}"
|
||||
if [[ -n "$source_path" && -f "$source_path" ]]; then
|
||||
SCRIPT_DIR="$(cd "$(dirname "$source_path")" && pwd)"
|
||||
fi
|
||||
for candidate in "${MOVIEPILOT_ROOT:-}" "$ORIGINAL_DIR" "${SCRIPT_DIR:+$SCRIPT_DIR/..}"; do
|
||||
if [[ -n "$candidate" && -f "$candidate/scripts/site_adapter_collector.py" ]]; then
|
||||
PROJECT_ROOT="$(cd "$candidate" && pwd)"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# 查找可用的项目虚拟环境或系统 Python。
|
||||
find_python() {
|
||||
local candidate=""
|
||||
local resolved=""
|
||||
for candidate in \
|
||||
"${PROJECT_ROOT:+$PROJECT_ROOT/venv/bin/python}" \
|
||||
"${PROJECT_ROOT:+$PROJECT_ROOT/.venv/bin/python}" \
|
||||
"${VIRTUAL_ENV:+$VIRTUAL_ENV/bin/python}" \
|
||||
python3.14 python3; do
|
||||
[[ -n "$candidate" ]] || continue
|
||||
if [[ "$candidate" == */* ]]; then
|
||||
resolved="$candidate"
|
||||
else
|
||||
resolved="$(command -v "$candidate" 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -n "$resolved" && -x "$resolved" ]] && python_version_ok "$resolved"; then
|
||||
PYTHON_BIN="$resolved"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# 判断 Python 是否已具备独立采集器所需的最小运行依赖。
|
||||
project_runtime_ready() {
|
||||
PYTHONPATH="$PROJECT_ROOT${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" - <<'PY' >/dev/null 2>&1
|
||||
import requests
|
||||
import websocket
|
||||
from bs4 import BeautifulSoup
|
||||
PY
|
||||
}
|
||||
|
||||
# 让脚本始终从终端安全读取交互输入。
|
||||
restore_terminal_input() {
|
||||
if [[ -r /dev/tty ]]; then
|
||||
exec </dev/tty
|
||||
else
|
||||
echo "需要可交互终端读取站点地址并等待浏览器采集确认。" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 定位本地项目运行环境后启动随发行包提供的采集器。
|
||||
main() {
|
||||
if ! find_project_root; then
|
||||
echo "未找到本地 MoviePilot 源码或安装目录,请在 MoviePilot 目录中运行此脚本。" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! find_python; then
|
||||
echo "未找到 Python 3.11 或更高版本,请先安装 Python。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! project_runtime_ready; then
|
||||
echo "本地 Python 环境缺少采集器依赖,请下载采集器独立 Release:https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true" >&2
|
||||
exit 1
|
||||
fi
|
||||
COLLECTOR_PATH="$PROJECT_ROOT/scripts/site_adapter_collector.py"
|
||||
|
||||
restore_terminal_input
|
||||
cd "$ORIGINAL_DIR"
|
||||
PYTHONPATH="$PROJECT_ROOT${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" "$COLLECTOR_PATH" "$@"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1 +0,0 @@
|
||||
"""站点采集器的独立发布工具。"""
|
||||
@@ -1,116 +0,0 @@
|
||||
"""比较采集器发布输入,按独立 Release 递增版本并写入构建副本。"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
TAG_PREFIX = "site-adapter-collector-v"
|
||||
RELEASE_PATHS = (
|
||||
"scripts/site_adapter_collector.py",
|
||||
"scripts/site_adapter_collector.spec",
|
||||
"scripts/site_adapter_collector_requirements.txt",
|
||||
"scripts/start-site-adapter-collector.command",
|
||||
"scripts/collect-site-adapter.sh",
|
||||
"scripts/collector",
|
||||
".github/workflows/site-adapter-collector.yml",
|
||||
)
|
||||
VERSION_PATTERN = re.compile(r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)")
|
||||
SOURCE_VERSION_PATTERN = re.compile(r'^COLLECTOR_VERSION = "([\d.]+)"$', re.MULTILINE)
|
||||
|
||||
|
||||
def _version_parts(version: str) -> tuple[int, int, int]:
|
||||
"""校验独立版本,避免外部标签进入 Git 参数或构建源码。"""
|
||||
match = VERSION_PATTERN.fullmatch(version)
|
||||
if not match:
|
||||
raise ValueError(f"无效采集器版本:{version}")
|
||||
return tuple(int(part) for part in match.groups())
|
||||
|
||||
|
||||
def latest_release_tag(tags: list[str]) -> Optional[str]:
|
||||
"""从已正式发布的标签中选取采集器最高版本,忽略后端版本。"""
|
||||
collector_tags = [
|
||||
tag for tag in tags
|
||||
if tag.startswith(TAG_PREFIX) and VERSION_PATTERN.fullmatch(tag[len(TAG_PREFIX):])
|
||||
]
|
||||
return max(collector_tags, key=lambda tag: _version_parts(tag[len(TAG_PREFIX):]), default=None)
|
||||
|
||||
|
||||
def release_plan(repository: Path, previous_tag: Optional[str]) -> dict[str, str]:
|
||||
"""仅比较采集器输入;未变化时跳过发布,首次沿用源码版本。"""
|
||||
source = repository / "scripts/site_adapter_collector.py"
|
||||
source_match = SOURCE_VERSION_PATTERN.search(source.read_text(encoding="utf-8"))
|
||||
if not source_match:
|
||||
raise ValueError("采集器源码缺少唯一版本声明")
|
||||
source_version = source_match.group(1)
|
||||
source_parts = _version_parts(source_version)
|
||||
source_sha = subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], cwd=repository, text=True,
|
||||
).strip()
|
||||
changed = True
|
||||
version = source_version
|
||||
if previous_tag:
|
||||
previous_parts = _version_parts(previous_tag.removeprefix(TAG_PREFIX))
|
||||
result = subprocess.run(
|
||||
["git", "diff", "--quiet", previous_tag, "HEAD", "--", *RELEASE_PATHS],
|
||||
cwd=repository, check=False,
|
||||
)
|
||||
if result.returncode not in (0, 1):
|
||||
raise RuntimeError(f"无法比较采集器 Release:{previous_tag}")
|
||||
changed = result.returncode == 1
|
||||
if changed:
|
||||
next_parts = max(source_parts, (*previous_parts[:2], previous_parts[2] + 1))
|
||||
version = ".".join(str(part) for part in next_parts)
|
||||
else:
|
||||
version = previous_tag[len(TAG_PREFIX):]
|
||||
return {
|
||||
"changed": str(changed).lower(),
|
||||
"version": version,
|
||||
"tag": f"{TAG_PREFIX}{version}",
|
||||
"previous_tag": previous_tag or "",
|
||||
"source_sha": source_sha,
|
||||
}
|
||||
|
||||
|
||||
def stamp_version(source: Path, version: str) -> None:
|
||||
"""只修改 Runner 构建副本,使程序及脱敏包版本与独立 Release 一致。"""
|
||||
_version_parts(version)
|
||||
content, count = SOURCE_VERSION_PATTERN.subn(
|
||||
f'COLLECTOR_VERSION = "{version}"', source.read_text(encoding="utf-8"),
|
||||
)
|
||||
if count != 1:
|
||||
raise ValueError("采集器源码缺少唯一版本声明")
|
||||
source.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""供工作流选择独立发布版本或给构建副本写入版本。"""
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
plan_parser = commands.add_parser("plan")
|
||||
plan_parser.add_argument("--tags-file", type=Path, required=True)
|
||||
plan_parser.add_argument("--output", type=Path, required=True)
|
||||
stamp_parser = commands.add_parser("stamp")
|
||||
stamp_parser.add_argument("--version", required=True)
|
||||
args = parser.parse_args()
|
||||
repository = Path(__file__).resolve().parents[2]
|
||||
if args.command == "stamp":
|
||||
stamp_version(repository / "scripts/site_adapter_collector.py", args.version)
|
||||
return
|
||||
tags = args.tags_file.read_text(encoding="utf-8").splitlines()
|
||||
previous_tag = latest_release_tag(tags)
|
||||
if previous_tag:
|
||||
subprocess.run(
|
||||
["git", "fetch", "--no-tags", "origin", f"refs/tags/{previous_tag}:refs/tags/{previous_tag}"],
|
||||
cwd=repository, check=True,
|
||||
)
|
||||
plan = release_plan(repository, previous_tag)
|
||||
with args.output.open("a", encoding="utf-8") as output:
|
||||
for name, value in plan.items():
|
||||
output.write(f"{name}={value}\n")
|
||||
print(f"采集器版本:{plan['version']};输入变化:{plan['changed']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,41 +0,0 @@
|
||||
# -*- mode: python ; coding: utf-8 -*-
|
||||
"""将站点适配采集器构建为不依赖本地 Python 的单文件程序。"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(SPECPATH).resolve().parent
|
||||
ENTRYPOINT = PROJECT_ROOT / "scripts" / "site_adapter_collector.py"
|
||||
analysis = Analysis(
|
||||
[str(ENTRYPOINT)],
|
||||
pathex=[str(PROJECT_ROOT)],
|
||||
binaries=[],
|
||||
datas=[],
|
||||
hiddenimports=[],
|
||||
hookspath=[],
|
||||
hooksconfig={},
|
||||
runtime_hooks=[],
|
||||
excludes=[],
|
||||
noarchive=False,
|
||||
optimize=0,
|
||||
)
|
||||
python_archive = PYZ(analysis.pure)
|
||||
|
||||
executable = EXE(
|
||||
python_archive,
|
||||
analysis.scripts,
|
||||
analysis.binaries,
|
||||
analysis.datas,
|
||||
[],
|
||||
name="moviepilot-site-collector",
|
||||
debug=False,
|
||||
bootloader_ignore_signals=False,
|
||||
strip=False,
|
||||
upx=False,
|
||||
console=True,
|
||||
disable_windowed_traceback=False,
|
||||
argv_emulation=False,
|
||||
target_arch=None,
|
||||
codesign_identity=None,
|
||||
entitlements_file=None,
|
||||
)
|
||||
@@ -1,4 +0,0 @@
|
||||
beautifulsoup4~=4.15.0
|
||||
PyInstaller>=6.14,<7.0
|
||||
requests>=2.32,<3.0
|
||||
websocket-client~=1.9.0
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -u
|
||||
|
||||
# 从解压目录启动 macOS 单文件采集器,并在结束后保留终端窗口供用户查看结果。
|
||||
main() {
|
||||
local script_dir=""
|
||||
local collector_path=""
|
||||
local status=0
|
||||
|
||||
script_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
collector_path="$script_dir/moviepilot-site-collector-macos"
|
||||
if [[ ! -f "$collector_path" ]]; then
|
||||
echo "未找到 moviepilot-site-collector-macos,请完整解压 ZIP 后再双击启动。" >&2
|
||||
status=1
|
||||
else
|
||||
chmod +x "$collector_path"
|
||||
cd "$script_dir" || status=1
|
||||
if [[ "$status" -eq 0 ]]; then
|
||||
"$collector_path" || status=$?
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
read -r -p "按回车关闭窗口..." _
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -241,7 +241,7 @@ def test_pylint_workflow_runs_for_v3_pull_requests_and_pushes():
|
||||
|
||||
|
||||
def test_upload_artifact_actions_share_node24_major():
|
||||
"""所有工件上传入口必须共用 Node 24 的 v7 主版本,避免 CI 告警与版本分叉。"""
|
||||
"""主仓工件上传统一为 v7;采集器工作流已迁出,不得重新混入主仓发布。"""
|
||||
actions_by_workflow: dict[str, list[str]] = {}
|
||||
for workflow_path in WORKFLOW_ROOT.glob("*.yml"):
|
||||
workflow = _load_workflow(workflow_path.name)
|
||||
@@ -258,7 +258,6 @@ def test_upload_artifact_actions_share_node24_major():
|
||||
"architecture-observe.yml": ["actions/upload-artifact@v7"],
|
||||
"native-dependency-update.yml": ["actions/upload-artifact@v7"],
|
||||
"pylint.yml": ["actions/upload-artifact@v7"],
|
||||
"site-adapter-collector.yml": ["actions/upload-artifact@v7"],
|
||||
"test.yml": [
|
||||
"actions/upload-artifact@v7",
|
||||
"actions/upload-artifact@v7",
|
||||
|
||||
@@ -1,488 +0,0 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts import site_adapter_collector as collector
|
||||
|
||||
SEARCH_HTML = """
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head><script>window.token = "embedded-secret-value";</script></head>
|
||||
<body>
|
||||
<nav id="account">alice@example.com <span>192.168.1.20</span></nav>
|
||||
<form><input type="hidden" name="csrf_token" value="embedded-secret-value"></form>
|
||||
<table id="torrent-table" class="torrents">
|
||||
<thead>
|
||||
<tr><th>标题</th><th>大小</th><th>做种</th><th>发布者</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr id="uploader-alice"
|
||||
class="torrent-row torrent-session-token-cell profile-alice contact-alice@example.com peer-192.168.1.20"
|
||||
data-id="987" data-private-user="alice" unknown="alice@example.com">
|
||||
<td>
|
||||
<a class="torrent-title" title="私密电影标题.2026.1080p" aria-label="私密电影标题"
|
||||
href="/details.php?id=123&passkey=embedded-secret-value">私密电影标题</a>
|
||||
<img data-orig="/covers/私密电影标题.jpg" data-original="/covers/private-title.jpg"
|
||||
data-lazy-src="https://images.example.net/private-title.jpg">
|
||||
</td>
|
||||
<td>18.45 GiB</td>
|
||||
<td>42</td>
|
||||
<td><a class="username" href="/users/alice">alice</a></td>
|
||||
<td><span class="date-added" title="2026-07-12 12:34:56">刚刚</span></td>
|
||||
</tr>
|
||||
<tr class="torrent-listings-global-freeleech">
|
||||
<td><a class="torrent-title" href="/details.php?id=124">另一部私密电影</a></td>
|
||||
<td>8.00 GiB</td><td>12</td><td>bob</td>
|
||||
</tr>
|
||||
<tr class="torrent-row">
|
||||
<td><a class="torrent-title" href="/details.php?id=125">第三部私密电影</a></td>
|
||||
<td>2.00 GiB</td><td>8</td><td>carol</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<footer>unrelated footer</footer>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
"""提供采集器测试所需的最小响应接口。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
url: str,
|
||||
body: bytes = b"",
|
||||
headers: dict = None,
|
||||
):
|
||||
"""初始化状态、地址、响应体和响应头。"""
|
||||
self.status_code = status_code
|
||||
self.url = url
|
||||
self._body = body
|
||||
self.headers = headers or {}
|
||||
self.encoding = "utf-8"
|
||||
self.apparent_encoding = "utf-8"
|
||||
self.closed = False
|
||||
|
||||
def iter_content(self, chunk_size: int):
|
||||
"""按给定块大小返回内存响应体。"""
|
||||
for start in range(0, len(self._body), chunk_size):
|
||||
yield self._body[start:start + chunk_size]
|
||||
|
||||
def close(self) -> None:
|
||||
"""记录响应已关闭。"""
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
"""按顺序返回预设响应,避免测试产生真实外网请求。"""
|
||||
|
||||
def __init__(self, responses: list[_FakeResponse]):
|
||||
"""保存待返回响应和请求记录。"""
|
||||
self.responses = responses
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def get_res(self, **kwargs) -> _FakeResponse:
|
||||
"""记录请求参数并返回下一条响应。"""
|
||||
self.calls.append(kwargs)
|
||||
return self.responses.pop(0)
|
||||
|
||||
|
||||
def test_prepare_capture_request_uses_safe_slug_and_keyword_placeholder():
|
||||
"""搜索请求应生成安全站点标识和严格关键词占位符。"""
|
||||
request = collector._prepare_capture_request(
|
||||
url="https://Tracker.Example.com/torrents.php?search={keyword}&category=1",
|
||||
keyword="Movie 2026",
|
||||
)
|
||||
|
||||
assert request.site_id == "tracker-example-com"
|
||||
assert request.origin == "https://tracker.example.com"
|
||||
assert request.path == "/torrents.php"
|
||||
assert request.params == {"search": "Movie 2026", "category": "1"}
|
||||
assert request.public_params == {"search": "{keyword}", "category": "1"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"ftp://tracker.example.com/torrents.php?search={keyword}",
|
||||
"http://tracker.example.com/torrents.php?search={keyword}",
|
||||
"https://tracker.example.com/passkey/value/torrents.php?search={keyword}",
|
||||
"https://tracker.example.com/torrents.php?token=value&search={keyword}",
|
||||
"https://tracker.example.com/torrents.php?search={keyword}&q={keyword}",
|
||||
],
|
||||
)
|
||||
def test_prepare_capture_request_rejects_unsafe_urls(url: str):
|
||||
"""采集器应拒绝非 HTTPS 地址及疑似内嵌凭据的 URL。"""
|
||||
with pytest.raises(ValueError):
|
||||
collector._prepare_capture_request(url=url, keyword="Movie")
|
||||
|
||||
|
||||
def test_user_details_link_is_not_a_torrent_result():
|
||||
"""用户详情链接不得被误判为种子详情链接。"""
|
||||
soup = collector.BeautifulSoup(
|
||||
'<a href="/userdetails.php?id=9">user</a>',
|
||||
"html.parser",
|
||||
)
|
||||
|
||||
assert collector._is_torrent_link(soup.a) is False
|
||||
|
||||
|
||||
def test_sanitize_search_html_crops_and_redacts_result_structure():
|
||||
"""脱敏结果应仅保留种子列表结构,不包含身份、标题和凭据。"""
|
||||
sanitized_html, row_count, report = collector._sanitize_search_html(
|
||||
html=SEARCH_HTML,
|
||||
origin="https://tracker.example.com",
|
||||
keyword="Movie",
|
||||
)
|
||||
lowered = sanitized_html.lower()
|
||||
|
||||
assert row_count == 3
|
||||
assert "torrent-table" in sanitized_html
|
||||
assert "torrent-row" in sanitized_html
|
||||
assert "torrent-listings-global-freeleech" in sanitized_html
|
||||
assert "torrent-session-token-cell" in sanitized_html
|
||||
assert "unrelated footer" not in sanitized_html
|
||||
assert "alice" not in lowered
|
||||
assert "私密电影标题" not in sanitized_html
|
||||
assert "private-title" not in sanitized_html
|
||||
assert "embedded-secret-value" not in sanitized_html
|
||||
assert "passkey" not in lowered
|
||||
assert "csrf" not in lowered
|
||||
assert "window.token" not in lowered
|
||||
sanitized_soup = collector.BeautifulSoup(sanitized_html, "html.parser")
|
||||
result_row = sanitized_soup.select_one("tr.torrent-row")
|
||||
assert result_row["id"] == "uploader-redacted"
|
||||
assert "profile-redacted" in result_row["class"]
|
||||
assert "redacted" in result_row["class"]
|
||||
assert "peer-redacted" in result_row["class"]
|
||||
assert result_row["data-id"] == "0"
|
||||
assert not result_row.has_attr("data-private-user")
|
||||
assert not result_row.has_attr("unknown")
|
||||
assert sanitized_soup.select_one("a.torrent-title")["href"] == "/details.php?id=1"
|
||||
assert sanitized_soup.select_one("a.username")["href"] == "#redacted-identity"
|
||||
assert sanitized_soup.select_one("a.torrent-title")["title"] == "[REDACTED]"
|
||||
assert sanitized_soup.select_one("span.date-added")["title"] == "2000-01-01 00:00"
|
||||
assert report["redacted"] is True
|
||||
assert report["contains_credentials"] is False
|
||||
assert report["captured_rows"] == 3
|
||||
|
||||
|
||||
def test_nested_nexus_table_counts_only_outer_result_rows():
|
||||
"""NexusPHP 资源名内嵌表格不得重复计数或被当成外层结果裁剪。"""
|
||||
result_rows = "".join(
|
||||
f"""
|
||||
<tr class="outer-result-row">
|
||||
<td>
|
||||
<table class="torrentname">
|
||||
<tbody><tr><td><a href="/details.php?id={index}">资源 {index}</a></td></tr></tbody>
|
||||
</table>
|
||||
</td>
|
||||
<td>{index}</td>
|
||||
</tr>
|
||||
"""
|
||||
for index in range(1, 31)
|
||||
)
|
||||
html = f"""
|
||||
<html><body>
|
||||
<table class="torrents">
|
||||
<thead><tr><th>名称</th><th>做种</th></tr></thead>
|
||||
<tbody>{result_rows}</tbody>
|
||||
</table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
sanitized_html, row_count, report = collector._sanitize_search_html(
|
||||
html=html,
|
||||
origin="https://tracker.example.com",
|
||||
keyword="Movie",
|
||||
)
|
||||
soup = collector.BeautifulSoup(sanitized_html, "html.parser")
|
||||
|
||||
assert row_count == collector.MAX_RESULT_ROWS
|
||||
assert len(soup.select("tr.outer-result-row")) == collector.MAX_RESULT_ROWS
|
||||
assert len(soup.select("table.torrentname")) == collector.MAX_RESULT_ROWS
|
||||
assert report["captured_rows"] == collector.MAX_RESULT_ROWS
|
||||
|
||||
|
||||
def test_fetch_search_page_blocks_cross_origin_redirect(monkeypatch):
|
||||
"""携带 Cookie 的采集请求遇到跨 origin 重定向时必须立即停止。"""
|
||||
response = _FakeResponse(
|
||||
status_code=302,
|
||||
url="https://tracker.example.com/torrents.php?search=Movie",
|
||||
headers={"Location": "https://login.example.net/sign-in"},
|
||||
)
|
||||
client = _FakeClient([response])
|
||||
monkeypatch.setattr(collector, "RequestUtils", lambda **_: client)
|
||||
request = collector._prepare_capture_request(
|
||||
url="https://tracker.example.com/torrents.php?search={keyword}",
|
||||
keyword="Movie",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="跨域重定向"):
|
||||
collector._fetch_search_page(request, "session=very-secret-cookie", "Browser UA")
|
||||
|
||||
assert len(client.calls) == 1
|
||||
assert client.calls[0]["allow_redirects"] is False
|
||||
assert client.calls[0]["verify"] is True
|
||||
assert response.closed is True
|
||||
|
||||
|
||||
def test_read_limited_response_rejects_oversized_content_length():
|
||||
"""响应头声明超过容量上限时不应读取响应体。"""
|
||||
response = _FakeResponse(
|
||||
status_code=200,
|
||||
url="https://tracker.example.com/torrents.php",
|
||||
headers={"Content-Length": str(collector.MAX_RESPONSE_BYTES + 1)},
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="超过 5 MiB"):
|
||||
collector._read_limited_response(response)
|
||||
|
||||
|
||||
def test_collect_site_capture_writes_fixed_protocol(monkeypatch, tmp_path: Path):
|
||||
"""采集结果应使用固定四文件协议,并通过摘要和脱敏声明自校验。"""
|
||||
cookie = "session=very-secret-cookie-value"
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_fetch_search_page",
|
||||
lambda request, cookie, user_agent: SEARCH_HTML,
|
||||
)
|
||||
|
||||
archive_path = collector.collect_site_capture(
|
||||
url="https://tracker.example.com/torrents.php?search={keyword}&category=1",
|
||||
keyword="Movie",
|
||||
cookie=cookie,
|
||||
output_dir=tmp_path,
|
||||
user_agent="Browser UA",
|
||||
site_name="示例站点",
|
||||
)
|
||||
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
assert tuple(archive.namelist()) == collector.ARCHIVE_FILE_NAMES
|
||||
contents = {name: archive.read(name) for name in archive.namelist()}
|
||||
|
||||
manifest = json.loads(contents["manifest.json"])
|
||||
request = json.loads(contents["request.json"])
|
||||
report = json.loads(contents["redaction-report.json"])
|
||||
assert manifest["format_version"] == 1
|
||||
assert manifest["site"] == {
|
||||
"id": "tracker-example-com",
|
||||
"name": "示例站点",
|
||||
"domain": "https://tracker.example.com",
|
||||
"public": False,
|
||||
}
|
||||
assert manifest["capture"]["kind"] == "search"
|
||||
assert manifest["capture"]["row_count"] == 3
|
||||
assert manifest["privacy"] == {
|
||||
"redacted": True,
|
||||
"contains_credentials": False,
|
||||
}
|
||||
assert manifest["files"]["search.html"] == hashlib.sha256(
|
||||
contents["search.html"]
|
||||
).hexdigest()
|
||||
assert request == {
|
||||
"method": "get",
|
||||
"path": "/torrents.php",
|
||||
"params": {"category": "1", "search": "{keyword}"},
|
||||
"origin": "https://tracker.example.com",
|
||||
}
|
||||
assert report["redacted"] is True
|
||||
assert report["contains_credentials"] is False
|
||||
assert b"very-secret-cookie-value" not in b"\n".join(contents.values())
|
||||
assert b"Browser UA" not in b"\n".join(contents.values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsafe_value",
|
||||
[
|
||||
"alice@example.com",
|
||||
"192.168.1.20",
|
||||
"AbCdEfGhIjKlMnOpQrStUvWxYz0123456789",
|
||||
],
|
||||
)
|
||||
def test_verify_payload_rejects_residual_private_values(unsafe_value: str):
|
||||
"""最终序列化检查应阻止残留身份信息和疑似高熵凭据写入。"""
|
||||
payload = {
|
||||
"manifest.json": b'{"files": {}}',
|
||||
"request.json": b"{}",
|
||||
"search.html": unsafe_value.encode("utf-8"),
|
||||
"redaction-report.json": b"{}",
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
collector._verify_payload(payload, "session=safe-cookie-value")
|
||||
|
||||
|
||||
def test_verify_payload_checks_complete_short_cookie():
|
||||
"""即使 Cookie 很短,最终序列化检查也不得忽略其完整原值。"""
|
||||
payload = {
|
||||
"manifest.json": b'{"files": {}}',
|
||||
"request.json": b"{}",
|
||||
"search.html": b"id=1",
|
||||
"redaction-report.json": b"{}",
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="凭据值"):
|
||||
collector._verify_payload(payload, "id=1")
|
||||
|
||||
|
||||
def test_collect_does_not_write_archive_when_final_scan_fails(monkeypatch, tmp_path: Path):
|
||||
"""最终序列化检查失败时不得在输出目录留下 ZIP。"""
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_fetch_search_page",
|
||||
lambda request, cookie, user_agent: SEARCH_HTML,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="身份信息"):
|
||||
collector.collect_site_capture(
|
||||
url="https://tracker.example.com/torrents.php?search={keyword}",
|
||||
keyword="Movie",
|
||||
cookie="session=safe-cookie-value",
|
||||
output_dir=tmp_path,
|
||||
site_name="alice@example.com",
|
||||
)
|
||||
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_infer_search_keyword_from_url_and_visible_input():
|
||||
"""普通浏览器模式应自动识别 URL 中与搜索框一致的关键词。"""
|
||||
capture = collector._BrowserCapture(
|
||||
url="https://tracker.example.com/browse.php?term=Movie%202026&category=1",
|
||||
html=SEARCH_HTML,
|
||||
cookie="session=safe-cookie-value",
|
||||
user_agent="Browser UA",
|
||||
search_inputs=[{"name": "term", "id": "search", "value": "Movie 2026"}],
|
||||
)
|
||||
|
||||
assert collector._infer_search_keyword(capture) == "Movie 2026"
|
||||
|
||||
|
||||
def test_infer_search_keyword_rejects_post_only_page():
|
||||
"""地址栏不包含搜索参数时应友好拒绝,避免生成不可复用的 GET 配置。"""
|
||||
capture = collector._BrowserCapture(
|
||||
url="https://tracker.example.com/browse.php",
|
||||
html=SEARCH_HTML,
|
||||
cookie="session=safe-cookie-value",
|
||||
user_agent="Browser UA",
|
||||
search_inputs=[{"name": "search", "id": "search", "value": "Movie"}],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="地址栏包含关键词"):
|
||||
collector._infer_search_keyword(capture)
|
||||
|
||||
|
||||
def test_read_browser_capture_filters_cookies_to_current_site(monkeypatch):
|
||||
"""浏览器采集只应保留当前站点域名的 Cookie 用于本地泄露检查。"""
|
||||
class _FakeCdpClient:
|
||||
"""模拟只读 CDP 页面与 Cookie 返回。"""
|
||||
|
||||
def __init__(self, websocket_url: str):
|
||||
"""保存测试传入的 WebSocket 地址。"""
|
||||
self.websocket_url = websocket_url
|
||||
|
||||
def __enter__(self):
|
||||
"""返回模拟 CDP 客户端。"""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
"""结束模拟 CDP 上下文。"""
|
||||
|
||||
def evaluate(self, expression: str):
|
||||
"""按表达式返回当前页面模拟数据。"""
|
||||
if "location.href" in expression:
|
||||
return "https://tracker.example.com/browse.php?search=Movie"
|
||||
if "outerHTML" in expression:
|
||||
return SEARCH_HTML
|
||||
if "userAgent" in expression:
|
||||
return "Browser UA"
|
||||
return [{"name": "search", "id": "search", "value": "Movie"}]
|
||||
|
||||
def call(self, method: str):
|
||||
"""返回当前站点与外部站点的模拟 Cookie。"""
|
||||
assert method == "Network.getAllCookies"
|
||||
return {
|
||||
"cookies": [
|
||||
{"domain": ".example.com", "name": "session", "value": "site-secret"},
|
||||
{"domain": ".external.test", "name": "other", "value": "external-secret"},
|
||||
]
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
collector,
|
||||
"_select_search_page_target",
|
||||
lambda port: {"webSocketDebuggerUrl": "ws://127.0.0.1/devtools/page/1"},
|
||||
)
|
||||
monkeypatch.setattr(collector, "_CdpClient", _FakeCdpClient)
|
||||
session = collector._BrowserSession(
|
||||
process=None,
|
||||
profile_guard=None,
|
||||
port=9222,
|
||||
browser_websocket_url="ws://127.0.0.1/devtools/browser/1",
|
||||
)
|
||||
|
||||
capture = collector._read_browser_capture(session)
|
||||
|
||||
assert capture.cookie == "session=site-secret"
|
||||
assert "external-secret" not in capture.cookie
|
||||
assert capture.user_agent == "Browser UA"
|
||||
|
||||
|
||||
def test_browser_capture_builds_archive_without_manual_cookie_input(monkeypatch, tmp_path: Path):
|
||||
"""普通模式应从临时浏览器数据直接生成四文件 ZIP。"""
|
||||
@contextmanager
|
||||
def fake_browser_session(start_url: str):
|
||||
"""提供无需启动真实浏览器的模拟会话。"""
|
||||
assert start_url == "https://tracker.example.com"
|
||||
yield object()
|
||||
|
||||
browser_capture = collector._BrowserCapture(
|
||||
url="https://tracker.example.com/torrents.php?search=Movie&category=1",
|
||||
html=SEARCH_HTML,
|
||||
cookie="session=very-secret-cookie-value",
|
||||
user_agent="Browser UA",
|
||||
search_inputs=[{"name": "search", "id": "search", "value": "Movie"}],
|
||||
)
|
||||
monkeypatch.setattr(collector, "_launch_browser_session", fake_browser_session)
|
||||
monkeypatch.setattr(collector, "_read_browser_capture", lambda session: browser_capture)
|
||||
monkeypatch.setattr("builtins.input", lambda prompt="": "")
|
||||
|
||||
archive_path = collector.collect_site_capture_with_browser(
|
||||
start_url="https://tracker.example.com",
|
||||
output_dir=tmp_path,
|
||||
)
|
||||
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
manifest = json.loads(archive.read("manifest.json"))
|
||||
request = json.loads(archive.read("request.json"))
|
||||
combined = b"\n".join(archive.read(name) for name in archive.namelist())
|
||||
assert manifest["collector_version"] == "1.0.1"
|
||||
assert request["params"]["search"] == "{keyword}"
|
||||
assert b"very-secret-cookie-value" not in combined
|
||||
|
||||
|
||||
def test_help_uses_utf8_when_parent_forces_legacy_code_page() -> None:
|
||||
"""父进程强制 cp1252 时,中文 argparse 帮助仍应以 UTF-8 正常输出。"""
|
||||
environment = os.environ.copy()
|
||||
environment["PYTHONIOENCODING"] = "cp1252"
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(Path(collector.__file__)), "--help"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
env=environment,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr.decode("utf-8", errors="replace")
|
||||
assert "站点适配" in result.stdout.decode("utf-8")
|
||||
@@ -1,193 +0,0 @@
|
||||
"""验证采集器独立版本及发布边界,所有 Git 操作仅使用临时仓库。"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
from scripts.collector import release
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parents[1]
|
||||
|
||||
|
||||
def _git(repository: Path, *arguments: str) -> str:
|
||||
"""在临时仓库运行 Git,隔离真实仓库和用户提交身份。"""
|
||||
return subprocess.check_output(
|
||||
["git", "-c", "user.name=Collector Test", "-c", "user.email=collector@example.invalid",
|
||||
*arguments], cwd=repository, text=True,
|
||||
).strip()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collector_repository(tmp_path: Path) -> Path:
|
||||
"""创建已发布 1.0.1 的临时采集器仓库。"""
|
||||
_git(tmp_path, "init", "-b", "v3")
|
||||
for relative in release.RELEASE_PATHS:
|
||||
path = tmp_path / relative
|
||||
if relative == "scripts/collector":
|
||||
path = path / "release.py"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("initial\n", encoding="utf-8")
|
||||
(tmp_path / "scripts/site_adapter_collector.py").write_text(
|
||||
'COLLECTOR_VERSION = "1.0.1"\n', encoding="utf-8",
|
||||
)
|
||||
_git(tmp_path, "add", ".")
|
||||
_git(tmp_path, "commit", "-m", "initial collector")
|
||||
_git(tmp_path, "tag", "site-adapter-collector-v1.0.1")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_latest_release_ignores_backend_and_orders_versions_numerically():
|
||||
"""后端版本和非法标签不得参与采集器版本排序。"""
|
||||
assert release.latest_release_tag([
|
||||
"v3.1.4", "site-adapter-collector-v1.0.9", "site-adapter-collector-v1.0.10",
|
||||
"site-adapter-collector-v1.0.11-rc1", "site-adapter-collector-v1.01.1",
|
||||
]) == "site-adapter-collector-v1.0.10"
|
||||
assert release.latest_release_tag(["v3.1.4"]) is None
|
||||
|
||||
|
||||
def test_first_release_uses_existing_collector_version(collector_repository: Path):
|
||||
"""首次独立发布沿用采集器版本,不能使用后端版本号。"""
|
||||
plan = release.release_plan(collector_repository, None)
|
||||
assert plan["tag"] == "site-adapter-collector-v1.0.1"
|
||||
assert plan["changed"] == "true"
|
||||
assert plan["source_sha"] == _git(collector_repository, "rev-parse", "HEAD")
|
||||
|
||||
|
||||
def test_backend_and_document_changes_do_not_release(collector_repository: Path):
|
||||
"""后端和文档提交、手动重复执行都不能消耗采集器版本。"""
|
||||
for name in ("version.py", "README.md", "tests/test_collector.py"):
|
||||
path = collector_repository / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("unrelated\n", encoding="utf-8")
|
||||
_git(collector_repository, "add", ".")
|
||||
_git(collector_repository, "commit", "-m", "backend and docs")
|
||||
for _ in range(2):
|
||||
plan = release.release_plan(collector_repository, "site-adapter-collector-v1.0.1")
|
||||
assert plan["changed"] == "false"
|
||||
assert plan["version"] == "1.0.1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("relative", release.RELEASE_PATHS)
|
||||
def test_collector_input_change_increments_patch(collector_repository: Path, relative: str):
|
||||
"""采集器任意发布输入改变都必须生成下一个补丁版本。"""
|
||||
path = collector_repository / relative
|
||||
if relative == "scripts/collector":
|
||||
path = path / "release.py"
|
||||
with path.open("a", encoding="utf-8") as output:
|
||||
output.write("# collector change\n")
|
||||
_git(collector_repository, "add", ".")
|
||||
_git(collector_repository, "commit", "-m", "change collector input")
|
||||
plan = release.release_plan(collector_repository, "site-adapter-collector-v1.0.1")
|
||||
assert plan["changed"] == "true"
|
||||
assert plan["version"] == "1.0.2"
|
||||
|
||||
|
||||
def test_subsequent_release_uses_published_version(collector_repository: Path):
|
||||
"""源码版本无需回写,下次补丁必须以正式 Release 版本为准。"""
|
||||
_git(collector_repository, "tag", "site-adapter-collector-v1.0.9")
|
||||
(collector_repository / "scripts/site_adapter_collector.spec").unlink()
|
||||
_git(collector_repository, "add", ".")
|
||||
_git(collector_repository, "commit", "-m", "remove collector build input")
|
||||
assert release.release_plan(
|
||||
collector_repository, "site-adapter-collector-v1.0.9",
|
||||
)["version"] == "1.0.10"
|
||||
|
||||
|
||||
def test_missing_previous_tag_fails_instead_of_publishing(collector_repository: Path):
|
||||
"""比较失败不能误判为采集器改变并发版。"""
|
||||
with pytest.raises(RuntimeError, match="无法比较"):
|
||||
release.release_plan(collector_repository, "site-adapter-collector-v1.0.8")
|
||||
|
||||
|
||||
def test_stamp_sets_runtime_and_manifest_version(tmp_path: Path):
|
||||
"""构建版本写入现有声明,其他源码和采集包协议版本保持一致。"""
|
||||
source = tmp_path / "collector.py"
|
||||
source.write_text('COLLECTOR_VERSION = "1.0.1"\nFORMAT_VERSION = 1\n', encoding="utf-8")
|
||||
release.stamp_version(source, "1.0.10")
|
||||
assert source.read_text(encoding="utf-8") == 'COLLECTOR_VERSION = "1.0.10"\nFORMAT_VERSION = 1\n'
|
||||
with pytest.raises(ValueError, match="无效采集器版本"):
|
||||
release.stamp_version(source, "1.0.10\ninvalid")
|
||||
|
||||
|
||||
def test_workflow_separates_releases_and_pins_all_build_sources():
|
||||
"""工作流只跟随采集器输入,三平台成功才发布且保留主程序 latest。"""
|
||||
workflow = YAML(typ="safe").load(
|
||||
(PROJECT_ROOT / ".github/workflows/site-adapter-collector.yml").read_text(encoding="utf-8"),
|
||||
)
|
||||
assert "release" not in workflow["on"]
|
||||
assert workflow["on"]["push"]["branches"] == ["v3"]
|
||||
paths = workflow["on"]["push"]["paths"]
|
||||
assert set(paths) == set(release.RELEASE_PATHS) - {"scripts/collector"} | {"scripts/collector/**"}
|
||||
assert workflow["concurrency"]["cancel-in-progress"] is False
|
||||
jobs = workflow["jobs"]
|
||||
assert "needs.prepare.outputs.changed == 'true'" in jobs["build"]["if"]
|
||||
assert "!inputs.publish" in jobs["build"]["if"]
|
||||
assert jobs["publish"]["needs"] == ["prepare", "build"]
|
||||
for job in ("build", "publish"):
|
||||
checkout = jobs[job]["steps"][0]
|
||||
assert checkout["with"]["ref"] == "${{ needs.prepare.outputs.source_sha }}"
|
||||
build_step = next(step for step in jobs["build"]["steps"] if step["name"] == "Build single-file collector")
|
||||
assert build_step["shell"] == "bash"
|
||||
assert "release.py stamp" in build_step["run"]
|
||||
publish = jobs["publish"]["steps"][-1]["run"]
|
||||
assert "--target \"$SOURCE_SHA\" --draft" in publish
|
||||
assert "--draft=false --latest=false" in publish
|
||||
assert "--json isDraft,targetCommitish" in publish
|
||||
assert '"$draft_source" != "$SOURCE_SHA"' in publish
|
||||
assert '"$RELEASE_TAG^{commit}"' not in publish
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("draft_flag", "draft_source", "accepted"),
|
||||
[("true", "source-sha", True), ("true", "other-sha", False), ("false", "source-sha", False)],
|
||||
)
|
||||
def test_draft_retry_checks_source_without_requiring_git_tag(
|
||||
tmp_path: Path, draft_flag: str, draft_source: str, accepted: bool,
|
||||
):
|
||||
"""实际运行发布 Shell,确认无标签草稿可重试且拒绝错误源码或正式版本。"""
|
||||
workflow = YAML(typ="safe").load(
|
||||
(PROJECT_ROOT / ".github/workflows/site-adapter-collector.yml").read_text(encoding="utf-8"),
|
||||
)
|
||||
commands = workflow["jobs"]["publish"]["steps"][-1]["run"]
|
||||
binaries = tmp_path / "bin"
|
||||
binaries.mkdir()
|
||||
gh_stub = binaries / "gh"
|
||||
gh_stub.write_text(
|
||||
'#!/bin/bash\n'
|
||||
'if [[ "$1 $2" == "release view" ]]; then\n'
|
||||
' printf "%s\\t%s\\n" "$DRAFT_FLAG" "$DRAFT_SOURCE"\n'
|
||||
'else\n'
|
||||
' printf "%s\\n" "$*" >> "$GH_LOG"\n'
|
||||
'fi\n', encoding="utf-8",
|
||||
)
|
||||
git_stub = binaries / "git"
|
||||
git_stub.write_text("#!/bin/bash\nexit 79\n", encoding="utf-8")
|
||||
for path in (gh_stub, git_stub):
|
||||
path.chmod(0o755)
|
||||
log = tmp_path / "gh.log"
|
||||
environment = {
|
||||
**os.environ,
|
||||
"PATH": f"{binaries}{os.pathsep}{os.environ['PATH']}",
|
||||
"DRAFT_FLAG": draft_flag,
|
||||
"DRAFT_SOURCE": draft_source,
|
||||
"GH_LOG": str(log),
|
||||
"SOURCE_SHA": "source-sha",
|
||||
"RELEASE_TAG": "site-adapter-collector-v1.0.2",
|
||||
"COLLECTOR_VERSION": "1.0.2",
|
||||
"GITHUB_REPOSITORY": "test/collector",
|
||||
}
|
||||
result = subprocess.run(
|
||||
["bash", "-e", "-o", "pipefail", "-c", commands],
|
||||
cwd=tmp_path, env=environment, capture_output=True, text=True, check=False,
|
||||
)
|
||||
if accepted:
|
||||
assert result.returncode == 0, result.stderr
|
||||
calls = log.read_text(encoding="utf-8")
|
||||
assert "release upload site-adapter-collector-v1.0.2" in calls
|
||||
assert "--draft=false --latest=false" in calls
|
||||
else:
|
||||
assert result.returncode != 0
|
||||
assert not log.exists()
|
||||
Reference in New Issue
Block a user