refactor: move site collector releases and sources to independent repository

This commit is contained in:
jxxghp
2026-10-11 14:12:52 +08:00
parent 01301b887a
commit 28e1cb3a2f
19 changed files with 22 additions and 2790 deletions

View File

@@ -8,7 +8,7 @@ body:
value: |
请说明你希望添加的功能。
站点适配请求请从 [采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 下载最新采集器,再按 [站点适配采集说明](https://github.com/jxxghp/MoviePilot/blob/v3/docs/site-adapter-capture.md) 生成脱敏 ZIP,并在下方附加。Issue 及附件是公开内容,提交前必须解压预览四个文件。不要上传 Cookie、Authorization、通行密钥、会话字段或任何原始数据。
站点适配请求请从 [采集器独立 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest) 下载最新采集器,再按 [站点适配采集说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md) 生成脱敏 ZIP,并在下方附加。Issue 及附件是公开内容,提交前必须解压预览四个文件。不要上传 Cookie、Authorization、通行密钥、会话字段或任何原始数据。
- type: input
id: version
attributes:

View File

@@ -1,257 +0,0 @@
name: Site Adapter Collector
on:
push:
branches:
- v3
paths:
- scripts/site_adapter_collector.py
- scripts/site_adapter_collector.spec
- scripts/site_adapter_collector_requirements.txt
- scripts/start-site-adapter-collector.command
- scripts/collect-site-adapter.sh
- scripts/collector/**
- .github/workflows/site-adapter-collector.yml
workflow_dispatch:
inputs:
publish:
description: Publish an independent release only when collector inputs changed
required: true
default: true
type: boolean
# 串行选择版本和发布,避免多个采集器提交争抢同一个版本。
concurrency:
group: site-adapter-collector-release
cancel-in-progress: false
permissions:
contents: read
jobs:
prepare:
name: Check collector changes and select independent version
if: github.ref == 'refs/heads/v3'
runs-on: ubuntu-latest
outputs:
changed: ${{ steps.version.outputs.changed }}
version: ${{ steps.version.outputs.version }}
tag: ${{ steps.version.outputs.tag }}
previous_tag: ${{ steps.version.outputs.previous_tag }}
source_sha: ${{ steps.version.outputs.source_sha }}
steps:
- name: Checkout current collector source
uses: actions/checkout@v7
with:
# 排队期间可能已有新提交;统一构建当前 v3,避免旧任务回退采集器。
ref: v3
fetch-depth: 0
fetch-tags: false
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.14'
- name: Select collector release version
id: version
env:
GH_TOKEN: ${{ github.token }}
run: |
gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" --paginate \
--jq '.[] | select(.draft == false and .prerelease == false) | .tag_name' \
> collector-release-tags.txt
python scripts/collector/release.py plan \
--tags-file collector-release-tags.txt --output "$GITHUB_OUTPUT"
build:
name: Build ${{ matrix.platform_name }} collector
needs: prepare
if: needs.prepare.outputs.changed == 'true' || (github.event_name == 'workflow_dispatch' && !inputs.publish)
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- platform_name: Windows
platform_id: windows
runner: windows-latest
source_name: moviepilot-site-collector.exe
asset_name: moviepilot-site-collector-windows.exe
artifact_name: site-adapter-collector-windows
- platform_name: macOS
platform_id: macos
runner: macos-latest
source_name: moviepilot-site-collector
asset_name: MoviePilot-Site-Collector-macOS.zip
artifact_name: site-adapter-collector-macos
- platform_name: Linux
platform_id: linux
runner: ubuntu-latest
source_name: moviepilot-site-collector
asset_name: moviepilot-site-collector-linux
artifact_name: site-adapter-collector-linux
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
ref: ${{ needs.prepare.outputs.source_sha }}
- name: Set up Python
uses: actions/setup-python@v7
with:
python-version: '3.14'
- name: Set up uv
uses: astral-sh/setup-uv@v10.0.1
with:
enable-cache: true
cache-dependency-glob: scripts/site_adapter_collector_requirements.txt
- name: Install build dependencies
run: uv pip install --system --requirement scripts/site_adapter_collector_requirements.txt
- name: Build single-file collector
shell: bash
env:
COLLECTOR_VERSION: ${{ needs.prepare.outputs.version }}
run: |
python scripts/collector/release.py stamp --version "$COLLECTOR_VERSION"
pyinstaller --clean --noconfirm scripts/site_adapter_collector.spec
- name: Smoke-test collector
env:
SOURCE_NAME: ${{ matrix.source_name }}
COLLECTOR_VERSION: ${{ needs.prepare.outputs.version }}
run: |
python -c "import os, subprocess; from pathlib import Path; subprocess.run([str((Path('dist') / os.environ['SOURCE_NAME']).resolve()), '--help'], check=True)"
python -c "import os, subprocess; from pathlib import Path; result = subprocess.run([str((Path('dist') / os.environ['SOURCE_NAME']).resolve()), '--version'], check=True, capture_output=True, text=True); assert result.stdout.strip() == os.environ['COLLECTOR_VERSION'], result.stdout"
- name: Package macOS double-click archive
if: matrix.platform_id == 'macos'
shell: bash
env:
ASSET_NAME: ${{ matrix.asset_name }}
SOURCE_NAME: ${{ matrix.source_name }}
run: |
package_dir="dist/MoviePilot-Collector"
mkdir -p "$package_dir"
cp "dist/$SOURCE_NAME" "$package_dir/moviepilot-site-collector-macos"
cp scripts/start-site-adapter-collector.command "$package_dir/start-site-adapter-collector.command"
chmod +x "$package_dir/moviepilot-site-collector-macos"
chmod +x "$package_dir/start-site-adapter-collector.command"
cd dist
COPYFILE_DISABLE=1 zip -q -r -X "$ASSET_NAME" MoviePilot-Collector
- name: Rename Windows and Linux collector
if: matrix.platform_id != 'macos'
env:
ASSET_NAME: ${{ matrix.asset_name }}
SOURCE_NAME: ${{ matrix.source_name }}
run: |
python -c "import os; from pathlib import Path; (Path('dist') / os.environ['SOURCE_NAME']).replace(Path('dist') / os.environ['ASSET_NAME'])"
- name: Generate SHA-256 checksum
env:
ASSET_NAME: ${{ matrix.asset_name }}
run: |
python -c "import hashlib, os; from pathlib import Path; path = Path('dist') / os.environ['ASSET_NAME']; path.with_name(path.name + '.sha256').write_text(f'{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n', encoding='utf-8', newline='\n')"
- name: Upload collector artifact
uses: actions/upload-artifact@v7
with:
name: ${{ matrix.artifact_name }}
path: |
dist/${{ matrix.asset_name }}
dist/${{ matrix.asset_name }}.sha256
if-no-files-found: error
retention-days: 3
publish:
name: Publish independent collector release
if: needs.prepare.outputs.changed == 'true' && (github.event_name == 'push' || inputs.publish)
needs:
- prepare
- build
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout release source
uses: actions/checkout@v7
with:
ref: ${{ needs.prepare.outputs.source_sha }}
fetch-depth: 0
fetch-tags: false
- name: Download collector artifacts
uses: actions/download-artifact@v8
with:
pattern: site-adapter-collector-*
path: release-assets
merge-multiple: true
- name: Generate collector release notes
env:
COLLECTOR_VERSION: ${{ needs.prepare.outputs.version }}
PREVIOUS_TAG: ${{ needs.prepare.outputs.previous_tag }}
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
run: |
if [[ -n "$PREVIOUS_TAG" ]]; then
git fetch --no-tags origin "refs/tags/$PREVIOUS_TAG:refs/tags/$PREVIOUS_TAG"
fi
python - <<'PY'
import os
import subprocess
from pathlib import Path
from scripts.collector.release import RELEASE_PATHS
version = os.environ['COLLECTOR_VERSION']
previous = os.environ['PREVIOUS_TAG']
revision = f"{previous}..HEAD" if previous else "HEAD"
records = subprocess.check_output(
['git', 'log', revision, '--no-merges', '--format=- %s (%h) — %an', '--', *RELEASE_PATHS],
text=True,
).strip()
notes = (
f"MoviePilot 站点适配采集器 {version},独立于后端版本发布。\n\n"
"下载与系统对应的程序及同名 `.sha256` 校验文件。\n\n"
"[使用说明](https://github.com/jxxghp/MoviePilot/blob/v3/docs/site-adapter-collector-release.md)\n\n"
f"源码提交:`{os.environ['SOURCE_SHA']}`\n\n## 变更记录\n\n{records}\n"
)
Path('collector-release-notes.md').write_text(notes, encoding='utf-8')
PY
- name: Create and publish independent collector release
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ needs.prepare.outputs.tag }}
COLLECTOR_VERSION: ${{ needs.prepare.outputs.version }}
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
run: |
if gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
--json isDraft,targetCommitish --jq '[.isDraft, .targetCommitish] | @tsv' \
> collector-draft-state.txt; then
# 草稿尚未发布时标签可能不存在,应核对创建草稿时绑定的源码提交。
IFS=$'\t' read -r is_draft draft_source < collector-draft-state.txt
if [[ "$is_draft" != "true" || "$draft_source" != "$SOURCE_SHA" ]]; then
echo "Refusing to overwrite a published release or a draft from another source commit" >&2
exit 1
fi
gh release upload "$RELEASE_TAG" release-assets/* \
--clobber --repo "$GITHUB_REPOSITORY"
gh release edit "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
--title "Site Adapter Collector v$COLLECTOR_VERSION" \
--notes-file collector-release-notes.md
else
gh release create "$RELEASE_TAG" release-assets/* \
--repo "$GITHUB_REPOSITORY" --target "$SOURCE_SHA" --draft \
--title "Site Adapter Collector v$COLLECTOR_VERSION" \
--notes-file collector-release-notes.md --latest=false
fi
# 全部附件上传完成才转正式发布,上传失败不会消耗正式版本。
gh release edit "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
--draft=false --latest=false

View File

@@ -61,8 +61,8 @@ curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v3/scripts/bootst
- 文档规则入口:[docs/rules/README.md](docs/rules/README.md)
- 开发环境与本地源码运行:[docs/development-setup.md](docs/development-setup.md)
- 测试说明:[docs/testing.md](docs/testing.md)
- 新站点适配采集与 Feature Request 提交:[docs/site-adapter-capture.md](docs/site-adapter-capture.md)
- 站点采集器下载:[独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true),仅在采集器变化时递增版本并发布。
- 新站点适配采集与 Feature Request 提交:[MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md)
- 站点采集器下载:[独立 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest),仅在采集器变化时递增版本并发布。
- REST API 文档:https://api.movie-pilot.org
- 插件开发说明:https://wiki.movie-pilot.org/zh/plugindev

View File

@@ -59,7 +59,8 @@ Before contributing, read the repository rules and local environment guide, keep
- Rule index: [docs/rules/README.md](docs/rules/README.md)
- Development setup and local source run: [docs/development-setup.md](docs/development-setup.md)
- Testing guide: [docs/testing.md](docs/testing.md)
- New site adapter capture and Feature Request submission: [docs/site-adapter-capture.md](docs/site-adapter-capture.md)
- New site adapter capture and Feature Request submission: [MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md)
- Site collector downloads: [independent repository Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest), published only when collector inputs change.
- REST API documentation: https://api.movie-pilot.org
- Plugin development guide: https://wiki.movie-pilot.org/zh/plugindev

View File

@@ -234,7 +234,7 @@ Docker 镜像直接基于官方基础镜像构建,只安装应用运行所需
报告,不自动更新基线。普通主仓改动不要求额外检出插件仓或运行观察任务。宿主架构门禁和 changed-file
Pylint 由 `v3` PR/push 的 GitHub Actions 执行,`app/` 全量 Pylint 是建议性报告。
站点采集器使用独立的 `site-adapter-collector-v1.0.x` Release,仅在采集器发布输入变化时递增版本并构建,不跟随后端 Release。下载入口为 [采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true),自动版本和手动测试构建规则见 [采集器发布说明](site-adapter-collector-release.md#维护者发布流程)。
站点采集器源码与构建工作流位于 [MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector),使用独立的 `site-adapter-collector-v1.0.x` Release,仅在采集器发布输入变化时递增版本并构建,不跟随后端 Release。下载入口为 [采集器独立 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest),自动版本和手动测试构建规则见 [采集器发布说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-collector-release.md#维护者发布流程)。
Ruff/Mypy 基线只允许收紧,不接受新增诊断或类型错误增长;受治零错误文件由 `mypy.ini` 的
`files=` 维护。Mypy 完整 ratchet 固定按 Linux/Python 3.14 分析。Coverage job 在 `v3` PR/push

View File

@@ -286,14 +286,15 @@ moviepilot help scheduler
## Site Adapter Capture — macOS / Linux
```bash
# Run from a MoviePilot source checkout and reuse its virtual environment
# Run from the independent MoviePilot-Site-Collector checkout and reuse its virtual environment
cd /path/to/MoviePilot-Site-Collector
bash scripts/collect-site-adapter.sh
```
**Rules:**
- The default collector asks only for the site HTTPS address, opens an isolated local Chrome/Edge profile, and reads the completed search page after the user confirms.
- Users must not be asked to inspect HTML or copy Cookie/User-Agent values in the default flow. `--manual-cookie` is an advanced fallback only.
- Run only the collector shipped with a trusted local MoviePilot source checkout or installation package. Do not pipe a remote branch script into a shell.
- Run only the collector from a trusted local [MoviePilot-Site-Collector checkout](https://github.com/jxxghp/MoviePilot-Site-Collector) or its [official Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest). Do not pipe a remote branch script into a shell.
- Never put a Cookie or other credential in command arguments or shell history.
- Feature Request attachments are public. Review all four files in the generated ZIP before attaching it, and never attach raw HTML, HAR, or browser network archives.

View File

@@ -1,60 +1,11 @@
# 站点适配采集
当开发者没有目标站点账号时,可以由已有账号的用户在本地采集一份经过裁剪和脱敏的搜索页结构,并把采集 ZIP 附加到站点适配 Feature Request。开发者和自动化流程只处理脱敏包,不需要获取用户账号。
站点采集器已迁入独立仓库 [MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector)。普通用户无需安装 Python、Git 或 MoviePilot 后端:
## 普通用户一键采集
1. 从 [采集器最新 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest) 下载对应系统的程序。
2. 按 [采集说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md) 在临时浏览器中登录、搜索并生成脱敏 ZIP。
3. 在本仓库创建“功能改进” Issue,选择“站点适配”,附加生成的 `moviepilot-site-capture-*.zip`。
普通用户只需准备两样东西:目标站点账号,以及已安装的 Chrome、Edge 或 Chromium。无需安装 Python、Git、MoviePilot、Docker,也不需要查看 HTML、复制 Cookie 或填写 User-Agent。
Issue 及附件是公开内容。提交前请解压预览 `manifest.json`、`request.json`、`search.html`、`redaction-report.json`,确认没有账号或隐私信息。不要上传 Cookie、Authorization、通行密钥、原始 HTML、HAR 或浏览器网络归档。
1. 从官方 [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 选择最新采集器版本,下载与 Windows、macOS 或 Linux 对应的程序;各系统文件名见下方下载说明。
2. 运行采集器,只输入站点首页地址,例如 `https://tracker.example.com`。
3. 程序会打开一个临时浏览器窗口。在这个窗口里正常登录站点,搜索一个能返回至少 3 条结果的常见关键词,并保持搜索结果页打开。
4. 回到采集器按回车。程序会自动识别搜索地址、关键词、Cookie 和 User-Agent,完成本地裁剪与脱敏后生成 ZIP。
临时浏览器使用独立的一次性用户目录,不会读取日常浏览器的历史登录状态。采集完成后程序会关闭临时浏览器并清理这次登录数据。原始页面和 Cookie 只在内存中处理,不会写入采集包。
各系统下载文件和首次运行方式见 [站点适配采集器下载说明](site-adapter-collector-release.md)。
## 开发者源码入口
已经有 MoviePilot 源码和 Python 环境的开发者,也可以在项目目录执行:
```bash
bash scripts/collect-site-adapter.sh
```
源码入口和独立程序使用同一套浏览器采集流程。只有排查兼容问题时才使用 `--manual-cookie` 高级模式;普通用户不需要接触 Cookie。
## 第一版限制
当前采集器会读取浏览器渲染后的页面,因此可由用户在临时窗口中完成验证码、Cloudflare 检查和普通登录。但自动适配协议仍要求搜索结果地址能够表示为 HTTPS GET URL。以下场景第一版不做自动适配:
- 必须 POST 表单才能搜索的站点。
- 搜索完成后地址栏完全没有关键词或可复用搜索参数的站点。
- 需要专用 API、复杂签名或无法从一次搜索结果页观察出 FREE/HR 规则的站点。
遇到这些场景时,请在 Feature Request 中说明失败步骤和终端错误文字,等待人工确认采集方案。不要用原始 HAR、原始 HTML 或包含账号信息的截图替代脱敏 ZIP。
## 脱敏范围
采集器只接受 HTTPS 地址。默认模式从本机临时浏览器只读当前搜索结果页;高级手动模式设置 30 秒超时和 5 MiB 响应上限,并禁止携带 Cookie 跨 origin 重定向。读取页面后会在本机完成以下处理:
- 只保留种子列表、表头和最多 25 条结果相关 DOM,丢弃账号导航、页脚、脚本、样式、隐藏表单和其他页面内容。
- 替换种子标题、用户名、邮箱、IP、时间、大小、统计值和长随机标识,只保留适配所需的标签、class、字段与链接结构。
- URL 转为同源相对路径,查询值统一替换为占位符,凭据语义的字段直接移除。
- Cookie 和浏览器 UA 仅用于本次请求,不写入采集包;写入前还会使用 Cookie 原值执行二次泄露检查。
输出 ZIP 根目录固定包含:
- `manifest.json`:包版本、站点标识、采集时间、结果行数、HTML 摘要和隐私声明。
- `request.json`:仅包含 GET、origin、相对路径和脱敏后的查询参数,搜索值固定为 `{keyword}`。
- `search.html`:本地裁剪并脱敏后的种子列表结构。
- `redaction-report.json`:脱敏状态和各类处理计数。
## 提交 Feature Request
在 GitHub 创建“功能改进” Issue,类型选择“站点适配”,然后把生成的 `moviepilot-site-capture-*.zip` 直接拖入“站点适配采集文件”输入框。
Feature Request 及其附件是公开内容。提交前请先在本地解压 ZIP,确认根目录只有上述四个文件,并逐一预览确认没有站点账号、搜索隐私或其他不希望公开的信息。
站点适配请求必须附加采集器生成并人工复核过的 ZIP。严禁手工上传 Cookie、Authorization、通行密钥、会话字段、原始 HTML、原始 HAR、浏览器网络归档或截图中的账号信息。如果采集失败,请只提交终端错误文字,不要用任何原始数据代替脱敏包。
源码运行、采集限制与脱敏协议均见 [独立仓库完整说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md)。本页面保留以兼容旧文档链接。

View File

@@ -1,42 +1,12 @@
# 站点适配采集器下载说明
普通用户优先使用 [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 提供的单文件采集器,选择列表中最新版本。采集器使用 `site-adapter-collector-v1.0.x` 独立标签,与 MoviePilot 后端版本分开发布。单文件已经包含 Python 和采集器依赖,不需要安装 Python、pip、Git、MoviePilot 后端,也不需要下载源码。电脑只需已安装 Chrome、Edge 或 Chromium 浏览器。
站点采集器的源码、版本、构建工作流和历史 Release 已迁入独立仓库 [MoviePilot-Site-Collector](https://github.com/jxxghp/MoviePilot-Site-Collector),与 MoviePilot 主程序发布完全分开。
## 选择下载文件
请从 [采集器最新 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest) 下载 Windows、macOS 或 Linux 程序及同名 `.sha256` 文件。
请只从官方 [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 下载与系统匹配的文件;MoviePilot 主程序 Release 不再提供采集器附件:
| 系统 | 下载文件 | 用户侧运行环境 |
|---|---|---|
| Windows | `moviepilot-site-collector-windows.exe` | Chrome、Edge 或 Chromium |
| macOS | `MoviePilot-Site-Collector-macOS.zip` | Chrome、Edge 或 Chromium |
| Linux | `moviepilot-site-collector-linux` | Chrome、Edge 或 Chromium |
每个程序旁边还有同名的 `.sha256` 文件,可用于核对下载文件是否完整。程序启动时显示版本,也可执行 `--version` 查看。GitHub Actions 的手动构建产物主要用于维护者测试;普通用户应使用采集器独立 Release 资产。
## 运行采集器
Windows 用户下载后双击 `.exe`,按窗口提示操作即可。macOS 用户解压 ZIP 后双击 `start-site-adapter-collector.command`,不要打开构建目录中的 `.pkg` 文件。Linux 用户在下载目录打开终端,只需首次赋予执行权限后运行:
```bash
chmod +x moviepilot-site-collector-linux
./moviepilot-site-collector-linux
```
运行后只需输入站点首页地址,随后在弹出的临时浏览器中登录并搜索,最后回到采集器按回车。采集器会在当前目录生成 `moviepilot-site-capture-*.zip`,用户只需把这个 ZIP 附加到站点适配 Feature Request,不需要提交任何源码、Cookie 或 HTML。
## 系统安全提示
当前自动构建产物尚未接入 Windows 或 Apple 代码签名。Windows SmartScreen 或 macOS Gatekeeper 可能因此显示安全提示。仅在文件来自官方 [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true),且校验摘要一致时运行;不要从聊天、网盘或第三方站点接收采集器。
如果系统阻止运行,可改用随 MoviePilot 源码提供的本地采集脚本;该方式需要 Python 3.14+ 及完整后端依赖,不适合作为普通用户的首选路径。
- [完整下载与运行说明](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-collector-release.md)
- [采集流程和源码入口](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-capture.md)
## 维护者发布流程
`.github/workflows/site-adapter-collector.yml` 在 `v3` 分支的采集器源码、依赖、启动脚本或打包工作流变化时触发,支持手动运行。后端 Release 发布及普通后端代码、文档、测试改动不会触发采集器构建。
构建前会与上一个已正式发布的采集器标签比较输入文件。没有变化就跳过构建和发版;首次发布沿用源码中的 `COLLECTOR_VERSION`(当前为 `1.0.1`),后续有变化时自动递增补丁版本。工作流只在 Runner 构建副本中写入版本,使程序、`--version` 和脱敏包中的 `collector_version` 一致,不向 `v3` 分支回写版本提交。采集包的 `format_version` 协议版本保持独立。
每个平台先执行 `--help` 和 `--version` 启动检查,再上传程序及 SHA-256 摘要。三个平台全部成功后,工作流创建独立版本 Release,自动生成仅包含采集器相关提交的变更记录,并设置 `latest=false`,使主程序的最新 Release 入口继续指向后端版本。
手动触发时 `publish` 默认为 `true`,仍遵守“输入变化才发版”。设为 `false` 时可重复构建测试,只生成保留 3 天的 Artifact,不创建 Release、不消耗版本号。历史主程序 Release 中的采集器附件会迁移移除,下载请始终使用上面的独立 Release 入口。
采集器仅在自身发布输入变化时自动递增版本并发布,不跟随 MoviePilot 发版。维护者请使用 [独立仓库发布流程](https://github.com/jxxghp/MoviePilot-Site-Collector/blob/main/docs/site-adapter-collector-release.md#维护者发布流程)。本页面保留以兼容旧文档链接。

View File

@@ -399,7 +399,7 @@ MoviePilot/app/application/site
- 后端用户数据解析器可以自动探测 Gazelle、NexusPHP 等站点变种。
- 解析模板支持更完整的标题、文件名、标签和可选字段回退。
- 构建产物覆盖 Linux、Windows、macOS、x86_64、Arm64,以及 V3t 所需的 Python 3.14t ABI。
- [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true) 提供生成脱敏样本的程序,维护者可通过自动化流程匹配模板并创建待审核的适配 PR。
- [站点采集器独立 Release](https://github.com/jxxghp/MoviePilot-Site-Collector/releases/latest) 提供生成脱敏样本的程序,维护者可通过自动化流程匹配模板并创建待审核的适配 PR。
### 7.4 用户遇到站点问题时应如何判断

View File

@@ -1,99 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ORIGINAL_DIR="$PWD"
SCRIPT_DIR=""
PROJECT_ROOT=""
PYTHON_BIN=""
COLLECTOR_PATH=""
# 判断候选 Python 是否满足 3.14 最低版本。
python_version_ok() {
"$1" - <<'PY' >/dev/null 2>&1
import sys
raise SystemExit(0 if sys.version_info >= (3, 14) else 1)
PY
}
# 从当前源码目录、环境变量或脚本位置查找 MoviePilot 根目录。
find_project_root() {
local candidate=""
local source_path="${BASH_SOURCE[0]:-}"
if [[ -n "$source_path" && -f "$source_path" ]]; then
SCRIPT_DIR="$(cd "$(dirname "$source_path")" && pwd)"
fi
for candidate in "${MOVIEPILOT_ROOT:-}" "$ORIGINAL_DIR" "${SCRIPT_DIR:+$SCRIPT_DIR/..}"; do
if [[ -n "$candidate" && -f "$candidate/scripts/site_adapter_collector.py" ]]; then
PROJECT_ROOT="$(cd "$candidate" && pwd)"
return 0
fi
done
return 1
}
# 查找可用的项目虚拟环境或系统 Python。
find_python() {
local candidate=""
local resolved=""
for candidate in \
"${PROJECT_ROOT:+$PROJECT_ROOT/venv/bin/python}" \
"${PROJECT_ROOT:+$PROJECT_ROOT/.venv/bin/python}" \
"${VIRTUAL_ENV:+$VIRTUAL_ENV/bin/python}" \
python3.14 python3; do
[[ -n "$candidate" ]] || continue
if [[ "$candidate" == */* ]]; then
resolved="$candidate"
else
resolved="$(command -v "$candidate" 2>/dev/null || true)"
fi
if [[ -n "$resolved" && -x "$resolved" ]] && python_version_ok "$resolved"; then
PYTHON_BIN="$resolved"
return 0
fi
done
return 1
}
# 判断 Python 是否已具备独立采集器所需的最小运行依赖。
project_runtime_ready() {
PYTHONPATH="$PROJECT_ROOT${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" - <<'PY' >/dev/null 2>&1
import requests
import websocket
from bs4 import BeautifulSoup
PY
}
# 让脚本始终从终端安全读取交互输入。
restore_terminal_input() {
if [[ -r /dev/tty ]]; then
exec </dev/tty
else
echo "需要可交互终端读取站点地址并等待浏览器采集确认。" >&2
exit 1
fi
}
# 定位本地项目运行环境后启动随发行包提供的采集器。
main() {
if ! find_project_root; then
echo "未找到本地 MoviePilot 源码或安装目录,请在 MoviePilot 目录中运行此脚本。" >&2
exit 1
fi
if ! find_python; then
echo "未找到 Python 3.11 或更高版本,请先安装 Python。" >&2
exit 1
fi
if ! project_runtime_ready; then
echo "本地 Python 环境缺少采集器依赖,请下载采集器独立 Release:https://github.com/jxxghp/MoviePilot/releases?q=tag%3Asite-adapter-collector-v1&expanded=true" >&2
exit 1
fi
COLLECTOR_PATH="$PROJECT_ROOT/scripts/site_adapter_collector.py"
restore_terminal_input
cd "$ORIGINAL_DIR"
PYTHONPATH="$PROJECT_ROOT${PYTHONPATH:+:$PYTHONPATH}" "$PYTHON_BIN" "$COLLECTOR_PATH" "$@"
}
main "$@"

View File

@@ -1 +0,0 @@
"""站点采集器的独立发布工具。"""

View File

@@ -1,116 +0,0 @@
"""比较采集器发布输入,按独立 Release 递增版本并写入构建副本。"""
import argparse
import re
import subprocess
from pathlib import Path
from typing import Optional
TAG_PREFIX = "site-adapter-collector-v"
RELEASE_PATHS = (
"scripts/site_adapter_collector.py",
"scripts/site_adapter_collector.spec",
"scripts/site_adapter_collector_requirements.txt",
"scripts/start-site-adapter-collector.command",
"scripts/collect-site-adapter.sh",
"scripts/collector",
".github/workflows/site-adapter-collector.yml",
)
VERSION_PATTERN = re.compile(r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)")
SOURCE_VERSION_PATTERN = re.compile(r'^COLLECTOR_VERSION = "([\d.]+)"$', re.MULTILINE)
def _version_parts(version: str) -> tuple[int, int, int]:
"""校验独立版本,避免外部标签进入 Git 参数或构建源码。"""
match = VERSION_PATTERN.fullmatch(version)
if not match:
raise ValueError(f"无效采集器版本:{version}")
return tuple(int(part) for part in match.groups())
def latest_release_tag(tags: list[str]) -> Optional[str]:
"""从已正式发布的标签中选取采集器最高版本,忽略后端版本。"""
collector_tags = [
tag for tag in tags
if tag.startswith(TAG_PREFIX) and VERSION_PATTERN.fullmatch(tag[len(TAG_PREFIX):])
]
return max(collector_tags, key=lambda tag: _version_parts(tag[len(TAG_PREFIX):]), default=None)
def release_plan(repository: Path, previous_tag: Optional[str]) -> dict[str, str]:
"""仅比较采集器输入;未变化时跳过发布,首次沿用源码版本。"""
source = repository / "scripts/site_adapter_collector.py"
source_match = SOURCE_VERSION_PATTERN.search(source.read_text(encoding="utf-8"))
if not source_match:
raise ValueError("采集器源码缺少唯一版本声明")
source_version = source_match.group(1)
source_parts = _version_parts(source_version)
source_sha = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=repository, text=True,
).strip()
changed = True
version = source_version
if previous_tag:
previous_parts = _version_parts(previous_tag.removeprefix(TAG_PREFIX))
result = subprocess.run(
["git", "diff", "--quiet", previous_tag, "HEAD", "--", *RELEASE_PATHS],
cwd=repository, check=False,
)
if result.returncode not in (0, 1):
raise RuntimeError(f"无法比较采集器 Release:{previous_tag}")
changed = result.returncode == 1
if changed:
next_parts = max(source_parts, (*previous_parts[:2], previous_parts[2] + 1))
version = ".".join(str(part) for part in next_parts)
else:
version = previous_tag[len(TAG_PREFIX):]
return {
"changed": str(changed).lower(),
"version": version,
"tag": f"{TAG_PREFIX}{version}",
"previous_tag": previous_tag or "",
"source_sha": source_sha,
}
def stamp_version(source: Path, version: str) -> None:
"""只修改 Runner 构建副本,使程序及脱敏包版本与独立 Release 一致。"""
_version_parts(version)
content, count = SOURCE_VERSION_PATTERN.subn(
f'COLLECTOR_VERSION = "{version}"', source.read_text(encoding="utf-8"),
)
if count != 1:
raise ValueError("采集器源码缺少唯一版本声明")
source.write_text(content, encoding="utf-8")
def main() -> None:
"""供工作流选择独立发布版本或给构建副本写入版本。"""
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest="command", required=True)
plan_parser = commands.add_parser("plan")
plan_parser.add_argument("--tags-file", type=Path, required=True)
plan_parser.add_argument("--output", type=Path, required=True)
stamp_parser = commands.add_parser("stamp")
stamp_parser.add_argument("--version", required=True)
args = parser.parse_args()
repository = Path(__file__).resolve().parents[2]
if args.command == "stamp":
stamp_version(repository / "scripts/site_adapter_collector.py", args.version)
return
tags = args.tags_file.read_text(encoding="utf-8").splitlines()
previous_tag = latest_release_tag(tags)
if previous_tag:
subprocess.run(
["git", "fetch", "--no-tags", "origin", f"refs/tags/{previous_tag}:refs/tags/{previous_tag}"],
cwd=repository, check=True,
)
plan = release_plan(repository, previous_tag)
with args.output.open("a", encoding="utf-8") as output:
for name, value in plan.items():
output.write(f"{name}={value}\n")
print(f"采集器版本:{plan['version']};输入变化:{plan['changed']}")
if __name__ == "__main__":
main()

File diff suppressed because it is too large Load Diff

View File

@@ -1,41 +0,0 @@
# -*- mode: python ; coding: utf-8 -*-
"""将站点适配采集器构建为不依赖本地 Python 的单文件程序。"""
from pathlib import Path
PROJECT_ROOT = Path(SPECPATH).resolve().parent
ENTRYPOINT = PROJECT_ROOT / "scripts" / "site_adapter_collector.py"
analysis = Analysis(
[str(ENTRYPOINT)],
pathex=[str(PROJECT_ROOT)],
binaries=[],
datas=[],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=0,
)
python_archive = PYZ(analysis.pure)
executable = EXE(
python_archive,
analysis.scripts,
analysis.binaries,
analysis.datas,
[],
name="moviepilot-site-collector",
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=False,
console=True,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)

View File

@@ -1,4 +0,0 @@
beautifulsoup4~=4.15.0
PyInstaller>=6.14,<7.0
requests>=2.32,<3.0
websocket-client~=1.9.0

View File

@@ -1,29 +0,0 @@
#!/usr/bin/env bash
set -u
# 从解压目录启动 macOS 单文件采集器,并在结束后保留终端窗口供用户查看结果。
main() {
local script_dir=""
local collector_path=""
local status=0
script_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
collector_path="$script_dir/moviepilot-site-collector-macos"
if [[ ! -f "$collector_path" ]]; then
echo "未找到 moviepilot-site-collector-macos,请完整解压 ZIP 后再双击启动。" >&2
status=1
else
chmod +x "$collector_path"
cd "$script_dir" || status=1
if [[ "$status" -eq 0 ]]; then
"$collector_path" || status=$?
fi
fi
echo
read -r -p "按回车关闭窗口..." _
exit "$status"
}
main "$@"

View File

@@ -241,7 +241,7 @@ def test_pylint_workflow_runs_for_v3_pull_requests_and_pushes():
def test_upload_artifact_actions_share_node24_major():
"""所有工件上传入口必须共用 Node 24 的 v7 主版本,避免 CI 告警与版本分叉。"""
"""主仓工件上传统一为 v7;采集器工作流已迁出,不得重新混入主仓发布。"""
actions_by_workflow: dict[str, list[str]] = {}
for workflow_path in WORKFLOW_ROOT.glob("*.yml"):
workflow = _load_workflow(workflow_path.name)
@@ -258,7 +258,6 @@ def test_upload_artifact_actions_share_node24_major():
"architecture-observe.yml": ["actions/upload-artifact@v7"],
"native-dependency-update.yml": ["actions/upload-artifact@v7"],
"pylint.yml": ["actions/upload-artifact@v7"],
"site-adapter-collector.yml": ["actions/upload-artifact@v7"],
"test.yml": [
"actions/upload-artifact@v7",
"actions/upload-artifact@v7",

View File

@@ -1,488 +0,0 @@
import hashlib
import json
import os
import subprocess
import sys
import zipfile
from contextlib import contextmanager
from pathlib import Path
import pytest
from scripts import site_adapter_collector as collector
SEARCH_HTML = """
<!doctype html>
<html>
<head><script>window.token = "embedded-secret-value";</script></head>
<body>
<nav id="account">alice@example.com <span>192.168.1.20</span></nav>
<form><input type="hidden" name="csrf_token" value="embedded-secret-value"></form>
<table id="torrent-table" class="torrents">
<thead>
<tr><th>标题</th><th>大小</th><th>做种</th><th>发布者</th></tr>
</thead>
<tbody>
<tr id="uploader-alice"
class="torrent-row torrent-session-token-cell profile-alice contact-alice@example.com peer-192.168.1.20"
data-id="987" data-private-user="alice" unknown="alice@example.com">
<td>
<a class="torrent-title" title="私密电影标题.2026.1080p" aria-label="私密电影标题"
href="/details.php?id=123&passkey=embedded-secret-value">私密电影标题</a>
<img data-orig="/covers/私密电影标题.jpg" data-original="/covers/private-title.jpg"
data-lazy-src="https://images.example.net/private-title.jpg">
</td>
<td>18.45 GiB</td>
<td>42</td>
<td><a class="username" href="/users/alice">alice</a></td>
<td><span class="date-added" title="2026-07-12 12:34:56">刚刚</span></td>
</tr>
<tr class="torrent-listings-global-freeleech">
<td><a class="torrent-title" href="/details.php?id=124">另一部私密电影</a></td>
<td>8.00 GiB</td><td>12</td><td>bob</td>
</tr>
<tr class="torrent-row">
<td><a class="torrent-title" href="/details.php?id=125">第三部私密电影</a></td>
<td>2.00 GiB</td><td>8</td><td>carol</td>
</tr>
</tbody>
</table>
<footer>unrelated footer</footer>
</body>
</html>
"""
class _FakeResponse:
"""提供采集器测试所需的最小响应接口。"""
def __init__(
self,
status_code: int,
url: str,
body: bytes = b"",
headers: dict = None,
):
"""初始化状态、地址、响应体和响应头。"""
self.status_code = status_code
self.url = url
self._body = body
self.headers = headers or {}
self.encoding = "utf-8"
self.apparent_encoding = "utf-8"
self.closed = False
def iter_content(self, chunk_size: int):
"""按给定块大小返回内存响应体。"""
for start in range(0, len(self._body), chunk_size):
yield self._body[start:start + chunk_size]
def close(self) -> None:
"""记录响应已关闭。"""
self.closed = True
class _FakeClient:
"""按顺序返回预设响应,避免测试产生真实外网请求。"""
def __init__(self, responses: list[_FakeResponse]):
"""保存待返回响应和请求记录。"""
self.responses = responses
self.calls: list[dict] = []
def get_res(self, **kwargs) -> _FakeResponse:
"""记录请求参数并返回下一条响应。"""
self.calls.append(kwargs)
return self.responses.pop(0)
def test_prepare_capture_request_uses_safe_slug_and_keyword_placeholder():
"""搜索请求应生成安全站点标识和严格关键词占位符。"""
request = collector._prepare_capture_request(
url="https://Tracker.Example.com/torrents.php?search={keyword}&category=1",
keyword="Movie 2026",
)
assert request.site_id == "tracker-example-com"
assert request.origin == "https://tracker.example.com"
assert request.path == "/torrents.php"
assert request.params == {"search": "Movie 2026", "category": "1"}
assert request.public_params == {"search": "{keyword}", "category": "1"}
@pytest.mark.parametrize(
"url",
[
"ftp://tracker.example.com/torrents.php?search={keyword}",
"http://tracker.example.com/torrents.php?search={keyword}",
"https://tracker.example.com/passkey/value/torrents.php?search={keyword}",
"https://tracker.example.com/torrents.php?token=value&search={keyword}",
"https://tracker.example.com/torrents.php?search={keyword}&q={keyword}",
],
)
def test_prepare_capture_request_rejects_unsafe_urls(url: str):
"""采集器应拒绝非 HTTPS 地址及疑似内嵌凭据的 URL。"""
with pytest.raises(ValueError):
collector._prepare_capture_request(url=url, keyword="Movie")
def test_user_details_link_is_not_a_torrent_result():
"""用户详情链接不得被误判为种子详情链接。"""
soup = collector.BeautifulSoup(
'<a href="/userdetails.php?id=9">user</a>',
"html.parser",
)
assert collector._is_torrent_link(soup.a) is False
def test_sanitize_search_html_crops_and_redacts_result_structure():
"""脱敏结果应仅保留种子列表结构,不包含身份、标题和凭据。"""
sanitized_html, row_count, report = collector._sanitize_search_html(
html=SEARCH_HTML,
origin="https://tracker.example.com",
keyword="Movie",
)
lowered = sanitized_html.lower()
assert row_count == 3
assert "torrent-table" in sanitized_html
assert "torrent-row" in sanitized_html
assert "torrent-listings-global-freeleech" in sanitized_html
assert "torrent-session-token-cell" in sanitized_html
assert "unrelated footer" not in sanitized_html
assert "alice" not in lowered
assert "私密电影标题" not in sanitized_html
assert "private-title" not in sanitized_html
assert "embedded-secret-value" not in sanitized_html
assert "passkey" not in lowered
assert "csrf" not in lowered
assert "window.token" not in lowered
sanitized_soup = collector.BeautifulSoup(sanitized_html, "html.parser")
result_row = sanitized_soup.select_one("tr.torrent-row")
assert result_row["id"] == "uploader-redacted"
assert "profile-redacted" in result_row["class"]
assert "redacted" in result_row["class"]
assert "peer-redacted" in result_row["class"]
assert result_row["data-id"] == "0"
assert not result_row.has_attr("data-private-user")
assert not result_row.has_attr("unknown")
assert sanitized_soup.select_one("a.torrent-title")["href"] == "/details.php?id=1"
assert sanitized_soup.select_one("a.username")["href"] == "#redacted-identity"
assert sanitized_soup.select_one("a.torrent-title")["title"] == "[REDACTED]"
assert sanitized_soup.select_one("span.date-added")["title"] == "2000-01-01 00:00"
assert report["redacted"] is True
assert report["contains_credentials"] is False
assert report["captured_rows"] == 3
def test_nested_nexus_table_counts_only_outer_result_rows():
"""NexusPHP 资源名内嵌表格不得重复计数或被当成外层结果裁剪。"""
result_rows = "".join(
f"""
<tr class="outer-result-row">
<td>
<table class="torrentname">
<tbody><tr><td><a href="/details.php?id={index}">资源 {index}</a></td></tr></tbody>
</table>
</td>
<td>{index}</td>
</tr>
"""
for index in range(1, 31)
)
html = f"""
<html><body>
<table class="torrents">
<thead><tr><th>名称</th><th>做种</th></tr></thead>
<tbody>{result_rows}</tbody>
</table>
</body></html>
"""
sanitized_html, row_count, report = collector._sanitize_search_html(
html=html,
origin="https://tracker.example.com",
keyword="Movie",
)
soup = collector.BeautifulSoup(sanitized_html, "html.parser")
assert row_count == collector.MAX_RESULT_ROWS
assert len(soup.select("tr.outer-result-row")) == collector.MAX_RESULT_ROWS
assert len(soup.select("table.torrentname")) == collector.MAX_RESULT_ROWS
assert report["captured_rows"] == collector.MAX_RESULT_ROWS
def test_fetch_search_page_blocks_cross_origin_redirect(monkeypatch):
"""携带 Cookie 的采集请求遇到跨 origin 重定向时必须立即停止。"""
response = _FakeResponse(
status_code=302,
url="https://tracker.example.com/torrents.php?search=Movie",
headers={"Location": "https://login.example.net/sign-in"},
)
client = _FakeClient([response])
monkeypatch.setattr(collector, "RequestUtils", lambda **_: client)
request = collector._prepare_capture_request(
url="https://tracker.example.com/torrents.php?search={keyword}",
keyword="Movie",
)
with pytest.raises(RuntimeError, match="跨域重定向"):
collector._fetch_search_page(request, "session=very-secret-cookie", "Browser UA")
assert len(client.calls) == 1
assert client.calls[0]["allow_redirects"] is False
assert client.calls[0]["verify"] is True
assert response.closed is True
def test_read_limited_response_rejects_oversized_content_length():
"""响应头声明超过容量上限时不应读取响应体。"""
response = _FakeResponse(
status_code=200,
url="https://tracker.example.com/torrents.php",
headers={"Content-Length": str(collector.MAX_RESPONSE_BYTES + 1)},
)
with pytest.raises(ValueError, match="超过 5 MiB"):
collector._read_limited_response(response)
def test_collect_site_capture_writes_fixed_protocol(monkeypatch, tmp_path: Path):
"""采集结果应使用固定四文件协议,并通过摘要和脱敏声明自校验。"""
cookie = "session=very-secret-cookie-value"
monkeypatch.setattr(
collector,
"_fetch_search_page",
lambda request, cookie, user_agent: SEARCH_HTML,
)
archive_path = collector.collect_site_capture(
url="https://tracker.example.com/torrents.php?search={keyword}&category=1",
keyword="Movie",
cookie=cookie,
output_dir=tmp_path,
user_agent="Browser UA",
site_name="示例站点",
)
with zipfile.ZipFile(archive_path) as archive:
assert tuple(archive.namelist()) == collector.ARCHIVE_FILE_NAMES
contents = {name: archive.read(name) for name in archive.namelist()}
manifest = json.loads(contents["manifest.json"])
request = json.loads(contents["request.json"])
report = json.loads(contents["redaction-report.json"])
assert manifest["format_version"] == 1
assert manifest["site"] == {
"id": "tracker-example-com",
"name": "示例站点",
"domain": "https://tracker.example.com",
"public": False,
}
assert manifest["capture"]["kind"] == "search"
assert manifest["capture"]["row_count"] == 3
assert manifest["privacy"] == {
"redacted": True,
"contains_credentials": False,
}
assert manifest["files"]["search.html"] == hashlib.sha256(
contents["search.html"]
).hexdigest()
assert request == {
"method": "get",
"path": "/torrents.php",
"params": {"category": "1", "search": "{keyword}"},
"origin": "https://tracker.example.com",
}
assert report["redacted"] is True
assert report["contains_credentials"] is False
assert b"very-secret-cookie-value" not in b"\n".join(contents.values())
assert b"Browser UA" not in b"\n".join(contents.values())
@pytest.mark.parametrize(
"unsafe_value",
[
"alice@example.com",
"192.168.1.20",
"AbCdEfGhIjKlMnOpQrStUvWxYz0123456789",
],
)
def test_verify_payload_rejects_residual_private_values(unsafe_value: str):
"""最终序列化检查应阻止残留身份信息和疑似高熵凭据写入。"""
payload = {
"manifest.json": b'{"files": {}}',
"request.json": b"{}",
"search.html": unsafe_value.encode("utf-8"),
"redaction-report.json": b"{}",
}
with pytest.raises(ValueError):
collector._verify_payload(payload, "session=safe-cookie-value")
def test_verify_payload_checks_complete_short_cookie():
"""即使 Cookie 很短,最终序列化检查也不得忽略其完整原值。"""
payload = {
"manifest.json": b'{"files": {}}',
"request.json": b"{}",
"search.html": b"id=1",
"redaction-report.json": b"{}",
}
with pytest.raises(ValueError, match="凭据值"):
collector._verify_payload(payload, "id=1")
def test_collect_does_not_write_archive_when_final_scan_fails(monkeypatch, tmp_path: Path):
"""最终序列化检查失败时不得在输出目录留下 ZIP。"""
monkeypatch.setattr(
collector,
"_fetch_search_page",
lambda request, cookie, user_agent: SEARCH_HTML,
)
with pytest.raises(ValueError, match="身份信息"):
collector.collect_site_capture(
url="https://tracker.example.com/torrents.php?search={keyword}",
keyword="Movie",
cookie="session=safe-cookie-value",
output_dir=tmp_path,
site_name="alice@example.com",
)
assert list(tmp_path.iterdir()) == []
def test_infer_search_keyword_from_url_and_visible_input():
"""普通浏览器模式应自动识别 URL 中与搜索框一致的关键词。"""
capture = collector._BrowserCapture(
url="https://tracker.example.com/browse.php?term=Movie%202026&category=1",
html=SEARCH_HTML,
cookie="session=safe-cookie-value",
user_agent="Browser UA",
search_inputs=[{"name": "term", "id": "search", "value": "Movie 2026"}],
)
assert collector._infer_search_keyword(capture) == "Movie 2026"
def test_infer_search_keyword_rejects_post_only_page():
"""地址栏不包含搜索参数时应友好拒绝,避免生成不可复用的 GET 配置。"""
capture = collector._BrowserCapture(
url="https://tracker.example.com/browse.php",
html=SEARCH_HTML,
cookie="session=safe-cookie-value",
user_agent="Browser UA",
search_inputs=[{"name": "search", "id": "search", "value": "Movie"}],
)
with pytest.raises(ValueError, match="地址栏包含关键词"):
collector._infer_search_keyword(capture)
def test_read_browser_capture_filters_cookies_to_current_site(monkeypatch):
"""浏览器采集只应保留当前站点域名的 Cookie 用于本地泄露检查。"""
class _FakeCdpClient:
"""模拟只读 CDP 页面与 Cookie 返回。"""
def __init__(self, websocket_url: str):
"""保存测试传入的 WebSocket 地址。"""
self.websocket_url = websocket_url
def __enter__(self):
"""返回模拟 CDP 客户端。"""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""结束模拟 CDP 上下文。"""
def evaluate(self, expression: str):
"""按表达式返回当前页面模拟数据。"""
if "location.href" in expression:
return "https://tracker.example.com/browse.php?search=Movie"
if "outerHTML" in expression:
return SEARCH_HTML
if "userAgent" in expression:
return "Browser UA"
return [{"name": "search", "id": "search", "value": "Movie"}]
def call(self, method: str):
"""返回当前站点与外部站点的模拟 Cookie。"""
assert method == "Network.getAllCookies"
return {
"cookies": [
{"domain": ".example.com", "name": "session", "value": "site-secret"},
{"domain": ".external.test", "name": "other", "value": "external-secret"},
]
}
monkeypatch.setattr(
collector,
"_select_search_page_target",
lambda port: {"webSocketDebuggerUrl": "ws://127.0.0.1/devtools/page/1"},
)
monkeypatch.setattr(collector, "_CdpClient", _FakeCdpClient)
session = collector._BrowserSession(
process=None,
profile_guard=None,
port=9222,
browser_websocket_url="ws://127.0.0.1/devtools/browser/1",
)
capture = collector._read_browser_capture(session)
assert capture.cookie == "session=site-secret"
assert "external-secret" not in capture.cookie
assert capture.user_agent == "Browser UA"
def test_browser_capture_builds_archive_without_manual_cookie_input(monkeypatch, tmp_path: Path):
"""普通模式应从临时浏览器数据直接生成四文件 ZIP。"""
@contextmanager
def fake_browser_session(start_url: str):
"""提供无需启动真实浏览器的模拟会话。"""
assert start_url == "https://tracker.example.com"
yield object()
browser_capture = collector._BrowserCapture(
url="https://tracker.example.com/torrents.php?search=Movie&category=1",
html=SEARCH_HTML,
cookie="session=very-secret-cookie-value",
user_agent="Browser UA",
search_inputs=[{"name": "search", "id": "search", "value": "Movie"}],
)
monkeypatch.setattr(collector, "_launch_browser_session", fake_browser_session)
monkeypatch.setattr(collector, "_read_browser_capture", lambda session: browser_capture)
monkeypatch.setattr("builtins.input", lambda prompt="": "")
archive_path = collector.collect_site_capture_with_browser(
start_url="https://tracker.example.com",
output_dir=tmp_path,
)
with zipfile.ZipFile(archive_path) as archive:
manifest = json.loads(archive.read("manifest.json"))
request = json.loads(archive.read("request.json"))
combined = b"\n".join(archive.read(name) for name in archive.namelist())
assert manifest["collector_version"] == "1.0.1"
assert request["params"]["search"] == "{keyword}"
assert b"very-secret-cookie-value" not in combined
def test_help_uses_utf8_when_parent_forces_legacy_code_page() -> None:
"""父进程强制 cp1252 时,中文 argparse 帮助仍应以 UTF-8 正常输出。"""
environment = os.environ.copy()
environment["PYTHONIOENCODING"] = "cp1252"
result = subprocess.run(
[sys.executable, str(Path(collector.__file__)), "--help"],
check=False,
capture_output=True,
env=environment,
timeout=30,
)
assert result.returncode == 0, result.stderr.decode("utf-8", errors="replace")
assert "站点适配" in result.stdout.decode("utf-8")

View File

@@ -1,193 +0,0 @@
"""验证采集器独立版本及发布边界,所有 Git 操作仅使用临时仓库。"""
import os
import subprocess
from pathlib import Path
import pytest
from ruamel.yaml import YAML
from scripts.collector import release
PROJECT_ROOT = Path(__file__).parents[1]
def _git(repository: Path, *arguments: str) -> str:
"""在临时仓库运行 Git,隔离真实仓库和用户提交身份。"""
return subprocess.check_output(
["git", "-c", "user.name=Collector Test", "-c", "user.email=collector@example.invalid",
*arguments], cwd=repository, text=True,
).strip()
@pytest.fixture
def collector_repository(tmp_path: Path) -> Path:
"""创建已发布 1.0.1 的临时采集器仓库。"""
_git(tmp_path, "init", "-b", "v3")
for relative in release.RELEASE_PATHS:
path = tmp_path / relative
if relative == "scripts/collector":
path = path / "release.py"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("initial\n", encoding="utf-8")
(tmp_path / "scripts/site_adapter_collector.py").write_text(
'COLLECTOR_VERSION = "1.0.1"\n', encoding="utf-8",
)
_git(tmp_path, "add", ".")
_git(tmp_path, "commit", "-m", "initial collector")
_git(tmp_path, "tag", "site-adapter-collector-v1.0.1")
return tmp_path
def test_latest_release_ignores_backend_and_orders_versions_numerically():
"""后端版本和非法标签不得参与采集器版本排序。"""
assert release.latest_release_tag([
"v3.1.4", "site-adapter-collector-v1.0.9", "site-adapter-collector-v1.0.10",
"site-adapter-collector-v1.0.11-rc1", "site-adapter-collector-v1.01.1",
]) == "site-adapter-collector-v1.0.10"
assert release.latest_release_tag(["v3.1.4"]) is None
def test_first_release_uses_existing_collector_version(collector_repository: Path):
"""首次独立发布沿用采集器版本,不能使用后端版本号。"""
plan = release.release_plan(collector_repository, None)
assert plan["tag"] == "site-adapter-collector-v1.0.1"
assert plan["changed"] == "true"
assert plan["source_sha"] == _git(collector_repository, "rev-parse", "HEAD")
def test_backend_and_document_changes_do_not_release(collector_repository: Path):
"""后端和文档提交、手动重复执行都不能消耗采集器版本。"""
for name in ("version.py", "README.md", "tests/test_collector.py"):
path = collector_repository / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("unrelated\n", encoding="utf-8")
_git(collector_repository, "add", ".")
_git(collector_repository, "commit", "-m", "backend and docs")
for _ in range(2):
plan = release.release_plan(collector_repository, "site-adapter-collector-v1.0.1")
assert plan["changed"] == "false"
assert plan["version"] == "1.0.1"
@pytest.mark.parametrize("relative", release.RELEASE_PATHS)
def test_collector_input_change_increments_patch(collector_repository: Path, relative: str):
"""采集器任意发布输入改变都必须生成下一个补丁版本。"""
path = collector_repository / relative
if relative == "scripts/collector":
path = path / "release.py"
with path.open("a", encoding="utf-8") as output:
output.write("# collector change\n")
_git(collector_repository, "add", ".")
_git(collector_repository, "commit", "-m", "change collector input")
plan = release.release_plan(collector_repository, "site-adapter-collector-v1.0.1")
assert plan["changed"] == "true"
assert plan["version"] == "1.0.2"
def test_subsequent_release_uses_published_version(collector_repository: Path):
"""源码版本无需回写,下次补丁必须以正式 Release 版本为准。"""
_git(collector_repository, "tag", "site-adapter-collector-v1.0.9")
(collector_repository / "scripts/site_adapter_collector.spec").unlink()
_git(collector_repository, "add", ".")
_git(collector_repository, "commit", "-m", "remove collector build input")
assert release.release_plan(
collector_repository, "site-adapter-collector-v1.0.9",
)["version"] == "1.0.10"
def test_missing_previous_tag_fails_instead_of_publishing(collector_repository: Path):
"""比较失败不能误判为采集器改变并发版。"""
with pytest.raises(RuntimeError, match="无法比较"):
release.release_plan(collector_repository, "site-adapter-collector-v1.0.8")
def test_stamp_sets_runtime_and_manifest_version(tmp_path: Path):
"""构建版本写入现有声明,其他源码和采集包协议版本保持一致。"""
source = tmp_path / "collector.py"
source.write_text('COLLECTOR_VERSION = "1.0.1"\nFORMAT_VERSION = 1\n', encoding="utf-8")
release.stamp_version(source, "1.0.10")
assert source.read_text(encoding="utf-8") == 'COLLECTOR_VERSION = "1.0.10"\nFORMAT_VERSION = 1\n'
with pytest.raises(ValueError, match="无效采集器版本"):
release.stamp_version(source, "1.0.10\ninvalid")
def test_workflow_separates_releases_and_pins_all_build_sources():
"""工作流只跟随采集器输入,三平台成功才发布且保留主程序 latest。"""
workflow = YAML(typ="safe").load(
(PROJECT_ROOT / ".github/workflows/site-adapter-collector.yml").read_text(encoding="utf-8"),
)
assert "release" not in workflow["on"]
assert workflow["on"]["push"]["branches"] == ["v3"]
paths = workflow["on"]["push"]["paths"]
assert set(paths) == set(release.RELEASE_PATHS) - {"scripts/collector"} | {"scripts/collector/**"}
assert workflow["concurrency"]["cancel-in-progress"] is False
jobs = workflow["jobs"]
assert "needs.prepare.outputs.changed == 'true'" in jobs["build"]["if"]
assert "!inputs.publish" in jobs["build"]["if"]
assert jobs["publish"]["needs"] == ["prepare", "build"]
for job in ("build", "publish"):
checkout = jobs[job]["steps"][0]
assert checkout["with"]["ref"] == "${{ needs.prepare.outputs.source_sha }}"
build_step = next(step for step in jobs["build"]["steps"] if step["name"] == "Build single-file collector")
assert build_step["shell"] == "bash"
assert "release.py stamp" in build_step["run"]
publish = jobs["publish"]["steps"][-1]["run"]
assert "--target \"$SOURCE_SHA\" --draft" in publish
assert "--draft=false --latest=false" in publish
assert "--json isDraft,targetCommitish" in publish
assert '"$draft_source" != "$SOURCE_SHA"' in publish
assert '"$RELEASE_TAG^{commit}"' not in publish
@pytest.mark.parametrize(
("draft_flag", "draft_source", "accepted"),
[("true", "source-sha", True), ("true", "other-sha", False), ("false", "source-sha", False)],
)
def test_draft_retry_checks_source_without_requiring_git_tag(
tmp_path: Path, draft_flag: str, draft_source: str, accepted: bool,
):
"""实际运行发布 Shell,确认无标签草稿可重试且拒绝错误源码或正式版本。"""
workflow = YAML(typ="safe").load(
(PROJECT_ROOT / ".github/workflows/site-adapter-collector.yml").read_text(encoding="utf-8"),
)
commands = workflow["jobs"]["publish"]["steps"][-1]["run"]
binaries = tmp_path / "bin"
binaries.mkdir()
gh_stub = binaries / "gh"
gh_stub.write_text(
'#!/bin/bash\n'
'if [[ "$1 $2" == "release view" ]]; then\n'
' printf "%s\\t%s\\n" "$DRAFT_FLAG" "$DRAFT_SOURCE"\n'
'else\n'
' printf "%s\\n" "$*" >> "$GH_LOG"\n'
'fi\n', encoding="utf-8",
)
git_stub = binaries / "git"
git_stub.write_text("#!/bin/bash\nexit 79\n", encoding="utf-8")
for path in (gh_stub, git_stub):
path.chmod(0o755)
log = tmp_path / "gh.log"
environment = {
**os.environ,
"PATH": f"{binaries}{os.pathsep}{os.environ['PATH']}",
"DRAFT_FLAG": draft_flag,
"DRAFT_SOURCE": draft_source,
"GH_LOG": str(log),
"SOURCE_SHA": "source-sha",
"RELEASE_TAG": "site-adapter-collector-v1.0.2",
"COLLECTOR_VERSION": "1.0.2",
"GITHUB_REPOSITORY": "test/collector",
}
result = subprocess.run(
["bash", "-e", "-o", "pipefail", "-c", commands],
cwd=tmp_path, env=environment, capture_output=True, text=True, check=False,
)
if accepted:
assert result.returncode == 0, result.stderr
calls = log.read_text(encoding="utf-8")
assert "release upload site-adapter-collector-v1.0.2" in calls
assert "--draft=false --latest=false" in calls
else:
assert result.returncode != 0
assert not log.exists()