fix(download): confirm tracker forms and blacklist invalid torrents

2026-11-03 22:04:57 +08:00 · 2026-10-13 21:54:37 +08:00
parent 8cbba9916e
commit c4cd6c8ac4
7 changed files with 207 additions and 49 deletions
--- a/app/application/torrent/download.py
+++ b/app/application/torrent/download.py
@@ -5,8 +5,9 @@ import time
 from functools import lru_cache
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Protocol, Tuple, Union
-from urllib.parse import parse_qs, unquote, urlsplit
+from urllib.parse import parse_qs, unquote, urljoin, urlsplit

+from lxml import html
 from torrentool.api import Torrent
 from torrentool.exceptions import TorrentoolException

@@ -158,6 +159,32 @@ def _cache_valid_torrent_content(
    return folder_name, file_list


+def _first_download_form(text: str, url: str) -> Optional[Tuple[str, dict[str, Any]]]:
+    """解析下载确认表单，仅向当前下载端点提交有效控件，避免泄露站点凭据。"""
+    try:
+        forms = html.fromstring(text).forms
+    except (ValueError, html.etree.ParserError):
+        return None
+    for form in forms:
+        action = form.get("action", "").strip()
+        target = url if action in ("", "?") else urljoin(url, action)
+        origin, destination = urlsplit(url), urlsplit(target)
+        if (destination[:2] != origin[:2] or destination.path != origin.path
+                or form.get("method", "post").lower() != "post"):
+            continue
+        data = dict(form.form_values())
+        # form_values 不包含用户点击的提交控件，首次下载确认可能依赖该字段。
+        submits = form.xpath(
+            './/input[@name and not(@disabled) and translate(@type, "SUBMIT", "submit")="submit"]'
+            ' | .//button[@name and not(@disabled) and (not(@type) or @type="submit")]'
+        )
+        if submits:
+            data[submits[0].get("name")] = submits[0].get("value", "")
+        if data:
+            return target, data
+    return None
+
+
 def _text_filter_failure(torrent: TorrentInfo, params: Dict[str, Any], content: str) -> Optional[Tuple[str, str]]:
    """按原有顺序检查包含、排除、质量等正则，并返回业务原因和运行诊断。"""
    for key, label, subject, expected in (
@@ -273,6 +300,7 @@ class TorrentHelper:
            return None, url, "", [], "磁力链接"
        # 构建 torrent 种子文件的缓存路径
        cache_path = Path(HashUtils.md5(url)).with_suffix(".torrent")
+        original_url = url
        # 种子二进制固定落 TEMP_PATH/torrents 不进 Redis，下载器与提交链须按同一路由读取
        cache_backend = FileCache(local_only=True)
        # 读取缓存的种子文件
@@ -324,41 +352,26 @@ class TorrentHelper:
                return cache_path, req.content, folder_name, file_list, ""
            if "下载种子文件".encode("utf-8") in req.content:
                # 首次下载提示页面
-                skip_flag = False
-                try:
-                    forms = re.findall(r'<form.*?action="(.*?)".*?>(.*?)</form>', req.text, re.S)
-                    for form in forms:
-                        action = form[0]
-                        if action != "?":
-                            continue
-                        action = url
-                        inputs = re.findall(r'<input.*?name="(.*?)".*?value="(.*?)".*?>', form[1], re.S)
-                        if inputs:
-                            data = {}
-                            for item in inputs:
-                                data[item[0]] = item[1]
-                            # 改写req
-                            req = http_port.request(
-                                method="POST", url=action, cookie=cookie, ua=ua,
-                                referer=referer, proxies=config.proxy if proxy else None,
-                                data=data,
-                            )
-                            if req and req.status_code == 200:
-                                # 检查是不是种子文件，如果不是抛出异常
-                                Torrent.from_string(req.content)
-                                # 跳过成功
-                                logger.info("触发了站点首次种子下载，已自动跳过")
-                                skip_flag = True
-                            elif req is not None:
-                                logger.warn(f"触发了站点首次种子下载，且无法自动跳过，"
-                                            f"返回码：{req.status_code}，错误原因：{req.reason}")
-                            else:
-                                logger.warn("触发了站点首次种子下载，且无法自动跳过")
-                        break
-                except Exception as err:
-                    logger.warn(f"触发了站点首次种子下载，尝试自动跳过时出现错误：{str(err)}")
-                if not skip_flag:
+                confirmation = _first_download_form(req.text, url)
+                if confirmation is None:
+                    logger.warning("触发首次种子下载，但未找到可安全提交的确认表单")
+                    if cache_invalid:
+                        self.add_invalid(original_url)
                    return cache_path, None, "", [], "种子数据有误，请确认链接是否正确，如为PT站点则需手工在站点下载一次种子"
+                action, data = confirmation
+                req = http_port.request(
+                    method="POST", url=action, cookie=cookie, ua=ua,
+                    referer=url, proxies=config.proxy if proxy else None, data=data,
+                )
+                if rate_limit_error := _torrent_rate_limit_error(req):
+                    return cache_path, None, "", [], rate_limit_error
+                if req is None:
+                    return cache_path, None, "", [], "无法打开链接"
+                if req.status_code != 200:
+                    return cache_path, None, "", [], f"下载种子出错，状态码：{req.status_code}"
+                if not req.content:
+                    return cache_path, None, "", [], "未下载到种子数据"
+                logger.info("已提交站点首次种子下载确认，正在校验种子内容")
            # 首次下载提示页面完成后，重新按原始二进制解析 POST 响应。
            assert req is not None
            if req.content:
@@ -369,13 +382,15 @@ class TorrentHelper:
                    # 成功拿到种子数据
                    return cache_path, req.content, folder_name, file_list, ""
            # 解析失败时终止，避免把 HTML 继续下沉到下载器或缓存。
+            if cache_invalid:
+                self.add_invalid(original_url)
            return cache_path, None, "", [], "种子数据有误，请确认链接是否正确"
        elif req is None:
            return cache_path, None, "", [], "无法打开链接"
        else:
            # 把错误的种子记下来，避免重复使用
            if cache_invalid and not _is_transient_torrent_status(req.status_code):
-                self.add_invalid(url)
+                self.add_invalid(original_url)
            return cache_path, None, "", [], f"下载种子出错，状态码：{req.status_code}"

    def get_torrent_info(self, torrent_path: Path) -> Tuple[str, List[str]]:
--- a/app/chain/download/failure.py
+++ b/app/chain/download/failure.py
@@ -29,6 +29,8 @@ DOWNLOAD_FAILURE_RESOURCE_ERROR_KEYWORDS = (
    "无法读取种子文件",
    "无法获取下载地址",
    "种子下载失败",
+    "种子数据有误",
+    "下载失败黑名单",
    "torrent not found",
    "not found",
    "404",
--- a/app/chain/download/submission.py
+++ b/app/chain/download/submission.py
@@ -237,6 +237,12 @@ class _DownloadResourceOwner(_DownloadOwnerBase):
            return None, "", []
        if torrent.enclosure.startswith("magnet:"):
            return torrent.enclosure, "", []
+        helper: TorrentHelper = cast(Any, TorrentHelper)()
+        if helper.is_invalid(torrent.enclosure):
+            if on_failure:
+                on_failure("种子已加入下载失败黑名单（24小时）")
+            logger.info(f"{torrent.title} 命中下载失败黑名单，跳过下载")
+            return None, "", []
        torrent = self._with_site_credentials(torrent)
        result = self._get_torrent_content(torrent)
        if result is None:
@@ -250,6 +256,8 @@ class _DownloadResourceOwner(_DownloadOwnerBase):
        if not content:
            if on_failure:
                on_failure(error_msg or "下载种子内容为空")
+            if error_msg and "种子数据有误" in error_msg:
+                helper.add_invalid(torrent.enclosure)
            logger.error(f"下载种子文件失败：{torrent.title}")
            self.post_message(Message(
                channel=channel,
@@ -582,16 +590,15 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
            context.selected_episodes = []
        download_error = "下载种子内容为空"

-        def remember_rate_limit(error_msg: str) -> None:
-            """仅保留本次限流原因，避免准备流程用空内容覆盖站点冷却窗口。"""
+        def remember_failure(error_msg: str) -> None:
+            """保留真实失败原因，使无效种子和限流分别使用正确的资源冷却。"""
            nonlocal download_error
-            if torrent_rules.rate_limit_cooldown(error_msg) is not None:
-                download_error = error_msg
+            download_error = error_msg

        if not torrent_file and not torrent_content:
            torrent_content, _, _ = self.download_torrent(
                torrent, channel=channel, source=source, userid=userid,
-                on_failure=remember_rate_limit,
+                on_failure=remember_failure,
            )
        elif torrent_file:
            torrent_content = (
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -109,7 +109,8 @@ with patch.object(SomeModule, "fetch", new=AsyncMock(return_value=FAKE)):

 种子下载重试的离线边界见 `tests/test_torrent_download_retry.py`：普通 GET 在网络失败、
 403/5xx、空响应或无法解析的响应后最多重试两次，退避为 1、2 秒。签名链接刷新、换票地址、
-首次下载确认 POST 保持既有流程。HTTP 429 或 JSON `message` / `msg` 明确提示限流时，
+首次下载确认按 HTML 表单语义读取控件，兼容单双引号、属性顺序、空 action 和提交按钮；
+仅向同来源的下载端点提交一次 POST，使用确认页地址作为 Referer。HTTP 429 或 JSON `message` / `msg` 明确提示限流时，
 停止短退避重试并在通知中保留原因，不把限流响应交给种子解析器或写入无效地址缓存。
 测试替换 HTTP 端口与等待，不访问真实站点。
 瞬时状态不会写入 24 小时无效链接缓存；订阅记录的泛化“下载种子内容为空”使用 1 小时冷却，
@@ -117,6 +118,11 @@ with patch.object(SomeModule, "fetch", new=AsyncMock(return_value=FAKE)):
 短期限流至少冷却 1 小时，提示等待更长时按提示延长。准备流程通过单次失败回调把限流原因
 交给持久化冷却，避免统一的空内容错误覆盖它；签名链接遇到明确限流也不刷新重试。
 此策略不追溯修改已经写入的失败记录。
+种子内容耗尽重试仍无效或首次下载确认仍返回无效内容时，加入现有 24 小时下载失败黑名单；
+下载链再次遇到该地址直接跳过请求和通知。黑名单保存原始资源地址，不使用重定向目标；
+短期签名或换票资源在下载链中屏蔽原始 enclosure，不将临时请求地址写入无效缓存。
+订阅持久化保留具体失败原因，无效种子使用 24 小时资源冷却，避免重启后每小时重复通知。
+网络异常、空响应、403/5xx 和限流仍沿用瞬时重试与流控规则，不因本次修改加入内容无效黑名单。

 换票资源的跨批次复用见 `tests/test_download_resource_cache.py`：种子成功解析后，
 本地 `torrents` 缓存以站点资源身份、原始换票请求和 Cookie 的摘要保存内容，命中后
--- a/tests/fixtures/architecture/complexity-baseline.json
+++ b/tests/fixtures/architecture/complexity-baseline.json
@@ -38,7 +38,7 @@
      "app/application/rss.py:RssHelper._parse_impl": 37,
      "app/application/security/cookie.py:CookieHelper._get_site_cookie_ua_impl": 45,
      "app/application/security/cookie.py:CookieHelper._get_site_cookie_ua_impl.__page_handler": 43,
-      "app/application/torrent/download.py:TorrentHelper.download_torrent": 25,
+      "app/application/torrent/download.py:TorrentHelper.download_torrent": 24,
      "app/application/torrent/download.py:TorrentHelper.match_torrent": 16,
      "app/application/transfer/models.py:TransferPlanCheckpoint.__post_init__": 18,
      "app/chain/download/existence.py:DownloadExistenceOwner.get_no_exists_info": 23,
@@ -141,7 +141,6 @@
      "app/agent/policy/sanitizer.py:sanitize_for_host": 6,
      "app/api/endpoints/transfer.py:_execute_manual_transfer": 6,
      "app/application/rss.py:RssHelper._parse_impl": 7,
-      "app/application/torrent/download.py:TorrentHelper.download_torrent": 6,
      "app/chain/scraping.py:ScrapingChain._download_and_save_image": 7,
      "app/chain/scraping.py:ScrapingChain.scrape_metadata_event": 6,
      "app/chain/subscribe/match.py:SubscribeMatchOwner._execute_match": 7,
--- a/tests/test_torrent_download_retry.py
+++ b/tests/test_torrent_download_retry.py
@@ -39,8 +39,9 @@ def boundary(monkeypatch):
    monkeypatch.setattr(download, "FileCache", Mock(return_value=cache))
    sleep = Mock()
    monkeypatch.setattr(download.time, "sleep", sleep)
+    monkeypatch.setattr(download, "TTLCache", Mock(return_value={}))
    helper = download.TorrentHelper()
-    helper.add_invalid = Mock()
+    helper.add_invalid = Mock(wraps=helper.add_invalid)
    return helper, port, cache, sleep


@@ -112,10 +113,137 @@ def test_first_download_confirmation_post_is_not_retried(boundary):
    page = '下载种子文件<form action="?"><input name="confirm" value="yes"></form>'.encode()
    port.request.side_effect = [_response(content=page), _response(503)]

-    assert helper.download_torrent(URL)[1] is None
+    result = helper.download_torrent(URL)
+    assert result[1] is None
+    assert result[4] == "下载种子出错，状态码：503"

    assert [call.kwargs["method"] for call in port.request.call_args_list] == ["GET", "POST"]
    sleep.assert_not_called()
+    helper.add_invalid.assert_not_called()
+
+
+def test_first_download_confirmation_preserves_quota_error(boundary):
+    """确认 POST 遇到站点限流时保留原因，不把限流页当作失效种子。"""
+    helper, port, _, _ = boundary
+    page = '下载种子文件<form action="?"><input name="confirm" value="yes"></form>'.encode()
+    message = "相同種子當天最多下載10次"
+    port.request.side_effect = [_response(content=page), _response(content=json.dumps({"message": message}).encode())]
+
+    assert helper.download_torrent(URL)[4] == f"站点限流：{message}"
+    assert port.request.call_count == 2
+    helper.add_invalid.assert_not_called()
+
+
+def test_first_download_confirmation_empty_response_is_transient(boundary):
+    """确认请求的空响应不证明种子无效，不能污染资源黑名单。"""
+    helper, port, _, _ = boundary
+    page = '下载种子文件<form action="?"><input name="confirm" value="yes"></form>'.encode()
+    port.request.side_effect = [_response(content=page), _response(content=b"")]
+
+    assert helper.download_torrent(URL)[4] == "未下载到种子数据"
+    assert port.request.call_count == 2
+    helper.add_invalid.assert_not_called()
+
+
+@pytest.mark.parametrize("form", [
+    '<form action="?"><input value="yes" name="confirm"></form>',
+    "<form method='POST' action='?'><input value='a&amp;b' name='token'></form>",
+    '<form method="post"><input name="confirm" value="yes"></form>',
+    '<form action="download.php?id=39" method="post"><input name="confirm" value="yes"></form>',
+    '<form action="?"><input name="token" value="a&amp;b"><button name="confirm" value="yes">下载</button></form>',
+])
+def test_first_download_form_variants_are_confirmed(boundary, form):
+    """按 HTML 表单语义提交确认，兼容属性顺序、引号、实体和提交按钮。"""
+    helper, port, cache, sleep = boundary
+    page = ("下载种子文件" + form).encode()
+    port.request.side_effect = [_response(content=page), _response()]
+
+    assert helper.download_torrent(URL, cookie="session=test")[1] == CONTENT
+
+    assert [call.kwargs["method"] for call in port.request.call_args_list] == ["GET", "POST"]
+    posted = port.request.call_args.kwargs
+    assert posted["url"] == URL
+    assert posted["cookie"] == "session=test"
+    assert posted["referer"] == URL
+    assert posted["data"] == ({"token": "a&b", "confirm": "yes"} if "button" in form else
+                              {"token": "a&b"} if "token" in form else {"confirm": "yes"})
+    cache.set.assert_called_once()
+    helper.add_invalid.assert_not_called()
+    sleep.assert_not_called()
+
+
+def test_first_download_form_never_posts_credentials_to_other_origin(boundary):
+    """确认表单不能把站点 Cookie 和隐藏凭据发送到其他来源。"""
+    helper, port, _, _ = boundary
+    page = '下载种子文件<form action="https://other.example/download.php"><input name="token" value="secret"></form>'
+    port.request.return_value = _response(content=page.encode())
+
+    assert helper.download_torrent(URL, cookie="session=test")[1] is None
+    port.request.assert_called_once()
+    helper.add_invalid.assert_called_once_with(URL)
+
+
+def test_failed_confirmation_blacklists_original_url_after_redirect(boundary):
+    """确认后仍返回 HTML 时屏蔽原资源地址，刷新候选使用同一个黑名单键。"""
+    helper, port, _, _ = boundary
+    redirect = _response(302)
+    redirect.headers["Location"] = "https://tracker.example/download.php?id=39&confirm=1"
+    page = '下载种子文件<form action="?"><input name="confirm" value="yes"></form>'.encode()
+    port.request.side_effect = [redirect, _response(content=page), _response(content=page)]
+
+    assert helper.download_torrent(URL)[1] is None
+    helper.add_invalid.assert_called_once_with(URL)
+    assert helper.is_invalid(URL)
+
+
+def test_chain_blacklist_suppresses_repeated_requests_and_notifications(boundary):
+    """多轮下载同一失败资源只提醒首次，并阻止后续请求和错误通知。"""
+    _, port, _, _ = boundary
+    port.request.return_value = _response(content=b"<html>invalid torrent</html>")
+    chain = object.__new__(DownloadChain)
+    chain.post_message = Mock()
+    torrent = TorrentInfo(site=1, site_name="test", title="Series S01E39", enclosure=URL)
+
+    for _ in range(5):
+        assert chain.download_torrent(torrent)[0] is None
+
+    assert port.request.call_count == 3
+    chain.post_message.assert_called_once()
+
+
+@pytest.mark.parametrize("error", ["种子数据有误，请确认链接是否正确",
+                                   "种子数据有误，请确认链接是否正确，如为PT站点则需手工在站点下载一次种子"])
+def test_invalid_torrent_data_has_resource_cooldown(error):
+    """无效种子及确认失败使用资源级冷却，重启也不会退回每小时通知。"""
+    assert DownloadFailureOwner._download_failure_ttl(error) == 86400
+
+
+def test_invalid_content_preparation_preserves_persisted_resource_cooldown(boundary, monkeypatch):
+    """单任务准备保留无效种子原因，持久化冷却不能再被统一空内容错误覆盖。"""
+    _, port, _, _ = boundary
+    port.request.return_value = _response(content=b"<html>invalid torrent</html>")
+    chain = object.__new__(DownloadChain)
+    chain.post_message = Mock()
+    chain.download_failure_repository = Mock()
+    chain._apply_resource_download_event = Mock(return_value=(None, None))
+    context = Context(
+        torrent_info=TorrentInfo(site=1, site_name="test", title="Series S01E39", enclosure=URL),
+        media_info=MediaInfo(title="Series", type=MediaType.TV), meta_info=MetaInfo("Series S01E39"),
+    )
+    media = Mock()
+    media.supplement_tmdb_info.return_value = context.media_info
+    monkeypatch.setattr(submission, "MediaChain", Mock(return_value=media))
+
+    prepared, error = chain._prepare_download_single(
+        context=context, torrent_file=None, torrent_content=None, episodes=None,
+        channel=None, source="Subscribe|1", downloader=None, save_path=None, userid=None, username=None,
+    )
+
+    assert prepared is None
+    failure = chain.download_failure_repository.record_failure.call_args.args[0]
+    assert failure.error_message == error == "种子数据有误，请确认链接是否正确"
+    assert (datetime.fromisoformat(failure.next_retry_at)
+            - datetime.fromisoformat(failure.failed_at)).total_seconds() == 86400


 def test_redirected_get_retries_at_destination(boundary):
@@ -144,13 +272,13 @@ def test_second_retry_can_recover(boundary):

@pytest.mark.parametrize("content", [b"", b"d4:info", b"<html>CF</html>"])
 def test_invalid_content_exhaustion_never_caches_response(boundary, content):
-    """空或无法解析的响应耗尽预算后仍不能进入种子及无效链接缓存。"""
+    """无效内容耗尽重试后进入黑名单，空响应仍按瞬时失败处理。"""
    helper, port, cache, _ = boundary
    port.request.return_value = _response(content=content)
    assert helper.download_torrent(URL)[1] is None
    assert port.request.call_count == 3
    cache.set.assert_not_called()
-    helper.add_invalid.assert_not_called()
+    assert helper.add_invalid.call_count == (1 if content else 0)


@pytest.mark.parametrize("content", [CONTENT, b"magnet:?xt=urn:btih:abc"])
--- a/tests/test_torrent_link_refresh.py
+++ b/tests/test_torrent_link_refresh.py
@@ -126,6 +126,7 @@ def download_chain(monkeypatch):
    chain.post_message = Mock()
    chain.search_site_torrents = Mock(return_value=[])
    helper = Mock()
+    helper.is_invalid.return_value = False
    helper.download_torrent.return_value = FAILED
    monkeypatch.setattr(submission, "TorrentHelper", Mock(return_value=helper))
    cache = Mock()
@@ -309,7 +310,7 @@ def test_real_torrent_parse_and_cooldown_follow_final_attempt(monkeypatch, final
        assert cache.set.call_args.args[1] == content
    else:
        assert prepared is None
-        assert error == "下载种子内容为空"
+        assert error == f"下载种子出错，状态码：{final_status}"
        chain._record_download_failure.assert_called_once()
        chain.post_message.assert_called_once()
        cache.set.assert_not_called()