diff --git a/app/application/torrent/download.py b/app/application/torrent/download.py index b27a08070..0d35edee9 100644 --- a/app/application/torrent/download.py +++ b/app/application/torrent/download.py @@ -5,8 +5,9 @@ import time from functools import lru_cache from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Protocol, Tuple, Union -from urllib.parse import parse_qs, unquote, urlsplit +from urllib.parse import parse_qs, unquote, urljoin, urlsplit +from lxml import html from torrentool.api import Torrent from torrentool.exceptions import TorrentoolException @@ -158,6 +159,32 @@ def _cache_valid_torrent_content( return folder_name, file_list +def _first_download_form(text: str, url: str) -> Optional[Tuple[str, dict[str, Any]]]: + """解析下载确认表单，仅向当前下载端点提交有效控件，避免泄露站点凭据。""" + try: + forms = html.fromstring(text).forms + except (ValueError, html.etree.ParserError): + return None + for form in forms: + action = form.get("action", "").strip() + target = url if action in ("", "?") else urljoin(url, action) + origin, destination = urlsplit(url), urlsplit(target) + if (destination[:2] != origin[:2] or destination.path != origin.path + or form.get("method", "post").lower() != "post"): + continue + data = dict(form.form_values()) + # form_values 不包含用户点击的提交控件，首次下载确认可能依赖该字段。 + submits = form.xpath( + './/input[@name and not(@disabled) and translate(@type, "SUBMIT", "submit")="submit"]' + ' | .//button[@name and not(@disabled) and (not(@type) or @type="submit")]' + ) + if submits: + data[submits[0].get("name")] = submits[0].get("value", "") + if data: + return target, data + return None + + def _text_filter_failure(torrent: TorrentInfo, params: Dict[str, Any], content: str) -> Optional[Tuple[str, str]]: """按原有顺序检查包含、排除、质量等正则，并返回业务原因和运行诊断。""" for key, label, subject, expected in ( @@ -273,6 +300,7 @@ class TorrentHelper: return None, url, "", [], "磁力链接" # 构建 torrent 种子文件的缓存路径 cache_path = Path(HashUtils.md5(url)).with_suffix(".torrent") + original_url = url # 种子二进制固定落 TEMP_PATH/torrents 不进 Redis，下载器与提交链须按同一路由读取 cache_backend = FileCache(local_only=True) # 读取缓存的种子文件 @@ -324,41 +352,26 @@ class TorrentHelper: return cache_path, req.content, folder_name, file_list, "" if "下载种子文件".encode("utf-8") in req.content: # 首次下载提示页面 - skip_flag = False - try: - forms = re.findall(r'(.*?)', req.text, re.S) - for form in forms: - action = form[0] - if action != "?": - continue - action = url - inputs = re.findall(r'', form[1], re.S) - if inputs: - data = {} - for item in inputs: - data[item[0]] = item[1] - # 改写req - req = http_port.request( - method="POST", url=action, cookie=cookie, ua=ua, - referer=referer, proxies=config.proxy if proxy else None, - data=data, - ) - if req and req.status_code == 200: - # 检查是不是种子文件，如果不是抛出异常 - Torrent.from_string(req.content) - # 跳过成功 - logger.info("触发了站点首次种子下载，已自动跳过") - skip_flag = True - elif req is not None: - logger.warn(f"触发了站点首次种子下载，且无法自动跳过，" - f"返回码：{req.status_code}，错误原因：{req.reason}") - else: - logger.warn("触发了站点首次种子下载，且无法自动跳过") - break - except Exception as err: - logger.warn(f"触发了站点首次种子下载，尝试自动跳过时出现错误：{str(err)}") - if not skip_flag: + confirmation = _first_download_form(req.text, url) + if confirmation is None: + logger.warning("触发首次种子下载，但未找到可安全提交的确认表单") + if cache_invalid: + self.add_invalid(original_url) return cache_path, None, "", [], "种子数据有误，请确认链接是否正确，如为PT站点则需手工在站点下载一次种子" + action, data = confirmation + req = http_port.request( + method="POST", url=action, cookie=cookie, ua=ua, + referer=url, proxies=config.proxy if proxy else None, data=data, + ) + if rate_limit_error := _torrent_rate_limit_error(req): + return cache_path, None, "", [], rate_limit_error + if req is None: + return cache_path, None, "", [], "无法打开链接" + if req.status_code != 200: + return cache_path, None, "", [], f"下载种子出错，状态码：{req.status_code}" + if not req.content: + return cache_path, None, "", [], "未下载到种子数据" + logger.info("已提交站点首次种子下载确认，正在校验种子内容") # 首次下载提示页面完成后，重新按原始二进制解析 POST 响应。 assert req is not None if req.content: @@ -369,13 +382,15 @@ class TorrentHelper: # 成功拿到种子数据 return cache_path, req.content, folder_name, file_list, "" # 解析失败时终止，避免把 HTML 继续下沉到下载器或缓存。 + if cache_invalid: + self.add_invalid(original_url) return cache_path, None, "", [], "种子数据有误，请确认链接是否正确" elif req is None: return cache_path, None, "", [], "无法打开链接" else: # 把错误的种子记下来，避免重复使用 if cache_invalid and not _is_transient_torrent_status(req.status_code): - self.add_invalid(url) + self.add_invalid(original_url) return cache_path, None, "", [], f"下载种子出错，状态码：{req.status_code}" def get_torrent_info(self, torrent_path: Path) -> Tuple[str, List[str]]: diff --git a/app/chain/download/failure.py b/app/chain/download/failure.py index d75910a54..830ee1a94 100644 --- a/app/chain/download/failure.py +++ b/app/chain/download/failure.py @@ -29,6 +29,8 @@ DOWNLOAD_FAILURE_RESOURCE_ERROR_KEYWORDS = ( "无法读取种子文件", "无法获取下载地址", "种子下载失败", + "种子数据有误", + "下载失败黑名单", "torrent not found", "not found", "404", diff --git a/app/chain/download/submission.py b/app/chain/download/submission.py index eb3095bff..cf25272b1 100644 --- a/app/chain/download/submission.py +++ b/app/chain/download/submission.py @@ -237,6 +237,12 @@ class _DownloadResourceOwner(_DownloadOwnerBase): return None, "", [] if torrent.enclosure.startswith("magnet:"): return torrent.enclosure, "", [] + helper: TorrentHelper = cast(Any, TorrentHelper)() + if helper.is_invalid(torrent.enclosure): + if on_failure: + on_failure("种子已加入下载失败黑名单（24小时）") + logger.info(f"{torrent.title} 命中下载失败黑名单，跳过下载") + return None, "", [] torrent = self._with_site_credentials(torrent) result = self._get_torrent_content(torrent) if result is None: @@ -250,6 +256,8 @@ class _DownloadResourceOwner(_DownloadOwnerBase): if not content: if on_failure: on_failure(error_msg or "下载种子内容为空") + if error_msg and "种子数据有误" in error_msg: + helper.add_invalid(torrent.enclosure) logger.error(f"下载种子文件失败：{torrent.title}") self.post_message(Message( channel=channel, @@ -582,16 +590,15 @@ class DownloadSubmissionOwner(_DownloadResourceOwner): context.selected_episodes = [] download_error = "下载种子内容为空" - def remember_rate_limit(error_msg: str) -> None: - """仅保留本次限流原因，避免准备流程用空内容覆盖站点冷却窗口。""" + def remember_failure(error_msg: str) -> None: + """保留真实失败原因，使无效种子和限流分别使用正确的资源冷却。""" nonlocal download_error - if torrent_rules.rate_limit_cooldown(error_msg) is not None: - download_error = error_msg + download_error = error_msg if not torrent_file and not torrent_content: torrent_content, _, _ = self.download_torrent( torrent, channel=channel, source=source, userid=userid, - on_failure=remember_rate_limit, + on_failure=remember_failure, ) elif torrent_file: torrent_content = ( diff --git a/docs/testing.md b/docs/testing.md index 7c3eebd4c..6d8f1d154 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -109,7 +109,8 @@ with patch.object(SomeModule, "fetch", new=AsyncMock(return_value=FAKE)): 种子下载重试的离线边界见 `tests/test_torrent_download_retry.py`：普通 GET 在网络失败、 403/5xx、空响应或无法解析的响应后最多重试两次，退避为 1、2 秒。签名链接刷新、换票地址、 -首次下载确认 POST 保持既有流程。HTTP 429 或 JSON `message` / `msg` 明确提示限流时， +首次下载确认按 HTML 表单语义读取控件，兼容单双引号、属性顺序、空 action 和提交按钮； +仅向同来源的下载端点提交一次 POST，使用确认页地址作为 Referer。HTTP 429 或 JSON `message` / `msg` 明确提示限流时， 停止短退避重试并在通知中保留原因，不把限流响应交给种子解析器或写入无效地址缓存。 测试替换 HTTP 端口与等待，不访问真实站点。 瞬时状态不会写入 24 小时无效链接缓存；订阅记录的泛化“下载种子内容为空”使用 1 小时冷却， @@ -117,6 +118,11 @@ with patch.object(SomeModule, "fetch", new=AsyncMock(return_value=FAKE)): 短期限流至少冷却 1 小时，提示等待更长时按提示延长。准备流程通过单次失败回调把限流原因 交给持久化冷却，避免统一的空内容错误覆盖它；签名链接遇到明确限流也不刷新重试。 此策略不追溯修改已经写入的失败记录。 +种子内容耗尽重试仍无效或首次下载确认仍返回无效内容时，加入现有 24 小时下载失败黑名单； +下载链再次遇到该地址直接跳过请求和通知。黑名单保存原始资源地址，不使用重定向目标； +短期签名或换票资源在下载链中屏蔽原始 enclosure，不将临时请求地址写入无效缓存。 +订阅持久化保留具体失败原因，无效种子使用 24 小时资源冷却，避免重启后每小时重复通知。 +网络异常、空响应、403/5xx 和限流仍沿用瞬时重试与流控规则，不因本次修改加入内容无效黑名单。 换票资源的跨批次复用见 `tests/test_download_resource_cache.py`：种子成功解析后， 本地 `torrents` 缓存以站点资源身份、原始换票请求和 Cookie 的摘要保存内容，命中后 diff --git a/tests/fixtures/architecture/complexity-baseline.json b/tests/fixtures/architecture/complexity-baseline.json index 846731188..a4b939e4c 100644 --- a/tests/fixtures/architecture/complexity-baseline.json +++ b/tests/fixtures/architecture/complexity-baseline.json @@ -38,7 +38,7 @@ "app/application/rss.py:RssHelper._parse_impl": 37, "app/application/security/cookie.py:CookieHelper._get_site_cookie_ua_impl": 45, "app/application/security/cookie.py:CookieHelper._get_site_cookie_ua_impl.__page_handler": 43, - "app/application/torrent/download.py:TorrentHelper.download_torrent": 25, + "app/application/torrent/download.py:TorrentHelper.download_torrent": 24, "app/application/torrent/download.py:TorrentHelper.match_torrent": 16, "app/application/transfer/models.py:TransferPlanCheckpoint.__post_init__": 18, "app/chain/download/existence.py:DownloadExistenceOwner.get_no_exists_info": 23, @@ -141,7 +141,6 @@ "app/agent/policy/sanitizer.py:sanitize_for_host": 6, "app/api/endpoints/transfer.py:_execute_manual_transfer": 6, "app/application/rss.py:RssHelper._parse_impl": 7, - "app/application/torrent/download.py:TorrentHelper.download_torrent": 6, "app/chain/scraping.py:ScrapingChain._download_and_save_image": 7, "app/chain/scraping.py:ScrapingChain.scrape_metadata_event": 6, "app/chain/subscribe/match.py:SubscribeMatchOwner._execute_match": 7, diff --git a/tests/test_torrent_download_retry.py b/tests/test_torrent_download_retry.py index 29119d452..72587863d 100644 --- a/tests/test_torrent_download_retry.py +++ b/tests/test_torrent_download_retry.py @@ -39,8 +39,9 @@ def boundary(monkeypatch): monkeypatch.setattr(download, "FileCache", Mock(return_value=cache)) sleep = Mock() monkeypatch.setattr(download.time, "sleep", sleep) + monkeypatch.setattr(download, "TTLCache", Mock(return_value={})) helper = download.TorrentHelper() - helper.add_invalid = Mock() + helper.add_invalid = Mock(wraps=helper.add_invalid) return helper, port, cache, sleep @@ -112,10 +113,137 @@ def test_first_download_confirmation_post_is_not_retried(boundary): page = '下载种子文件
'.encode() port.request.side_effect = [_response(content=page), _response(503)] - assert helper.download_torrent(URL)[1] is None + result = helper.download_torrent(URL) + assert result[1] is None + assert result[4] == "下载种子出错，状态码：503" assert [call.kwargs["method"] for call in port.request.call_args_list] == ["GET", "POST"] sleep.assert_not_called() + helper.add_invalid.assert_not_called() + + +def test_first_download_confirmation_preserves_quota_error(boundary): + """确认 POST 遇到站点限流时保留原因，不把限流页当作失效种子。""" + helper, port, _, _ = boundary + page = '下载种子文件
'.encode() + message = "相同種子當天最多下載10次" + port.request.side_effect = [_response(content=page), _response(content=json.dumps({"message": message}).encode())] + + assert helper.download_torrent(URL)[4] == f"站点限流：{message}" + assert port.request.call_count == 2 + helper.add_invalid.assert_not_called() + + +def test_first_download_confirmation_empty_response_is_transient(boundary): + """确认请求的空响应不证明种子无效，不能污染资源黑名单。""" + helper, port, _, _ = boundary + page = '下载种子文件
'.encode() + port.request.side_effect = [_response(content=page), _response(content=b"")] + + assert helper.download_torrent(URL)[4] == "未下载到种子数据" + assert port.request.call_count == 2 + helper.add_invalid.assert_not_called() + + +@pytest.mark.parametrize("form", [ + '
', + "
", + '
', + '
', + '
', +]) +def test_first_download_form_variants_are_confirmed(boundary, form): + """按 HTML 表单语义提交确认，兼容属性顺序、引号、实体和提交按钮。""" + helper, port, cache, sleep = boundary + page = ("下载种子文件" + form).encode() + port.request.side_effect = [_response(content=page), _response()] + + assert helper.download_torrent(URL, cookie="session=test")[1] == CONTENT + + assert [call.kwargs["method"] for call in port.request.call_args_list] == ["GET", "POST"] + posted = port.request.call_args.kwargs + assert posted["url"] == URL + assert posted["cookie"] == "session=test" + assert posted["referer"] == URL + assert posted["data"] == ({"token": "a&b", "confirm": "yes"} if "button" in form else + {"token": "a&b"} if "token" in form else {"confirm": "yes"}) + cache.set.assert_called_once() + helper.add_invalid.assert_not_called() + sleep.assert_not_called() + + +def test_first_download_form_never_posts_credentials_to_other_origin(boundary): + """确认表单不能把站点 Cookie 和隐藏凭据发送到其他来源。""" + helper, port, _, _ = boundary + page = '下载种子文件
' + port.request.return_value = _response(content=page.encode()) + + assert helper.download_torrent(URL, cookie="session=test")[1] is None + port.request.assert_called_once() + helper.add_invalid.assert_called_once_with(URL) + + +def test_failed_confirmation_blacklists_original_url_after_redirect(boundary): + """确认后仍返回 HTML 时屏蔽原资源地址，刷新候选使用同一个黑名单键。""" + helper, port, _, _ = boundary + redirect = _response(302) + redirect.headers["Location"] = "https://tracker.example/download.php?id=39&confirm=1" + page = '下载种子文件
'.encode() + port.request.side_effect = [redirect, _response(content=page), _response(content=page)] + + assert helper.download_torrent(URL)[1] is None + helper.add_invalid.assert_called_once_with(URL) + assert helper.is_invalid(URL) + + +def test_chain_blacklist_suppresses_repeated_requests_and_notifications(boundary): + """多轮下载同一失败资源只提醒首次，并阻止后续请求和错误通知。""" + _, port, _, _ = boundary + port.request.return_value = _response(content=b"invalid torrent") + chain = object.__new__(DownloadChain) + chain.post_message = Mock() + torrent = TorrentInfo(site=1, site_name="test", title="Series S01E39", enclosure=URL) + + for _ in range(5): + assert chain.download_torrent(torrent)[0] is None + + assert port.request.call_count == 3 + chain.post_message.assert_called_once() + + +@pytest.mark.parametrize("error", ["种子数据有误，请确认链接是否正确", + "种子数据有误，请确认链接是否正确，如为PT站点则需手工在站点下载一次种子"]) +def test_invalid_torrent_data_has_resource_cooldown(error): + """无效种子及确认失败使用资源级冷却，重启也不会退回每小时通知。""" + assert DownloadFailureOwner._download_failure_ttl(error) == 86400 + + +def test_invalid_content_preparation_preserves_persisted_resource_cooldown(boundary, monkeypatch): + """单任务准备保留无效种子原因，持久化冷却不能再被统一空内容错误覆盖。""" + _, port, _, _ = boundary + port.request.return_value = _response(content=b"invalid torrent") + chain = object.__new__(DownloadChain) + chain.post_message = Mock() + chain.download_failure_repository = Mock() + chain._apply_resource_download_event = Mock(return_value=(None, None)) + context = Context( + torrent_info=TorrentInfo(site=1, site_name="test", title="Series S01E39", enclosure=URL), + media_info=MediaInfo(title="Series", type=MediaType.TV), meta_info=MetaInfo("Series S01E39"), + ) + media = Mock() + media.supplement_tmdb_info.return_value = context.media_info + monkeypatch.setattr(submission, "MediaChain", Mock(return_value=media)) + + prepared, error = chain._prepare_download_single( + context=context, torrent_file=None, torrent_content=None, episodes=None, + channel=None, source="Subscribe|1", downloader=None, save_path=None, userid=None, username=None, + ) + + assert prepared is None + failure = chain.download_failure_repository.record_failure.call_args.args[0] + assert failure.error_message == error == "种子数据有误，请确认链接是否正确" + assert (datetime.fromisoformat(failure.next_retry_at) + - datetime.fromisoformat(failure.failed_at)).total_seconds() == 86400 def test_redirected_get_retries_at_destination(boundary): @@ -144,13 +272,13 @@ def test_second_retry_can_recover(boundary): @pytest.mark.parametrize("content", [b"", b"d4:info", b"CF"]) def test_invalid_content_exhaustion_never_caches_response(boundary, content): - """空或无法解析的响应耗尽预算后仍不能进入种子及无效链接缓存。""" + """无效内容耗尽重试后进入黑名单，空响应仍按瞬时失败处理。""" helper, port, cache, _ = boundary port.request.return_value = _response(content=content) assert helper.download_torrent(URL)[1] is None assert port.request.call_count == 3 cache.set.assert_not_called() - helper.add_invalid.assert_not_called() + assert helper.add_invalid.call_count == (1 if content else 0) @pytest.mark.parametrize("content", [CONTENT, b"magnet:?xt=urn:btih:abc"]) diff --git a/tests/test_torrent_link_refresh.py b/tests/test_torrent_link_refresh.py index 338ebaf83..ec43c9625 100644 --- a/tests/test_torrent_link_refresh.py +++ b/tests/test_torrent_link_refresh.py @@ -126,6 +126,7 @@ def download_chain(monkeypatch): chain.post_message = Mock() chain.search_site_torrents = Mock(return_value=[]) helper = Mock() + helper.is_invalid.return_value = False helper.download_torrent.return_value = FAILED monkeypatch.setattr(submission, "TorrentHelper", Mock(return_value=helper)) cache = Mock() @@ -309,7 +310,7 @@ def test_real_torrent_parse_and_cooldown_follow_final_attempt(monkeypatch, final assert cache.set.call_args.args[1] == content else: assert prepared is None - assert error == "下载种子内容为空" + assert error == f"下载种子出错，状态码：{final_status}" chain._record_download_failure.assert_called_once() chain.post_message.assert_called_once() cache.set.assert_not_called()