From f4a68f360defc7226cdd30dd22bbc16e3b339cf3 Mon Sep 17 00:00:00 2001 From: Aqr-K <95741669+Aqr-K@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:57:10 +0800 Subject: [PATCH] =?UTF-8?q?fix(indexer):=20=E7=AB=99=E7=82=B9=E6=90=9C?= =?UTF-8?q?=E7=B4=A2=E5=85=9C=E5=BA=95=E5=A4=B1=E8=B4=A5=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E9=99=84=E5=B8=A6=E7=8A=B6=E6=80=81=E7=A0=81=E4=B8=8E=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E6=A0=87=E9=A2=98=20(#6819)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/modules/indexer/__init__.py | 46 +++++++++++-------- app/modules/indexer/spider/__init__.py | 47 +++++++++++++++++--- tests/test_indexer_spider_response_parity.py | 46 +++++++++++++++++++ tests/test_indexer_sync_async_parity.py | 8 ++-- tests/test_subscription_search_governance.py | 2 +- tests/test_subscription_site_budget.py | 26 ++++++++--- 6 files changed, 141 insertions(+), 34 deletions(-) diff --git a/app/modules/indexer/__init__.py b/app/modules/indexer/__init__.py index cd1f800e0..a3a81ba13 100644 --- a/app/modules/indexer/__init__.py +++ b/app/modules/indexer/__init__.py @@ -78,6 +78,8 @@ class _IndexerSearchOutcome: result: List[dict[str, Any]] seconds: int error: Optional[Exception] = None + # 爬虫判定失败时给出的原因（状态码、页面标题等），只用于展示，不参与冷却分类 + failure_detail: Optional[str] = None def _classify_search_failure(error: Optional[Exception]) -> str: @@ -101,6 +103,8 @@ def _search_failure_message(outcome: _IndexerSearchOutcome) -> Optional[str]: """返回供站点预算持久化和任务聚合展示的失败原因。""" if outcome.error is not None: return str(outcome.error) + if outcome.error_flag and outcome.failure_detail: + return f"{_UNKNOWN_SEARCH_FAILURE_MESSAGE}（{outcome.failure_detail}）" if outcome.error_flag: return _UNKNOWN_SEARCH_FAILURE_MESSAGE return None @@ -372,26 +376,28 @@ class IndexerModule(_ModuleBase): def __execute_search( site: dict[str, Any], request: _IndexerSearchRequest, - ) -> Tuple[bool, List[dict[str, Any]]]: - """通过同步解析器执行已冻结的搜索请求""" + ) -> Tuple[bool, List[dict[str, Any]], Optional[str]]: + """通过同步解析器执行已冻结的搜索请求，返回错误标志、结果与失败原因""" if request.parser_class: - return cast( + error_flag, result = cast( Tuple[bool, List[dict[str, Any]]], request.parser_class(site).search(**request.arguments), ) + return error_flag, result, None return IndexerModule.__spider_search(**request.arguments) @staticmethod async def __async_execute_search( site: dict[str, Any], request: _IndexerSearchRequest, - ) -> Tuple[bool, List[dict[str, Any]]]: - """通过异步解析器执行已冻结的搜索请求""" + ) -> Tuple[bool, List[dict[str, Any]], Optional[str]]: + """通过异步解析器执行已冻结的搜索请求，返回错误标志、结果与失败原因""" if request.parser_class: - return cast( + error_flag, result = cast( Tuple[bool, List[dict[str, Any]]], await request.parser_class(site).async_search(**request.arguments), ) + return error_flag, result, None return await IndexerModule.__async_spider_search(**request.arguments) @staticmethod @@ -400,12 +406,14 @@ class IndexerModule(_ModuleBase): error_flag: bool, result: List[dict[str, Any]], error: Optional[Exception] = None, + failure_detail: Optional[str] = None, ) -> _IndexerSearchOutcome: """把同步、异步 I/O 结果整理为共用的搜索完成状态""" return _IndexerSearchOutcome( error_flag=error_flag, result=result, seconds=(datetime.now() - start_time).seconds, + failure_detail=failure_detail, error=error, ) @@ -450,13 +458,14 @@ class IndexerModule(_ModuleBase): # 开始搜索 error: Optional[Exception] = None + failure_detail: Optional[str] = None try: - error_flag, result = self.__execute_search(site, request) + error_flag, result, failure_detail = self.__execute_search(site, request) except Exception as err: error = err self.__log_search_error(site, "torrents", err) - outcome = self.__create_search_outcome(start_time, error_flag, result, error) + outcome = self.__create_search_outcome(start_time, error_flag, result, error, failure_detail) # 统计索引情况 self.__indexer_statistic( @@ -506,7 +515,7 @@ class IndexerModule(_ModuleBase): return [] try: - error_flag, result = self.__execute_search(site, request) + error_flag, result, _ = self.__execute_search(site, request) except Exception as err: self.__log_search_error(site, "subtitles", err) @@ -557,13 +566,14 @@ class IndexerModule(_ModuleBase): # 开始搜索 error: Optional[Exception] = None + failure_detail: Optional[str] = None try: - error_flag, result = await self.__async_execute_search(site, request) + error_flag, result, failure_detail = await self.__async_execute_search(site, request) except Exception as err: error = err self.__log_search_error(site, "torrents", err) - outcome = self.__create_search_outcome(start_time, error_flag, result, error) + outcome = self.__create_search_outcome(start_time, error_flag, result, error, failure_detail) # 统计索引情况 await self.__async_indexer_statistic( @@ -613,7 +623,7 @@ class IndexerModule(_ModuleBase): return [] try: - error_flag, result = await self.__async_execute_search(site, request) + error_flag, result, _ = await self.__async_execute_search(site, request) except Exception as err: self.__log_search_error(site, "subtitles", err) @@ -635,7 +645,7 @@ class IndexerModule(_ModuleBase): mtype: MediaType = None, cat: Optional[str] = None, page: Optional[int] = 0, - search_type: Optional[str] = "torrents") -> Tuple[bool, List[dict]]: + search_type: Optional[str] = "torrents") -> Tuple[bool, List[dict], Optional[str]]: """ 根据关键字搜索单个站点 :param: indexer: 站点配置 @@ -644,7 +654,7 @@ class IndexerModule(_ModuleBase): :param: page: 页码 :param: mtype: 媒体类型 :param: timeout: 超时时间 - :return: 是否发生错误, 种子列表 + :return: 是否发生错误, 种子列表, 失败原因 """ _spider = SiteSpider(indexer=indexer, keyword=search_word, @@ -655,7 +665,7 @@ class IndexerModule(_ModuleBase): try: result = _spider.get_torrents() - return _spider.is_error, result + return _spider.is_error, result, _spider.error_detail finally: del _spider @@ -665,7 +675,7 @@ class IndexerModule(_ModuleBase): mtype: MediaType = None, cat: Optional[str] = None, page: Optional[int] = 0, - search_type: Optional[str] = "torrents") -> Tuple[bool, List[dict]]: + search_type: Optional[str] = "torrents") -> Tuple[bool, List[dict], Optional[str]]: """ 异步根据关键字搜索单个站点 :param: indexer: 站点配置 @@ -674,7 +684,7 @@ class IndexerModule(_ModuleBase): :param: page: 页码 :param: mtype: 媒体类型 :param: timeout: 超时时间 - :return: 是否发生错误, 种子列表 + :return: 是否发生错误, 种子列表, 失败原因 """ _spider = SiteSpider(indexer=indexer, keyword=search_word, @@ -685,7 +695,7 @@ class IndexerModule(_ModuleBase): try: result = await _spider.async_get_torrents() - return _spider.is_error, result + return _spider.is_error, result, _spider.error_detail finally: del _spider diff --git a/app/modules/indexer/spider/__init__.py b/app/modules/indexer/spider/__init__.py index e185c415b..e3f58f3dc 100644 --- a/app/modules/indexer/spider/__init__.py +++ b/app/modules/indexer/spider/__init__.py @@ -18,6 +18,35 @@ from app.runtime.log import logger from app.runtime.settings import get_runtime_setting from app.schemas.types import MediaType +# 失败原因中页面摘要的最大字符数，以及查找页面标题时扫描的正文前缀长度 +_FAILURE_SUMMARY_MAX_CHARS = 80 +_FAILURE_TITLE_SCAN_CHARS = 4096 +_HTML_TITLE_PATTERN = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) +_HTML_TAG_PATTERN = re.compile(r"<[^>]+>") + + +def _describe_unusable_response(response: Any) -> str: + """ + 概括未得到可解析页面的搜索响应，供搜索失败原因展示。 + + :param response: 同步或异步 HTTP 响应，请求失败时为 None + :return: 状态码与页面标题（无标题时为正文开头的纯文本）组成的说明 + """ + if response is None: + return "站点无响应（网络错误或超时）" + status_code = getattr(response, "status_code", None) + try: + body = str(response.text or "") + except Exception: # noqa: BLE001 - 正文解码失败时只保留状态码 + body = "" + title = _HTML_TITLE_PATTERN.search(body[:_FAILURE_TITLE_SCAN_CHARS]) + source = title.group(1) if title else body[:_FAILURE_TITLE_SCAN_CHARS] + summary = " ".join(_HTML_TAG_PATTERN.sub(" ", source).split())[:_FAILURE_SUMMARY_MAX_CHARS] + if not summary: + return f"HTTP {status_code}，响应内容为空" + label = "页面标题" if title else "响应内容" + return f"HTTP {status_code}，{label}：{summary}" + def select_media_categories(category: Optional[dict], mtype: Optional[MediaType]) -> list[dict]: """根据媒体类型选择站点索引配置中的分类列表。""" @@ -143,6 +172,8 @@ class SiteSpider: self.referer = referer # 初始化属性 self.is_error = False + # is_error 为真时的失败原因，供搜索失败提示展示 + self.error_detail: Optional[str] = None self.torrents_info = {} self.torrents_info_array = [] @@ -371,8 +402,11 @@ class SiteSpider: referer=self.referer, proxies=self.proxies ).get_res(searchurl, allow_redirects=True) + html_text = self.__decode_response(ret) + if not html_text: + self.error_detail = _describe_unusable_response(ret) # 解析返回 - return self.parse(self.__decode_response(ret)) + return self.parse(html_text) async def async_get_torrents(self) -> List[dict]: """ @@ -394,11 +428,11 @@ class SiteSpider: referer=self.referer, proxies=self.proxies ).get_res(searchurl, allow_redirects=True) + html_text = self.__decode_response(ret) + if not html_text: + self.error_detail = _describe_unusable_response(ret) # 解析返回 - return await run_in_threadpool( - self.parse, - self.__decode_response(ret) - ) + return await run_in_threadpool(self.parse, html_text) def __get_title(self, torrent: Any): """按站点字段配置提取并清洗种子标题。""" @@ -1095,10 +1129,12 @@ class SiteSpider: status_doc = PyQuery(html_text) if self.__is_login_or_permission_page(status_doc): self.is_error = True + self.error_detail = "返回登录或权限提示页" logger.warn(f"错误：{self.indexername} 返回登录或权限提示页") return [] except Exception as err: self.is_error = True + self.error_detail = f"页面解析失败：{err}" logger.warn(f"错误：{self.indexername} {str(err)}") return [] finally: @@ -1160,6 +1196,7 @@ class SiteSpider: return self.__apply_result_media_type(self.torrents_info_array.copy()) except Exception as err: self.is_error = True + self.error_detail = f"页面解析失败：{err}" logger.warn(f"错误：{self.indexername} {str(err)}") return [] finally: diff --git a/tests/test_indexer_spider_response_parity.py b/tests/test_indexer_spider_response_parity.py index b958ad6bb..54e017ce4 100644 --- a/tests/test_indexer_spider_response_parity.py +++ b/tests/test_indexer_spider_response_parity.py @@ -425,3 +425,49 @@ def test_site_spider_sync_async_share_admission_and_decoding(monkeypatch): rejected = SiteSpider(indexer={**indexer, "media_type": "music"}, mtype=MediaType.MOVIE) assert rejected.get_torrents() == [] assert asyncio.run(rejected.async_get_torrents()) == [] + + +@pytest.mark.parametrize( + ("response", "detail"), + [ + (None, "站点无响应（网络错误或超时）"), + ( + _FakeResponse( + status_code=403, + text="\n Just a moment...\n" + "Enable JavaScript and cookies to continue", + ), + "HTTP 403，页面标题：Just a moment...", + ), + (_FakeResponse(status_code=502, text="Bad Gateway"), "HTTP 502，响应内容：Bad Gateway"), + (_FakeResponse(status_code=200, text=""), "HTTP 200，响应内容为空"), + ( + _FakeResponse(text="登录"), + "返回登录或权限提示页", + ), + ], +) +def test_site_spider_records_failure_detail_for_unusable_page(monkeypatch, response, detail): + """未拿到可用页面时同步、异步入口都记录状态码与页面摘要，供搜索失败原因展示。""" + indexer = { + "id": "generic", + "name": "Generic", + "domain": "https://tracker.example/", + "search": {"paths": [{"path": "browse.php"}]}, + "torrents": {"list": {}, "fields": {}}, + } + + async def async_request(*_args, **_kwargs): + """回放普通页面异步响应。""" + return response + + monkeypatch.setattr(RequestUtils, "get_res", lambda *_args, **_kwargs: response) + monkeypatch.setattr(AsyncRequestUtils, "get_res", async_request) + + sync_spider = SiteSpider(indexer=indexer, keyword="Movie") + async_spider = SiteSpider(indexer=indexer, keyword="Movie") + + assert sync_spider.get_torrents() == [] + assert asyncio.run(async_spider.async_get_torrents()) == [] + assert sync_spider.is_error and async_spider.is_error + assert sync_spider.error_detail == async_spider.error_detail == detail diff --git a/tests/test_indexer_sync_async_parity.py b/tests/test_indexer_sync_async_parity.py index 094bc824e..634fde29a 100644 --- a/tests/test_indexer_sync_async_parity.py +++ b/tests/test_indexer_sync_async_parity.py @@ -259,12 +259,12 @@ def test_generic_torrent_search_uses_identical_request_projection(monkeypatch): def spider_search(**kwargs): calls.append(("sync", kwargs)) events.append(("io", kwargs)) - return False, [{"title": "generic"}] + return False, [{"title": "generic"}], None async def async_spider_search(**kwargs): calls.append(("async", kwargs)) events.append(("io", kwargs)) - return False, [{"title": "generic"}] + return False, [{"title": "generic"}], None monkeypatch.setattr( IndexerModule, @@ -338,12 +338,12 @@ def test_subtitle_search_uses_one_request_and_preserves_order(monkeypatch): def spider_search(**kwargs): calls.append(("sync", kwargs)) events.append(("io", kwargs)) - return True, [] + return True, [], None async def async_spider_search(**kwargs): calls.append(("async", kwargs)) events.append(("io", kwargs)) - return True, [] + return True, [], None monkeypatch.setattr( IndexerModule, diff --git a/tests/test_subscription_search_governance.py b/tests/test_subscription_search_governance.py index 0a9ab60e8..e8b60c6f9 100644 --- a/tests/test_subscription_search_governance.py +++ b/tests/test_subscription_search_governance.py @@ -509,7 +509,7 @@ def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp attempts += 1 if attempts == 1: raise RuntimeError("HTTP 429") - return False, [] + return False, [], None monkeypatch.setattr( IndexerModule, diff --git a/tests/test_subscription_site_budget.py b/tests/test_subscription_site_budget.py index 136ab6385..b19ed948b 100644 --- a/tests/test_subscription_site_budget.py +++ b/tests/test_subscription_site_budget.py @@ -460,8 +460,22 @@ def test_search_provider_aggregates_swallowed_indexer_failure(monkeypatch): assert snapshot.cooldown_seconds == 900.0 -def test_search_provider_explains_error_flag_without_exception(monkeypatch): - """索引器仅返回错误标志时也必须持久化可读原因，不能向任务暴露裸 `error`。""" +@pytest.mark.parametrize( + ("failure_detail", "expected_error"), + [ + (None, "站点请求或页面解析失败"), + ( + "HTTP 403，页面标题：Just a moment...", + "站点请求或页面解析失败（HTTP 403，页面标题：Just a moment...）", + ), + ], +) +def test_search_provider_explains_error_flag_without_exception( + monkeypatch, + failure_detail, + expected_error, +): + """索引器仅返回错误标志时也必须持久化可读原因，爬虫给出的失败原因附在兜底文案后且不改变冷却分类。""" captured = {} class _Repository(_WaitingRepository): @@ -491,7 +505,7 @@ def test_search_provider_explains_error_flag_without_exception(monkeypatch): monkeypatch.setattr( IndexerModule, "_IndexerModule__execute_search", - staticmethod(lambda _site, _request: (True, [])), + staticmethod(lambda _site, _request: (True, [], failure_detail)), ) monkeypatch.setattr( IndexerModule, @@ -527,11 +541,11 @@ def test_search_provider_explains_error_flag_without_exception(monkeypatch): assert result == [] assert captured["outcome"] == "error" - assert captured["error"] == "站点请求或页面解析失败" + assert captured["error"] == expected_error assert chain.consume_subscription_site_budget_failures() == ( - "站点 Generic 搜索失败：站点请求或页面解析失败", + f"站点 Generic 搜索失败：{expected_error}", ) - assert warnings == ["站点 Generic 搜索失败：站点请求或页面解析失败"] + assert warnings == [f"站点 Generic 搜索失败：{expected_error}"] def test_search_provider_logs_site_budget_release_failure(monkeypatch):