fix(indexer): 站点搜索兜底失败信息附带状态码与页面标题 (#6819)

app/modules/indexer/__init__.py

@@ -78,6 +78,8 @@ class _IndexerSearchOutcome:
     result: List[dict[str, Any]]
     seconds: int
     error: Optional[Exception] = None
+    # 爬虫判定失败时给出的原因（状态码、页面标题等），只用于展示，不参与冷却分类
+    failure_detail: Optional[str] = None
 
 
 def _classify_search_failure(error: Optional[Exception]) -> str:
@@ -101,6 +103,8 @@ def _search_failure_message(outcome: _IndexerSearchOutcome) -> Optional[str]:
     """返回供站点预算持久化和任务聚合展示的失败原因。"""
     if outcome.error is not None:
         return str(outcome.error)
+    if outcome.error_flag and outcome.failure_detail:
+        return f"{_UNKNOWN_SEARCH_FAILURE_MESSAGE}（{outcome.failure_detail}）"
     if outcome.error_flag:
         return _UNKNOWN_SEARCH_FAILURE_MESSAGE
     return None
@@ -372,26 +376,28 @@ class IndexerModule(_ModuleBase):
     def __execute_search(
         site: dict[str, Any],
         request: _IndexerSearchRequest,
-    ) -> Tuple[bool, List[dict[str, Any]]]:
-        """通过同步解析器执行已冻结的搜索请求"""
+    ) -> Tuple[bool, List[dict[str, Any]], Optional[str]]:
+        """通过同步解析器执行已冻结的搜索请求，返回错误标志、结果与失败原因"""
         if request.parser_class:
-            return cast(
+            error_flag, result = cast(
                 Tuple[bool, List[dict[str, Any]]],
                 request.parser_class(site).search(**request.arguments),
             )
+            return error_flag, result, None
         return IndexerModule.__spider_search(**request.arguments)
 
     @staticmethod
     async def __async_execute_search(
         site: dict[str, Any],
         request: _IndexerSearchRequest,
-    ) -> Tuple[bool, List[dict[str, Any]]]:
-        """通过异步解析器执行已冻结的搜索请求"""
+    ) -> Tuple[bool, List[dict[str, Any]], Optional[str]]:
+        """通过异步解析器执行已冻结的搜索请求，返回错误标志、结果与失败原因"""
         if request.parser_class:
-            return cast(
+            error_flag, result = cast(
                 Tuple[bool, List[dict[str, Any]]],
                 await request.parser_class(site).async_search(**request.arguments),
             )
+            return error_flag, result, None
         return await IndexerModule.__async_spider_search(**request.arguments)
 
     @staticmethod
@@ -400,12 +406,14 @@ class IndexerModule(_ModuleBase):
         error_flag: bool,
         result: List[dict[str, Any]],
         error: Optional[Exception] = None,
+        failure_detail: Optional[str] = None,
     ) -> _IndexerSearchOutcome:
         """把同步、异步 I/O 结果整理为共用的搜索完成状态"""
         return _IndexerSearchOutcome(
             error_flag=error_flag,
             result=result,
             seconds=(datetime.now() - start_time).seconds,
+            failure_detail=failure_detail,
             error=error,
         )
 
@@ -450,13 +458,14 @@ class IndexerModule(_ModuleBase):
 
         # 开始搜索
         error: Optional[Exception] = None
+        failure_detail: Optional[str] = None
         try:
-            error_flag, result = self.__execute_search(site, request)
+            error_flag, result, failure_detail = self.__execute_search(site, request)
         except Exception as err:
             error = err
             self.__log_search_error(site, "torrents", err)
 
-        outcome = self.__create_search_outcome(start_time, error_flag, result, error)
+        outcome = self.__create_search_outcome(start_time, error_flag, result, error, failure_detail)
 
         # 统计索引情况
         self.__indexer_statistic(
@@ -506,7 +515,7 @@ class IndexerModule(_ModuleBase):
             return []
 
         try:
-            error_flag, result = self.__execute_search(site, request)
+            error_flag, result, _ = self.__execute_search(site, request)
         except Exception as err:
             self.__log_search_error(site, "subtitles", err)
 
@@ -557,13 +566,14 @@ class IndexerModule(_ModuleBase):
 
         # 开始搜索
         error: Optional[Exception] = None
+        failure_detail: Optional[str] = None
         try:
-            error_flag, result = await self.__async_execute_search(site, request)
+            error_flag, result, failure_detail = await self.__async_execute_search(site, request)
         except Exception as err:
             error = err
             self.__log_search_error(site, "torrents", err)
 
-        outcome = self.__create_search_outcome(start_time, error_flag, result, error)
+        outcome = self.__create_search_outcome(start_time, error_flag, result, error, failure_detail)
 
         # 统计索引情况
         await self.__async_indexer_statistic(
@@ -613,7 +623,7 @@ class IndexerModule(_ModuleBase):
             return []
 
         try:
-            error_flag, result = await self.__async_execute_search(site, request)
+            error_flag, result, _ = await self.__async_execute_search(site, request)
         except Exception as err:
             self.__log_search_error(site, "subtitles", err)
 
@@ -635,7 +645,7 @@ class IndexerModule(_ModuleBase):
                         mtype: MediaType = None,
                         cat: Optional[str] = None,
                         page: Optional[int] = 0,
-                        search_type: Optional[str] = "torrents") -> Tuple[bool, List[dict]]:
+                        search_type: Optional[str] = "torrents") -> Tuple[bool, List[dict], Optional[str]]:
         """
         根据关键字搜索单个站点
         :param: indexer: 站点配置
@@ -644,7 +654,7 @@ class IndexerModule(_ModuleBase):
         :param: page: 页码
         :param: mtype: 媒体类型
         :param: timeout: 超时时间
-        :return: 是否发生错误, 种子列表
+        :return: 是否发生错误, 种子列表, 失败原因
         """
         _spider = SiteSpider(indexer=indexer,
                              keyword=search_word,
@@ -655,7 +665,7 @@ class IndexerModule(_ModuleBase):
 
         try:
             result = _spider.get_torrents()
-            return _spider.is_error, result
+            return _spider.is_error, result, _spider.error_detail
         finally:
             del _spider
 
@@ -665,7 +675,7 @@ class IndexerModule(_ModuleBase):
                                     mtype: MediaType = None,
                                     cat: Optional[str] = None,
                                     page: Optional[int] = 0,
-                                    search_type: Optional[str] = "torrents") -> Tuple[bool, List[dict]]:
+                                    search_type: Optional[str] = "torrents") -> Tuple[bool, List[dict], Optional[str]]:
         """
         异步根据关键字搜索单个站点
         :param: indexer: 站点配置
@@ -674,7 +684,7 @@ class IndexerModule(_ModuleBase):
         :param: page: 页码
         :param: mtype: 媒体类型
         :param: timeout: 超时时间
-        :return: 是否发生错误, 种子列表
+        :return: 是否发生错误, 种子列表, 失败原因
         """
         _spider = SiteSpider(indexer=indexer,
                              keyword=search_word,
@@ -685,7 +695,7 @@ class IndexerModule(_ModuleBase):
 
         try:
             result = await _spider.async_get_torrents()
-            return _spider.is_error, result
+            return _spider.is_error, result, _spider.error_detail
         finally:
             del _spider
 

app/modules/indexer/spider/__init__.py

@@ -18,6 +18,35 @@ from app.runtime.log import logger
 from app.runtime.settings import get_runtime_setting
 from app.schemas.types import MediaType
 
+# 失败原因中页面摘要的最大字符数，以及查找页面标题时扫描的正文前缀长度
+_FAILURE_SUMMARY_MAX_CHARS = 80
+_FAILURE_TITLE_SCAN_CHARS = 4096
+_HTML_TITLE_PATTERN = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
+_HTML_TAG_PATTERN = re.compile(r"<[^>]+>")
+
+
+def _describe_unusable_response(response: Any) -> str:
+    """
+    概括未得到可解析页面的搜索响应，供搜索失败原因展示。
+
+    :param response: 同步或异步 HTTP 响应，请求失败时为 None
+    :return: 状态码与页面标题（无标题时为正文开头的纯文本）组成的说明
+    """
+    if response is None:
+        return "站点无响应（网络错误或超时）"
+    status_code = getattr(response, "status_code", None)
+    try:
+        body = str(response.text or "")
+    except Exception:  # noqa: BLE001 - 正文解码失败时只保留状态码
+        body = ""
+    title = _HTML_TITLE_PATTERN.search(body[:_FAILURE_TITLE_SCAN_CHARS])
+    source = title.group(1) if title else body[:_FAILURE_TITLE_SCAN_CHARS]
+    summary = " ".join(_HTML_TAG_PATTERN.sub(" ", source).split())[:_FAILURE_SUMMARY_MAX_CHARS]
+    if not summary:
+        return f"HTTP {status_code}，响应内容为空"
+    label = "页面标题" if title else "响应内容"
+    return f"HTTP {status_code}，{label}：{summary}"
+
 
 def select_media_categories(category: Optional[dict], mtype: Optional[MediaType]) -> list[dict]:
     """根据媒体类型选择站点索引配置中的分类列表。"""
@@ -143,6 +172,8 @@ class SiteSpider:
         self.referer = referer
         # 初始化属性
         self.is_error = False
+        # is_error 为真时的失败原因，供搜索失败提示展示
+        self.error_detail: Optional[str] = None
         self.torrents_info = {}
         self.torrents_info_array = []
 
@@ -371,8 +402,11 @@ class SiteSpider:
             referer=self.referer,
             proxies=self.proxies
         ).get_res(searchurl, allow_redirects=True)
+        html_text = self.__decode_response(ret)
+        if not html_text:
+            self.error_detail = _describe_unusable_response(ret)
         # 解析返回
-        return self.parse(self.__decode_response(ret))
+        return self.parse(html_text)
 
     async def async_get_torrents(self) -> List[dict]:
         """
@@ -394,11 +428,11 @@ class SiteSpider:
             referer=self.referer,
             proxies=self.proxies
         ).get_res(searchurl, allow_redirects=True)
+        html_text = self.__decode_response(ret)
+        if not html_text:
+            self.error_detail = _describe_unusable_response(ret)
         # 解析返回
-        return await run_in_threadpool(
-            self.parse,
-            self.__decode_response(ret)
-        )
+        return await run_in_threadpool(self.parse, html_text)
 
     def __get_title(self, torrent: Any):
         """按站点字段配置提取并清洗种子标题。"""
@@ -1095,10 +1129,12 @@ class SiteSpider:
             status_doc = PyQuery(html_text)
             if self.__is_login_or_permission_page(status_doc):
                 self.is_error = True
+                self.error_detail = "返回登录或权限提示页"
                 logger.warn(f"错误：{self.indexername} 返回登录或权限提示页")
                 return []
         except Exception as err:
             self.is_error = True
+            self.error_detail = f"页面解析失败：{err}"
             logger.warn(f"错误：{self.indexername} {str(err)}")
             return []
         finally:
@@ -1160,6 +1196,7 @@ class SiteSpider:
             return self.__apply_result_media_type(self.torrents_info_array.copy())
         except Exception as err:
             self.is_error = True
+            self.error_detail = f"页面解析失败：{err}"
             logger.warn(f"错误：{self.indexername} {str(err)}")
             return []
         finally:

tests/test_indexer_spider_response_parity.py

@@ -425,3 +425,49 @@ def test_site_spider_sync_async_share_admission_and_decoding(monkeypatch):
     rejected = SiteSpider(indexer={**indexer, "media_type": "music"}, mtype=MediaType.MOVIE)
     assert rejected.get_torrents() == []
     assert asyncio.run(rejected.async_get_torrents()) == []
+
+
+@pytest.mark.parametrize(
+    ("response", "detail"),
+    [
+        (None, "站点无响应（网络错误或超时）"),
+        (
+            _FakeResponse(
+                status_code=403,
+                text="<html><head><title>\n  Just a moment...\n</title></head>"
+                     "<body>Enable JavaScript and cookies to continue</body></html>",
+            ),
+            "HTTP 403，页面标题：Just a moment...",
+        ),
+        (_FakeResponse(status_code=502, text="Bad Gateway"), "HTTP 502，响应内容：Bad Gateway"),
+        (_FakeResponse(status_code=200, text=""), "HTTP 200，响应内容为空"),
+        (
+            _FakeResponse(text="<html><head><title>登录</title></head><body></body></html>"),
+            "返回登录或权限提示页",
+        ),
+    ],
+)
+def test_site_spider_records_failure_detail_for_unusable_page(monkeypatch, response, detail):
+    """未拿到可用页面时同步、异步入口都记录状态码与页面摘要，供搜索失败原因展示。"""
+    indexer = {
+        "id": "generic",
+        "name": "Generic",
+        "domain": "https://tracker.example/",
+        "search": {"paths": [{"path": "browse.php"}]},
+        "torrents": {"list": {}, "fields": {}},
+    }
+
+    async def async_request(*_args, **_kwargs):
+        """回放普通页面异步响应。"""
+        return response
+
+    monkeypatch.setattr(RequestUtils, "get_res", lambda *_args, **_kwargs: response)
+    monkeypatch.setattr(AsyncRequestUtils, "get_res", async_request)
+
+    sync_spider = SiteSpider(indexer=indexer, keyword="Movie")
+    async_spider = SiteSpider(indexer=indexer, keyword="Movie")
+
+    assert sync_spider.get_torrents() == []
+    assert asyncio.run(async_spider.async_get_torrents()) == []
+    assert sync_spider.is_error and async_spider.is_error
+    assert sync_spider.error_detail == async_spider.error_detail == detail

tests/test_indexer_sync_async_parity.py

@@ -259,12 +259,12 @@ def test_generic_torrent_search_uses_identical_request_projection(monkeypatch):
     def spider_search(**kwargs):
         calls.append(("sync", kwargs))
         events.append(("io", kwargs))
-        return False, [{"title": "generic"}]
+        return False, [{"title": "generic"}], None
 
     async def async_spider_search(**kwargs):
         calls.append(("async", kwargs))
         events.append(("io", kwargs))
-        return False, [{"title": "generic"}]
+        return False, [{"title": "generic"}], None
 
     monkeypatch.setattr(
         IndexerModule,
@@ -338,12 +338,12 @@ def test_subtitle_search_uses_one_request_and_preserves_order(monkeypatch):
     def spider_search(**kwargs):
         calls.append(("sync", kwargs))
         events.append(("io", kwargs))
-        return True, []
+        return True, [], None
 
     async def async_spider_search(**kwargs):
         calls.append(("async", kwargs))
         events.append(("io", kwargs))
-        return True, []
+        return True, [], None
 
     monkeypatch.setattr(
         IndexerModule,

tests/test_subscription_search_governance.py

@@ -509,7 +509,7 @@ def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp
         attempts += 1
         if attempts == 1:
             raise RuntimeError("HTTP 429")
-        return False, []
+        return False, [], None
 
     monkeypatch.setattr(
         IndexerModule,

tests/test_subscription_site_budget.py

@@ -460,8 +460,22 @@ def test_search_provider_aggregates_swallowed_indexer_failure(monkeypatch):
     assert snapshot.cooldown_seconds == 900.0
 
 
-def test_search_provider_explains_error_flag_without_exception(monkeypatch):
-    """索引器仅返回错误标志时也必须持久化可读原因，不能向任务暴露裸 `error`。"""
+@pytest.mark.parametrize(
+    ("failure_detail", "expected_error"),
+    [
+        (None, "站点请求或页面解析失败"),
+        (
+            "HTTP 403，页面标题：Just a moment...",
+            "站点请求或页面解析失败（HTTP 403，页面标题：Just a moment...）",
+        ),
+    ],
+)
+def test_search_provider_explains_error_flag_without_exception(
+    monkeypatch,
+    failure_detail,
+    expected_error,
+):
+    """索引器仅返回错误标志时也必须持久化可读原因，爬虫给出的失败原因附在兜底文案后且不改变冷却分类。"""
     captured = {}
 
     class _Repository(_WaitingRepository):
@@ -491,7 +505,7 @@ def test_search_provider_explains_error_flag_without_exception(monkeypatch):
     monkeypatch.setattr(
         IndexerModule,
         "_IndexerModule__execute_search",
-        staticmethod(lambda _site, _request: (True, [])),
+        staticmethod(lambda _site, _request: (True, [], failure_detail)),
     )
     monkeypatch.setattr(
         IndexerModule,
@@ -527,11 +541,11 @@ def test_search_provider_explains_error_flag_without_exception(monkeypatch):
 
     assert result == []
     assert captured["outcome"] == "error"
-    assert captured["error"] == "站点请求或页面解析失败"
+    assert captured["error"] == expected_error
     assert chain.consume_subscription_site_budget_failures() == (
-        "站点 Generic 搜索失败：站点请求或页面解析失败",
+        f"站点 Generic 搜索失败：{expected_error}",
     )
-    assert warnings == ["站点 Generic 搜索失败：站点请求或页面解析失败"]
+    assert warnings == [f"站点 Generic 搜索失败：{expected_error}"]
 
 
 def test_search_provider_logs_site_budget_release_failure(monkeypatch):