diff --git a/app/modules/indexer/spider/__init__.py b/app/modules/indexer/spider/__init__.py index e5f0f4df0..745dc1a81 100644 --- a/app/modules/indexer/spider/__init__.py +++ b/app/modules/indexer/spider/__init__.py @@ -163,6 +163,7 @@ class SiteSpider: self.result_num = int(result_num or self.default_result_num()) self._timeout = int(indexer.get('timeout') or 15) self.page = page + self._result_offset = 0 if self.domain and not str(self.domain).endswith("/"): self.domain = self.domain + "/" self.ua = indexer.get('ua') or get_runtime_setting('USER_AGENT') @@ -201,6 +202,7 @@ class SiteSpider: """ 获取搜索URL """ + self._result_offset = 0 # 种子搜索相对路径 paths = self.search.get('paths', []) torrentspath = "" @@ -320,6 +322,8 @@ class SiteSpider: } # 无额外参数 searchurl = self.domain + str(torrentspath).format(**inputs_dict) + # 没有页码入口的搜索一次返回完整列表，需在解析前按原始行分页。 + self._result_offset = 0 if "{page}" in str(torrentspath) else int(self.page or 0) * self.result_num # 列表浏览 else: @@ -1117,6 +1121,15 @@ class SiteSpider: ) ) + def __slice_search_results(self, html_doc: Any, html_text: str) -> str: + """无服务端页码时先保留当前页原始行，确保 Python 与 Rust 使用相同窗口。""" + if not self._result_offset: + return html_text + rows = html_doc(self.list.get('selector', '')) + PyQuery(rows[:self._result_offset]).remove() + PyQuery(rows[self._result_offset + self.result_num:]).remove() + return str(html_doc) + def parse(self, html_text: str) -> List[dict]: """ 解析页面或 RSS 索引，统一使用 HTML 选择器语义。 @@ -1134,6 +1147,7 @@ class SiteSpider: self.error_detail = "返回登录或权限提示页" logger.warn(f"错误：{self.indexername} 返回登录或权限提示页") return [] + html_text = self.__slice_search_results(status_doc, html_text) except Exception as err: self.is_error = True self.error_detail = f"页面解析失败：{err}" diff --git a/docs/smart-search.md b/docs/smart-search.md index 4931ebf6b..aa14f4dc9 100644 --- a/docs/smart-search.md +++ b/docs/smart-search.md @@ -23,6 +23,8 @@ 媒体匹配、季集过滤、排序、下载选择和订阅完成判断都复用原有流程；分页只决定何时提交哪些集。 +MiKan 等搜索入口一次返回完整列表的通用站点，按站点单页数量在响应中切出对应页，默认每页 100 条。每次续页仍需重新请求搜索页面，可翻页范围以站点实际返回的结果为限。 + ## 中断与恢复 - 进度保存在 `searchsession` 表，每个订阅搜索任务一行，由队列任务租约和版本号防止迟到的执行者覆盖。只保存仍覆盖剩余缺集的候选，不保存 Cookie、API Key、passkey 等凭据；下载链接能重建的用当前凭据重建，否则重新请求原页取得。同一候选的原页连续 3 次请求失败时，本轮放弃该候选。 diff --git a/tests/test_indexer_spider_pagination.py b/tests/test_indexer_spider_pagination.py new file mode 100644 index 000000000..2b11a0bd2 --- /dev/null +++ b/tests/test_indexer_spider_pagination.py @@ -0,0 +1,132 @@ +"""通用站点没有服务端页码时，从完整响应中提取互不重叠的搜索页。""" + +import asyncio +from unittest.mock import AsyncMock, Mock +from urllib.parse import parse_qs, urlparse + +import pytest +from requests import Response + +from app.modules.indexer import IndexerModule +from app.modules.indexer import spider as spider_module +from app.modules.indexer.spider import SiteSpider + + +@pytest.fixture +def indexer(): + """按 MiKan 的搜索路径与表格选择器构造离线配置。""" + return { + "id": "mikanani", + "name": "MiKan", + "domain": "https://mikan.example/", + "search": {"paths": [{"path": "Home/Search?searchstr={keyword}"}]}, + "torrents": { + "list": {"selector": "div.episode-table > table > tbody > tr.js-search-results-row"}, + "fields": { + "title": {"selector": "td:nth-child(2) > a.magnet-link-wrap"}, + "download": { + "selector": "td:nth-child(2) > a.js-magnet", + "attribute": "data-clipboard-text", + }, + "size": {"selector": "td:nth-child(3)"}, + }, + }, + } + + +@pytest.fixture(params=["python", "rust"]) +def parser_engine(request, monkeypatch): + """分别验证 Python 回退和真实 Rust 引擎，避免开关掩盖原生路径。""" + if request.param == "python": + monkeypatch.setattr(spider_module.rust_accel, "parse_indexer_torrents", Mock(return_value=None)) + return + native = pytest.importorskip("moviepilot_rust") + + def parse_native(**kwargs): + """将同一窗口交给已安装的 Rust 解析入口。""" + return native.parse_indexer_torrents_fast( + kwargs["html_text"], kwargs["domain"], kwargs["list_config"], + kwargs["fields"], kwargs["category"], kwargs["result_num"], + ) + + monkeypatch.setattr(spider_module.rust_accel, "parse_indexer_torrents", parse_native) + + +def _response(count: int = 250) -> Response: + """生成一次返回多页结果的响应，每条标题携带可断言的原始行号。""" + rows = "".join( + f'{i}' + f'番剧 {i:03d}' + f'' + '1 GB' + for i in range(count) + ) + response = Response() + response.status_code = 200 + response.encoding = "utf-8" + response._content = ( + f'搜索结果
{rows}' + '
' + ).encode("utf-8") + return response + + +@pytest.mark.usefixtures("parser_engine") +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("page, start, stop", [(0, 0, 100), (1, 100, 200), (2, 200, 250), (3, 250, 250)]) +def test_unpaged_keyword_search_returns_requested_window(indexer, monkeypatch, asynchronous, page, start, stop): + """相同响应在两种请求与解析路径中产生独立页、尾页和成功空页。""" + requester = AsyncMock(return_value=_response()) if asynchronous else Mock(return_value=_response()) + transport = spider_module.AsyncRequestUtils if asynchronous else spider_module.RequestUtils + monkeypatch.setattr(transport, "get_res", requester) + spider = SiteSpider(indexer, keyword="魔女 & SP", page=page) + + rows = asyncio.run(spider.async_get_torrents()) if asynchronous else spider.get_torrents() + + assert [row["title"] for row in rows] == [f"番剧 {i:03d}" for i in range(start, stop)] + assert not spider.is_error + query = parse_qs(urlparse(requester.call_args.args[0]).query) + assert query == {"searchstr": ["魔女 & SP"]} + assert IndexerModule.get_search_page_size(indexer, keyword="魔女 & SP") == 100 + + +@pytest.mark.usefixtures("parser_engine") +def test_unpaged_search_uses_configured_page_size(indexer, monkeypatch): + """本地窗口使用站点 result_num，与搜索编排的单页容量一致。""" + indexer["result_num"] = 50 + monkeypatch.setattr(spider_module.RequestUtils, "get_res", Mock(return_value=_response())) + + rows = SiteSpider(indexer, keyword="魔女", page=2).get_torrents() + + assert [row["title"] for row in rows] == [f"番剧 {i:03d}" for i in range(100, 150)] + assert IndexerModule.get_search_page_size(indexer, keyword="魔女") == 50 + + +@pytest.mark.usefixtures("parser_engine") +@pytest.mark.parametrize("search", [ + {"paths": [{"path": "search?q={keyword}&page={page}"}]}, + {"paths": [{"path": "search"}], "params": {"q": "{keyword}"}}, +]) +def test_server_paged_search_keeps_response_first_window(indexer, monkeypatch, search): + """路径占位符和通用查询参数已传递页码时，不再对返回页应用偏移。""" + indexer["search"] = search + requester = Mock(return_value=_response()) + monkeypatch.setattr(spider_module.RequestUtils, "get_res", requester) + + rows = SiteSpider(indexer, keyword="魔女", page=2).get_torrents() + + assert [row["title"] for row in rows] == [f"番剧 {i:03d}" for i in range(100)] + assert parse_qs(urlparse(requester.call_args.args[0]).query)["page"] == ["2"] + + +@pytest.mark.usefixtures("parser_engine") +def test_browse_with_server_page_keeps_response_first_window(indexer, monkeypatch): + """无关键词浏览仍按浏览路径的服务端页码处理响应。""" + indexer["browse"] = {"path": "Home/Classic/{page}", "start": 1} + requester = Mock(return_value=_response()) + monkeypatch.setattr(spider_module.RequestUtils, "get_res", requester) + + rows = SiteSpider(indexer, page=2).get_torrents() + + assert len(rows) == 100 and rows[0]["title"] == "番剧 000" + assert urlparse(requester.call_args.args[0]).path == "/Home/Classic/3"