From 378944aeba0e1e7a4471c27ffc8ab0d69dbed7e7 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sat, 3 Oct 2026 23:47:04 +0800 Subject: [PATCH] fix(indexer): support XML feeds in site search --- app/modules/indexer/spider/__init__.py | 8 +- tests/test_indexer_spider_rss.py | 116 +++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 tests/test_indexer_spider_rss.py diff --git a/app/modules/indexer/spider/__init__.py b/app/modules/indexer/spider/__init__.py index e3f58f3dc..e5f0f4df0 100644 --- a/app/modules/indexer/spider/__init__.py +++ b/app/modules/indexer/spider/__init__.py @@ -1119,14 +1119,16 @@ class SiteSpider: def parse(self, html_text: str) -> List[dict]: """ - 解析整个页面 + 解析页面或 RSS 索引，统一使用 HTML 选择器语义。 """ if not html_text: self.is_error = True return [] + # 响应已经解码，XML 编码声明会让 lxml 拒绝 Unicode；标签大小写与 Rust HTML 解析保持一致。 + html_text = re.sub(r"^[\s\ufeff]*<\?xml\b[^>]*\?>", "", html_text, count=1) try: - status_doc = PyQuery(html_text) + status_doc = PyQuery(html_text, parser="html") if self.__is_login_or_permission_page(status_doc): self.is_error = True self.error_detail = "返回登录或权限提示页" @@ -1169,7 +1171,7 @@ class SiteSpider: html_doc = None try: # 解析站点文本对象 - html_doc = PyQuery(html_text) + html_doc = PyQuery(html_text, parser="html") # 种子筛选器 torrents_selector = self.list.get('selector', '') rows = html_doc(torrents_selector) diff --git a/tests/test_indexer_spider_rss.py b/tests/test_indexer_spider_rss.py new file mode 100644 index 000000000..5a137b13e --- /dev/null +++ b/tests/test_indexer_spider_rss.py @@ -0,0 +1,116 @@ +"""RSS 索引在 HTTP 解码后应与网页索引共用同步、异步和原生解析入口。""" + +import asyncio +from unittest.mock import AsyncMock, Mock +from urllib.parse import parse_qs, urlparse + +import pytest +from requests import Response + +from app.modules.indexer import spider as spider_module +from app.modules.indexer.spider import SiteSpider + +RSS_BODY = """索引 +示例番剧 01 & SP [1080p] +Sat, 03 Oct 2026 10:20:30 +0800 +详情介绍

]]> +2026-10-03T10:20:30+08:00 + +示例番剧 02 [1080p] +2026-10-03T11:20:30+08:00 + +""" + + +@pytest.fixture +def rss_indexer(): + """构造与资源包约定一致的 RSS 选择器，避免依赖本机资源扩展。""" + return { + "id": "rss-example", + "name": "RSS 示例", + "domain": "https://example.com/", + "search": {"paths": [{"path": "rss.xml"}], "params": {"q": "{keyword}"}}, + "torrents": { + "list": {"selector": "item"}, + "fields": { + "title": {"selector": "title"}, + "download": {"selector": "enclosure", "attribute": "url"}, + "size": {"selector": "enclosure", "attribute": "length"}, + "date_added": { + "selector": "torrent pubdate", + "filters": [ + {"name": "re_search", "args": [r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", 0]}, + {"name": "replace", "args": ["T", " "]}, + ], + }, + }, + }, + } + + +@pytest.fixture(params=["python", "rust"]) +def parser_engine(request, monkeypatch): + """显式选择真实 Python 或 Rust 解析器，原生依赖缺失时只跳过原生路径。""" + if request.param == "python": + monkeypatch.setattr(spider_module.rust_accel, "parse_indexer_torrents", Mock(return_value=None)) + return + native = pytest.importorskip("moviepilot_rust") + + def parse_native(**kwargs): + """直接调用已安装的 Rust 引擎，避免配置开关让原生测试落到 Python。""" + return native.parse_indexer_torrents_fast( + kwargs["html_text"], kwargs["domain"], kwargs["list_config"], + kwargs["fields"], kwargs["category"], kwargs["result_num"], + ) + + monkeypatch.setattr(spider_module.rust_accel, "parse_indexer_torrents", parse_native) + + +@pytest.mark.usefixtures("parser_engine") +@pytest.mark.parametrize("declaration", [ + "", + '', + "\ufeff ", +]) +def test_rss_indexer_preserves_decoded_text_and_item_fields(rss_indexer, declaration): + """已解码内容不再使用 XML 声明二次解码，混合大小写日期和多条种子均完整保留。""" + spider = SiteSpider(rss_indexer) + rows = spider.parse(declaration + RSS_BODY) + + assert not spider.is_error + assert [(row["title"], row["enclosure"], row["size"], row["pubdate"]) for row in rows] == [ + ("示例番剧 01 & SP [1080p]", "https://example.com/one.torrent", 123456789, "2026-10-03 10:20:30"), + ("示例番剧 02 [1080p]", "https://example.com/two.torrent", 987654321, "2026-10-03 11:20:30"), + ] + + +@pytest.mark.usefixtures("parser_engine") +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_rss_indexer_http_paths_decode_xml_response(rss_indexer, monkeypatch, asynchronous): + """真实响应解码之后，同步和异步请求都能解析公开 RSS 并正确编码搜索关键词。""" + response = Response() + response.status_code = 200 + response.headers["Content-Type"] = "application/rss+xml; charset=utf-8" + response.encoding = "utf-8" + response._content = ('' + RSS_BODY).encode("utf-8") + requester = AsyncMock(return_value=response) if asynchronous else Mock(return_value=response) + transport = spider_module.AsyncRequestUtils if asynchronous else spider_module.RequestUtils + monkeypatch.setattr(transport, "get_res", requester) + keyword = "示例 & SP" + spider = SiteSpider(rss_indexer, keyword=keyword) + + rows = asyncio.run(spider.async_get_torrents()) if asynchronous else spider.get_torrents() + + assert len(rows) == 2 + assert rows[0]["title"] == "示例番剧 01 & SP [1080p]" + assert not spider.is_error + query = parse_qs(urlparse(requester.call_args.args[0]).query) + assert query["q"] == [keyword] + + +@pytest.mark.usefixtures("parser_engine") +def test_empty_rss_channel_is_valid(rss_indexer): + """公开接口没有匹配资源时，空 channel 应作为成功的空结果。""" + spider = SiteSpider(rss_indexer) + assert spider.parse('') == [] + assert not spider.is_error