From 99ef7e508b829e81cedaf55b3aea0387baca2515 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Mon, 12 Oct 2026 08:36:19 +0800 Subject: [PATCH] fix(indexer): support custom seeding and Unit3D bonus pages --- app/modules/indexer/parser/__init__.py | 5 + app/modules/indexer/parser/nexus_php.py | 139 ++++++++++++++---- app/modules/indexer/parser/unit3d.py | 5 + docs/refactor/optimization-checklist.md | 2 + .../architecture/complexity-baseline.json | 1 - tests/test_nexus_php_parser.py | 134 +++++++++++++++++ tests/test_unit3d_parser.py | 17 +++ 7 files changed, 270 insertions(+), 33 deletions(-) diff --git a/app/modules/indexer/parser/__init__.py b/app/modules/indexer/parser/__init__.py index a7ace57ea..c634b1545 100644 --- a/app/modules/indexer/parser/__init__.py +++ b/app/modules/indexer/parser/__init__.py @@ -417,6 +417,11 @@ class SiteParserBase(metaclass=ABCMeta): proxies=proxies, headers=req_headers, ) + if res is None: + logger.warning(f"{self._site_name} 用户数据请求未收到响应") + elif res.status_code != 200: + # 不输出带用户标识或令牌的 URL，保留状态码用于诊断入口失效和访问限制。 + logger.warning(f"{self._site_name} 用户数据请求返回 HTTP {res.status_code}") if res is not None and res.status_code in (200, 500, 403): if req_headers and "application/json" in str(req_headers.get("Accept")): try: diff --git a/app/modules/indexer/parser/nexus_php.py b/app/modules/indexer/parser/nexus_php.py index d923e4c6d..ac635fec7 100644 --- a/app/modules/indexer/parser/nexus_php.py +++ b/app/modules/indexer/parser/nexus_php.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import re -from typing import Optional +from typing import Any, Optional from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from lxml import etree @@ -18,8 +18,11 @@ class NexusPhpSiteUserInfo(SiteParserBase): """ schema = SiteSchema.NexusPhp + _summary_seeding: Optional[int] = None + _summary_seeding_size: Optional[int] = None def _parse_site_page(self, html_text: str): + """识别用户详情入口，并为定制站点选择已知的做种接口。""" html_text = self._prepare_html_text(html_text) # NexusPHP v1.10+ 使用 UUID 标识用户，详情链接参数为 uuid @@ -55,6 +58,24 @@ class NexusPhpSiteUserInfo(SiteParserBase): """ if not self._has_valid_userid(): return "" + site_domain = self._site_domain.lower().removeprefix("www.") + if site_domain in {"pterclub.net", "pterclub.com"}: + # 猫站的列表入口要求完整的浏览器导航请求头，仅作用于该站用户数据。 + headers = { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "same-origin", + } + # 基类的历史字段仅推断为 None，保持现有字典请求头运行时契约。 + self._torrent_seeding_headers = headers # type: ignore[assignment] + self._user_detail_headers = dict(headers) # type: ignore[assignment] + return f"getusertorrentlist.php?userid={self.userid}&type=seeding" + if site_domain == "keepfrds.com": + return f"torrents.php?option-torrents=3&userid={self.userid}" + if site_domain == "pt.sjtu.edu.cn": + return f"viewtorrentsdetail.php?id={self.userid}" user_id_param = "useruuid" if self._is_uuid(self.userid) else "userid" return f"getusertorrentlistajax.php?{user_id_param}={self.userid}&type=seeding" @@ -247,6 +268,30 @@ class NexusPhpSiteUserInfo(SiteParserBase): if profile_seeding_info: self.seeding_info = profile_seeding_info + # 定制站点明细可能只返回部分记录，明确的汇总值优先，明细仍用于做种人数分布。 + if self._summary_seeding is not None: + self.seeding = self._summary_seeding + if self._summary_seeding_size is not None: + self.seeding_size = self._summary_seeding_size + + def _parse_seeding_summary(self, html: Any) -> None: + """读取已知定制站点的明确汇总，不把缺失体积推算为真实值。""" + if self._site_domain.lower().removeprefix("www.") not in { + "pterclub.net", "pterclub.com", "keepfrds.com", "pt.sjtu.edu.cn" + }: + return + page_text = " ".join(html.xpath('//text()[not(ancestor::script) and not(ancestor::style)]')) + count_match = re.search(r"当前做种\s*[:：]?\s*([\d,]+)", page_text) + if not count_match and self._site_domain.lower().removeprefix("www.") in {"pterclub.net", "pterclub.com"}: + count_match = re.search(r"正在做种的种子\s*[（(]\s*共\s*([\d,]+)\s*个", page_text) + size_match = re.search(r"做种大小\s*[:：]?\s*([\d,.]+\s*[KMGTPI]i?B)", page_text, re.I) + if count_match: + self._summary_seeding = text_tools.parse_int(count_match.group(1).replace(",", "")) + self.seeding = self._summary_seeding + if size_match: + self._summary_seeding_size = self.num_filesize(size_match.group(1)) + self.seeding_size = self._summary_seeding_size + def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]: """ 做种相关信息 @@ -254,11 +299,20 @@ class NexusPhpSiteUserInfo(SiteParserBase): :param multi_page: 是否多页数据 :return: 下页地址 """ + if not html_text: + logger.warning(f"{self._site_name} 做种接口未返回内容，请检查站点接口和访问权限") + return None html = etree.HTML(str(html_text).replace(r'\/', '/')) try: if not DomUtils.has_child_elements(html): return None + self._parse_seeding_summary(html) + if self._site_domain.lower().removeprefix("www.") == "pt.sjtu.edu.cn": + if self._summary_seeding is None: + logger.warning(f"{self._site_name} 做种汇总为空或无法解析，请检查站点接口和访问权限") + return None + # 首页存在扩展链接，使用扩展链接 seeding_url_text = html.xpath('//a[contains(@href,"torrents.php") ' 'and contains(@href,"seeding")]/@href') @@ -266,22 +320,7 @@ class NexusPhpSiteUserInfo(SiteParserBase): self._torrent_seeding_page = seeding_url_text[0].strip() return self._torrent_seeding_page - size_col = 3 - seeders_col = 4 - # 搜索size列 - size_col_xpath = '//tr[position()=1]/' \ - 'td[(img[@class="size"] and img[@alt="size"])' \ - ' or (text() = "大小")' \ - ' or (a/img[@class="size" and @alt="size"])]' - if html.xpath(size_col_xpath): - size_col = len(html.xpath(f'{size_col_xpath}/preceding-sibling::td')) + 1 - # 搜索seeders列 - seeders_col_xpath = '//tr[position()=1]/' \ - 'td[(img[@class="seeders"] and img[@alt="seeders"])' \ - ' or (text() = "在做种")' \ - ' or (a/img[@class="seeders" and @alt="seeders"])]' - if html.xpath(seeders_col_xpath): - seeders_col = len(html.xpath(f'{seeders_col_xpath}/preceding-sibling::td')) + 1 + size_col, seeders_col = self._seeding_columns(html) page_seeding = 0 page_seeding_size = 0 @@ -322,31 +361,63 @@ class NexusPhpSiteUserInfo(SiteParserBase): page_seeding_size += size page_seeding_info.append([seeders, size]) + if not page_seeding and not multi_page and self._summary_seeding is None: + logger.warning(f"{self._site_name} 做种列表为空或无法解析，请检查站点接口和访问权限") self.seeding += page_seeding self.seeding_size += page_seeding_size self.seeding_info.extend(page_seeding_info) - # 是否存在下页数据 - next_page = None - next_page_text = html.xpath( - '//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href') - - # 防止识别到详情页 - while next_page_text: - next_page = next_page_text.pop().strip() - if not next_page.startswith('details.php'): - break - next_page = None - - # fix up page url - if next_page: - next_page = self._fixup_next_page_url(next_page, self.userid) + next_page = self._seeding_next_page(html) finally: if html is not None: del html return next_page + def _seeding_columns(self, html: Any) -> tuple[int, int]: + """优先使用表头标记，朋友的无标记模板使用已确认的固定列位。""" + size_col = 3 + seeders_col = 4 + # 搜索size列 + size_col_xpath = '//tr[position()=1]/' \ + 'td[(img[@class="size"] and img[@alt="size"])' \ + ' or (text() = "大小")' \ + ' or (a/img[@class="size" and @alt="size"])]' + if html.xpath(size_col_xpath): + size_col = len(html.xpath(f'{size_col_xpath}/preceding-sibling::td')) + 1 + # 搜索seeders列 + seeders_col_xpath = '//tr[position()=1]/' \ + 'td[(img[@class="seeders"] and img[@alt="seeders"])' \ + ' or (text() = "在做种")' \ + ' or (a/img[@class="seeders" and @alt="seeders"])]' + if html.xpath(seeders_col_xpath): + seeders_col = len(html.xpath(f'{seeders_col_xpath}/preceding-sibling::td')) + 1 + + if self._site_domain.lower().removeprefix("www.") == "keepfrds.com": + # 朋友没有列标记，第四列为做种时长，第五、六列才是体积和做种人数。 + size_col, seeders_col = 5, 6 + + return size_col, seeders_col + + def _seeding_next_page(self, html: Any) -> Optional[str]: + """筛除详情链接并补齐下一页的用户标识。""" + # 是否存在下页数据 + next_page = None + next_page_text = html.xpath( + '//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href') + + # 防止识别到详情页 + while next_page_text: + next_page = next_page_text.pop().strip() + if not next_page.startswith('details.php'): + break + next_page = None + + # fix up page url + if next_page: + next_page = self._fixup_next_page_url(next_page, self.userid) + return next_page + def _fixup_next_page_url(self, next_page: str, userid: Optional[str]) -> Optional[str]: """ 修正做种下一页地址，无法补齐用户 ID 时停止翻页。 @@ -378,6 +449,7 @@ class NexusPhpSiteUserInfo(SiteParserBase): if not DomUtils.has_child_elements(html): return + self._parse_seeding_summary(html) self._get_user_level(html) self._fixup_traffic_info(html) @@ -526,6 +598,7 @@ class NexusPhpSiteUserInfo(SiteParserBase): self.user_level = user_level # type: ignore[assignment] def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]: + """解析未读消息链接及下一页入口。""" html = etree.HTML(html_text) try: if not DomUtils.has_child_elements(html): @@ -545,6 +618,7 @@ class NexusPhpSiteUserInfo(SiteParserBase): return next_page def _parse_message_content(self, html_text): + """解析消息标题、时间和正文。""" html = etree.HTML(html_text) try: if not DomUtils.has_child_elements(html): @@ -576,6 +650,7 @@ class NexusPhpSiteUserInfo(SiteParserBase): return message_head_text, message_date_text, message_content_text def _fixup_traffic_info(self, html): + """从详情页补充首页缺失的魔力值。""" # fixup bonus if not self.bonus: bonus_text = html.xpath('//tr/td[text()="魔力值" or text()="猫粮"]/following-sibling::td[1]/text()') diff --git a/app/modules/indexer/parser/unit3d.py b/app/modules/indexer/parser/unit3d.py index db1b4ed10..c89f919d4 100644 --- a/app/modules/indexer/parser/unit3d.py +++ b/app/modules/indexer/parser/unit3d.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re from typing import Optional +from urllib.parse import urlsplit from lxml import etree @@ -31,6 +32,10 @@ class Unit3dSiteUserInfo(SiteParserBase): self._user_detail_page = f"/users/{self.username}" tmps = html.xpath('//a[contains(@href, "bonus/earnings")]') + if not tmps and self.username: + tmps = html.xpath('//a[contains(@href, "/users/") and contains(@href, "/bonus/")]') + tmps = [link for link in tmps + if urlsplit(link.get("href")).path.startswith(f"/users/{self.username}/bonus/")] if tmps: bonus_text = tmps[0].xpath("string(.)") bonus_match = re.search(r"([\d,.]+)", bonus_text) diff --git a/docs/refactor/optimization-checklist.md b/docs/refactor/optimization-checklist.md index bf6e6fc4a..a080a36a3 100644 --- a/docs/refactor/optimization-checklist.md +++ b/docs/refactor/optimization-checklist.md @@ -26,6 +26,8 @@ R2 原复杂度 v1/v2 实际使用源码行跨度，只覆盖 API、Application 范围、初始基线与后续建议见 [代码门禁说明](../code-quality.md)。 2026-09-30 音乐真实样本回归统一了文件名曲序入口，删除 `MetaMusic.apply_path_context` 重复的曲序回填分支，C901 从 16 降至阈值 15 以内，已同步固化低水位。 +2026-10-12 站点用户数据兼容修复将 NexusPHP 做种列探测与翻页解析拆为独立方法， +`_parse_user_torrent_seeding_info` 的 C901 从 16 降至阈值 15 以内，基线仅移除此超限项。 `concurrency.py` 扫描完整宿主源码，按 canonical import/alias、TaskGroup、可证明的 loop/executor 来源和词法 owner 聚合数量。新增 owner、数量增长以及静态质量事实下降后 未刷新低水位都会阻断 CI；覆盖率只要求 Application 与 Domain 达到固定 80%。 diff --git a/tests/fixtures/architecture/complexity-baseline.json b/tests/fixtures/architecture/complexity-baseline.json index 9d9e2826f..667359a4d 100644 --- a/tests/fixtures/architecture/complexity-baseline.json +++ b/tests/fixtures/architecture/complexity-baseline.json @@ -97,7 +97,6 @@ "app/modules/filemanager/transhandler.py:TransHandler.execute_transfer_plan": 21, "app/modules/filter/__init__.py:FilterModule.__match_rule": 24, "app/modules/imdb/__init__.py:ImdbModule._to_media_info": 17, - "app/modules/indexer/parser/nexus_php.py:NexusPhpSiteUserInfo._parse_user_torrent_seeding_info": 16, "app/modules/indexer/spider/__init__.py:SiteSpider.__get_search_url": 24, "app/modules/jellyfin/jellyfin.py:Jellyfin.get_tv_episodes": 18, "app/modules/qbittorrent/__init__.py:QbittorrentModule.download": 24, diff --git a/tests/test_nexus_php_parser.py b/tests/test_nexus_php_parser.py index 156cf48f4..d707ce659 100644 --- a/tests/test_nexus_php_parser.py +++ b/tests/test_nexus_php_parser.py @@ -305,3 +305,137 @@ def test_nexus_php_seeding_list_replaces_profile_fallback_without_double_countin assert parser.seeding == 1 assert parser.seeding_size == 500 * 1024**3 assert parser.seeding_info == [[7, 500 * 1024**3]] + + +def test_custom_seeding_endpoints_and_pterclub_navigation_headers(): + """已确认的定制站点直接使用真实入口，猫站请求头不影响其他站点。""" + endpoints = { + "pterclub.net": "getusertorrentlist.php?userid=123&type=seeding", + "pterclub.com": "getusertorrentlist.php?userid=123&type=seeding", + "keepfrds.com": "torrents.php?option-torrents=3&userid=123", + "pt.sjtu.edu.cn": "viewtorrentsdetail.php?id=123", + } + for domain, endpoint in endpoints.items(): + parser = _build_parser() + parser._site_domain = domain + parser._parse_site_page('user') + assert parser._torrent_seeding_page == endpoint + if domain.startswith("pterclub"): + assert parser._torrent_seeding_headers["Accept-Language"] + assert parser._torrent_seeding_headers["Accept"].startswith("text/html") + assert parser._torrent_seeding_headers["Sec-Fetch-Dest"] == "document" + assert parser._torrent_seeding_headers["Sec-Fetch-Mode"] == "navigate" + assert parser._torrent_seeding_headers["Sec-Fetch-Site"] == "same-origin" + assert parser._user_detail_headers == parser._torrent_seeding_headers + else: + assert parser._torrent_seeding_headers is None + + +def test_keepfrds_unmarked_columns_preserve_profile_count(monkeypatch): + """朋友列表含做种时长，汇总数量与可见明细不一致时保留汇总和真实明细体积。""" + parser = _build_parser() + parser._site_domain = "keepfrds.com" + parser._parse_site_page('user') + parser._parse_user_detail_info('
当前做种7
') + page = ''' + + +
类型标题魔力时长体积人数
电影test1231月7天7.29GiB4
''' + monkeypatch.setattr(parser, "_get_page_content", lambda **_kwargs: page) + parser._parse_seeding_pages() + assert parser.seeding == 7 + assert parser.seeding_size == parser.num_filesize("7.29GiB") + assert parser.seeding_info == [[4, parser.num_filesize("7.29GiB")]] + + +def test_pterclub_profile_size_overrides_partial_paginated_list(monkeypatch): + """猫站汇总体积优先于分页明细累加，翻页继续沿用导航头且不重复汇总。""" + parser = _build_parser() + parser._site_domain = "pterclub.net" + parser._parse_site_page('user') + parser._parse_user_detail_info('
做种大小5.522 TB
') + page = '''

正在做种的种子 (共 470 个)

+ +
类型标题时间大小人数
电影test昨天1GB4
''' + calls = [] + + def get_page(**kwargs): + """回放分页内容并记录请求参数。""" + calls.append(kwargs) + if len(calls) == 1: + return page + '下一页' + return page + + monkeypatch.setattr(parser, "_get_page_content", get_page) + parser._parse_seeding_pages() + assert parser.seeding == 470 + assert parser.seeding_size == parser.num_filesize("5.522 TB") + assert len(parser.seeding_info) == 2 + assert len(calls) == 2 + assert calls[1]["headers"]["Sec-Fetch-Mode"] == "navigate" + assert "userid=123" in calls[1]["url"] + + +def test_putao_summary_only_does_not_follow_global_torrent_list(monkeypatch): + """葡萄只取明确做种计数，不推算体积或扫描全站列表。""" + parser = _build_parser() + parser._site_domain = "pt.sjtu.edu.cn" + parser._parse_site_page('user') + calls = [] + + def get_page(**kwargs): + """回放葡萄异步汇总响应。""" + calls.append(kwargs) + return '
当前做种 21 完成种子数 2 未完成种子数 138
' + + monkeypatch.setattr(parser, "_get_page_content", get_page) + parser._parse_seeding_pages() + assert len(calls) == 1 + assert calls[0]["url"].endswith("viewtorrentsdetail.php?id=123") + assert parser.seeding == 21 + assert parser.seeding_size == 0 + assert parser.seeding_info == [] + + +def test_nexus_empty_seeding_response_logs_warning(monkeypatch): + """接口失败为空时应提示可诊断警告，并保留详情页汇总。""" + parser = _build_parser() + parser.userid = "123" + parser.seeding = 7 + parser.seeding_size = 1024 + warnings = [] + monkeypatch.setattr("app.modules.indexer.parser.nexus_php.logger.warning", warnings.append) + monkeypatch.setattr(parser, "_get_page_content", lambda **_kwargs: "") + parser._parse_seeding_pages() + assert parser.seeding == 7 + assert parser.seeding_size == 1024 + assert any("未返回内容" in warning for warning in warnings) + + +def test_pterclub_headers_reach_http_transport_and_failures_are_visible(monkeypatch): + """导航头实际传递给统一 HTTP 客户端，404/403 警告不暴露 Cookie 或用户标识。""" + from types import SimpleNamespace + + parser = _build_parser() + parser._site_domain = "pterclub.net" + parser._parse_site_page('user') + calls, warnings = [], [] + + def get_response(**kwargs): + """回放 HTTP 失败响应，验证传输层参数。""" + calls.append(kwargs) + return SimpleNamespace(status_code=404, text="File not found.") + + monkeypatch.setattr(parser._request_utils, "get_res", get_response) + monkeypatch.setattr("app.modules.indexer.parser.logger.warning", warnings.append) + assert parser._get_page_content(url="https://pterclub.net/getusertorrentlist.php?userid=123", + headers=parser._torrent_seeding_headers) == "" + assert calls[0]["headers"]["Sec-Fetch-Site"] == "same-origin" + assert any("HTTP 404" in warning for warning in warnings) + assert all("userid" not in warning for warning in warnings) + monkeypatch.setattr(parser._request_utils, "get_res", + lambda **_kwargs: SimpleNamespace(status_code=403, text="Forbidden")) + monkeypatch.setattr("app.modules.indexer.parser.RequestUtils.get_decoded_html_content", + lambda *_args: "Forbidden") + parser._get_page_content(url="https://pterclub.net/", headers=parser._torrent_seeding_headers) + assert any("HTTP 403" in warning for warning in warnings) diff --git a/tests/test_unit3d_parser.py b/tests/test_unit3d_parser.py index 4a6382d27..496932e61 100644 --- a/tests/test_unit3d_parser.py +++ b/tests/test_unit3d_parser.py @@ -34,3 +34,20 @@ def test_unit3d_user_name_is_used_as_valid_userid() -> None: assert parser._user_detail_page == "/users/monica" assert parser._torrent_seeding_page.startswith("/users/monica/active?") assert parser.bonus == 1234.5 + + +def test_unit3d_user_bonus_transaction_link(): + """魔力值交易入口仅解析当前用户，忽略其他用户链接。""" + parser = _build_parser() + parser._parse_user_base_info('''设置 + 999 + 33,035,774''') + assert parser.bonus == 33035774 + + +def test_unit3d_earnings_link_has_priority(): + """保留标准 earnings 入口的解析优先级。""" + parser = _build_parser() + parser._parse_user_base_info(USER_BASE_HTML + + '999') + assert parser.bonus == 1234.5