nas/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-11-04 06:15:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(indexer): support custom seeding and Unit3D bonus pages

Browse Source
This commit is contained in:
jxxghp
2026-10-12 08:36:19 +08:00
parent bd1e6eb899
commit 99ef7e508b
7 changed files with 270 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/modules/indexer/parser/__init__.py
View File

@@ -417,6 +417,11 @@ class SiteParserBase(metaclass=ABCMeta):
proxies=proxies,
headers=req_headers,
)
if res is None:
logger.warning(f"{self._site_name} 用户数据请求未收到响应")
elif res.status_code != 200:
# 不输出带用户标识或令牌的 URL，保留状态码用于诊断入口失效和访问限制。
logger.warning(f"{self._site_name} 用户数据请求返回 HTTP {res.status_code}")
if res is not None and res.status_code in (200, 500, 403):
if req_headers and "application/json" in str(req_headers.get("Accept")):
try:

139
app/modules/indexer/parser/nexus_php.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
import re
from typing import Optional
from typing import Any, Optional
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from lxml import etree
@@ -18,8 +18,11 @@ class NexusPhpSiteUserInfo(SiteParserBase):
"""
schema = SiteSchema.NexusPhp
_summary_seeding: Optional[int] = None
_summary_seeding_size: Optional[int] = None
def _parse_site_page(self, html_text: str):
"""识别用户详情入口，并为定制站点选择已知的做种接口。"""
html_text = self._prepare_html_text(html_text)
# NexusPHP v1.10+ 使用 UUID 标识用户，详情链接参数为 uuid
@@ -55,6 +58,24 @@ class NexusPhpSiteUserInfo(SiteParserBase):
"""
if not self._has_valid_userid():
return ""
site_domain = self._site_domain.lower().removeprefix("www.")
if site_domain in {"pterclub.net", "pterclub.com"}:
# 猫站的列表入口要求完整的浏览器导航请求头，仅作用于该站用户数据。
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "same-origin",
}
# 基类的历史字段仅推断为 None，保持现有字典请求头运行时契约。
self._torrent_seeding_headers = headers # type: ignore[assignment]
self._user_detail_headers = dict(headers) # type: ignore[assignment]
return f"getusertorrentlist.php?userid={self.userid}&type=seeding"
if site_domain == "keepfrds.com":
return f"torrents.php?option-torrents=3&userid={self.userid}"
if site_domain == "pt.sjtu.edu.cn":
return f"viewtorrentsdetail.php?id={self.userid}"
user_id_param = "useruuid" if self._is_uuid(self.userid) else "userid"
return f"getusertorrentlistajax.php?{user_id_param}={self.userid}&type=seeding"
@@ -247,6 +268,30 @@ class NexusPhpSiteUserInfo(SiteParserBase):
if profile_seeding_info:
self.seeding_info = profile_seeding_info
# 定制站点明细可能只返回部分记录，明确的汇总值优先，明细仍用于做种人数分布。
if self._summary_seeding is not None:
self.seeding = self._summary_seeding
if self._summary_seeding_size is not None:
self.seeding_size = self._summary_seeding_size
def _parse_seeding_summary(self, html: Any) -> None:
"""读取已知定制站点的明确汇总，不把缺失体积推算为真实值。"""
if self._site_domain.lower().removeprefix("www.") not in {
"pterclub.net", "pterclub.com", "keepfrds.com", "pt.sjtu.edu.cn"
}:
return
page_text = " ".join(html.xpath('//text()[not(ancestor::script) and not(ancestor::style)]'))
count_match = re.search(r"当前做种\s*[:：]?\s*([\d,]+)", page_text)
if not count_match and self._site_domain.lower().removeprefix("www.") in {"pterclub.net", "pterclub.com"}:
count_match = re.search(r"正在做种的种子\s*[（(]\s*共\s*([\d,]+)\s*个", page_text)
size_match = re.search(r"做种大小\s*[:：]?\s*([\d,.]+\s*[KMGTPI]i?B)", page_text, re.I)
if count_match:
self._summary_seeding = text_tools.parse_int(count_match.group(1).replace(",", ""))
self.seeding = self._summary_seeding
if size_match:
self._summary_seeding_size = self.num_filesize(size_match.group(1))
self.seeding_size = self._summary_seeding_size
def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]:
"""
做种相关信息
@@ -254,11 +299,20 @@ class NexusPhpSiteUserInfo(SiteParserBase):
:param multi_page: 是否多页数据
:return: 下页地址
"""
if not html_text:
logger.warning(f"{self._site_name} 做种接口未返回内容，请检查站点接口和访问权限")
return None
html = etree.HTML(str(html_text).replace(r'\/', '/'))
try:
if not DomUtils.has_child_elements(html):
return None
self._parse_seeding_summary(html)
if self._site_domain.lower().removeprefix("www.") == "pt.sjtu.edu.cn":
if self._summary_seeding is None:
logger.warning(f"{self._site_name} 做种汇总为空或无法解析，请检查站点接口和访问权限")
return None
# 首页存在扩展链接，使用扩展链接
seeding_url_text = html.xpath('//a[contains(@href,"torrents.php") '
'and contains(@href,"seeding")]/@href')
@@ -266,22 +320,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
self._torrent_seeding_page = seeding_url_text[0].strip()
return self._torrent_seeding_page
size_col = 3
seeders_col = 4
# 搜索size列
size_col_xpath = '//tr[position()=1]/' \
'td[(img[@class="size"] and img[@alt="size"])' \
' or (text() = "大小")' \
' or (a/img[@class="size" and @alt="size"])]'
if html.xpath(size_col_xpath):
size_col = len(html.xpath(f'{size_col_xpath}/preceding-sibling::td')) + 1
# 搜索seeders列
seeders_col_xpath = '//tr[position()=1]/' \
'td[(img[@class="seeders"] and img[@alt="seeders"])' \
' or (text() = "在做种")' \
' or (a/img[@class="seeders" and @alt="seeders"])]'
if html.xpath(seeders_col_xpath):
seeders_col = len(html.xpath(f'{seeders_col_xpath}/preceding-sibling::td')) + 1
size_col, seeders_col = self._seeding_columns(html)
page_seeding = 0
page_seeding_size = 0
@@ -322,31 +361,63 @@ class NexusPhpSiteUserInfo(SiteParserBase):
page_seeding_size += size
page_seeding_info.append([seeders, size])
if not page_seeding and not multi_page and self._summary_seeding is None:
logger.warning(f"{self._site_name} 做种列表为空或无法解析，请检查站点接口和访问权限")
self.seeding += page_seeding
self.seeding_size += page_seeding_size
self.seeding_info.extend(page_seeding_info)
# 是否存在下页数据
next_page = None
next_page_text = html.xpath(
'//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href')
# 防止识别到详情页
while next_page_text:
next_page = next_page_text.pop().strip()
if not next_page.startswith('details.php'):
break
next_page = None
# fix up page url
if next_page:
next_page = self._fixup_next_page_url(next_page, self.userid)
next_page = self._seeding_next_page(html)
finally:
if html is not None:
del html
return next_page
def _seeding_columns(self, html: Any) -> tuple[int, int]:
"""优先使用表头标记，朋友的无标记模板使用已确认的固定列位。"""
size_col = 3
seeders_col = 4
# 搜索size列
size_col_xpath = '//tr[position()=1]/' \
'td[(img[@class="size"] and img[@alt="size"])' \
' or (text() = "大小")' \
' or (a/img[@class="size" and @alt="size"])]'
if html.xpath(size_col_xpath):
size_col = len(html.xpath(f'{size_col_xpath}/preceding-sibling::td')) + 1
# 搜索seeders列
seeders_col_xpath = '//tr[position()=1]/' \
'td[(img[@class="seeders"] and img[@alt="seeders"])' \
' or (text() = "在做种")' \
' or (a/img[@class="seeders" and @alt="seeders"])]'
if html.xpath(seeders_col_xpath):
seeders_col = len(html.xpath(f'{seeders_col_xpath}/preceding-sibling::td')) + 1
if self._site_domain.lower().removeprefix("www.") == "keepfrds.com":
# 朋友没有列标记，第四列为做种时长，第五、六列才是体积和做种人数。
size_col, seeders_col = 5, 6
return size_col, seeders_col
def _seeding_next_page(self, html: Any) -> Optional[str]:
"""筛除详情链接并补齐下一页的用户标识。"""
# 是否存在下页数据
next_page = None
next_page_text = html.xpath(
'//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href')
# 防止识别到详情页
while next_page_text:
next_page = next_page_text.pop().strip()
if not next_page.startswith('details.php'):
break
next_page = None
# fix up page url
if next_page:
next_page = self._fixup_next_page_url(next_page, self.userid)
return next_page
def _fixup_next_page_url(self, next_page: str, userid: Optional[str]) -> Optional[str]:
"""
修正做种下一页地址，无法补齐用户 ID 时停止翻页。
@@ -378,6 +449,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
if not DomUtils.has_child_elements(html):
return
self._parse_seeding_summary(html)
self._get_user_level(html)
self._fixup_traffic_info(html)
@@ -526,6 +598,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
self.user_level = user_level # type: ignore[assignment]
def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]:
"""解析未读消息链接及下一页入口。"""
html = etree.HTML(html_text)
try:
if not DomUtils.has_child_elements(html):
@@ -545,6 +618,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
return next_page
def _parse_message_content(self, html_text):
"""解析消息标题、时间和正文。"""
html = etree.HTML(html_text)
try:
if not DomUtils.has_child_elements(html):
@@ -576,6 +650,7 @@ class NexusPhpSiteUserInfo(SiteParserBase):
return message_head_text, message_date_text, message_content_text
def _fixup_traffic_info(self, html):
"""从详情页补充首页缺失的魔力值。"""
# fixup bonus
if not self.bonus:
bonus_text = html.xpath('//tr/td[text()="魔力值" or text()="猫粮"]/following-sibling::td[1]/text()')

5
app/modules/indexer/parser/unit3d.py
View File

@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
import re
from typing import Optional
from urllib.parse import urlsplit
from lxml import etree
@@ -31,6 +32,10 @@ class Unit3dSiteUserInfo(SiteParserBase):
self._user_detail_page = f"/users/{self.username}"
tmps = html.xpath('//a[contains(@href, "bonus/earnings")]')
if not tmps and self.username:
tmps = html.xpath('//a[contains(@href, "/users/") and contains(@href, "/bonus/")]')
tmps = [link for link in tmps
if urlsplit(link.get("href")).path.startswith(f"/users/{self.username}/bonus/")]
if tmps:
bonus_text = tmps[0].xpath("string(.)")
bonus_match = re.search(r"([\d,.]+)", bonus_text)

2
docs/refactor/optimization-checklist.md
View File

@@ -26,6 +26,8 @@ R2 原复杂度 v1/v2 实际使用源码行跨度，只覆盖 API、Application
范围、初始基线与后续建议见 [代码门禁说明](../code-quality.md)。
2026-09-30 音乐真实样本回归统一了文件名曲序入口，删除 `MetaMusic.apply_path_context`
重复的曲序回填分支，C901 从 16 降至阈值 15 以内，已同步固化低水位。
2026-10-12 站点用户数据兼容修复将 NexusPHP 做种列探测与翻页解析拆为独立方法，
`_parse_user_torrent_seeding_info` 的 C901 从 16 降至阈值 15 以内，基线仅移除此超限项。
`concurrency.py` 扫描完整宿主源码，按 canonical import/alias、TaskGroup、可证明的
loop/executor 来源和词法 owner 聚合数量。新增 owner、数量增长以及静态质量事实下降后
未刷新低水位都会阻断 CI；覆盖率只要求 Application 与 Domain 达到固定 80%。

1
tests/fixtures/architecture/complexity-baseline.json vendored
View File

@@ -97,7 +97,6 @@
"app/modules/filemanager/transhandler.py:TransHandler.execute_transfer_plan": 21,
"app/modules/filter/__init__.py:FilterModule.__match_rule": 24,
"app/modules/imdb/__init__.py:ImdbModule._to_media_info": 17,
"app/modules/indexer/parser/nexus_php.py:NexusPhpSiteUserInfo._parse_user_torrent_seeding_info": 16,
"app/modules/indexer/spider/__init__.py:SiteSpider.__get_search_url": 24,
"app/modules/jellyfin/jellyfin.py:Jellyfin.get_tv_episodes": 18,
"app/modules/qbittorrent/__init__.py:QbittorrentModule.download": 24,

134
tests/test_nexus_php_parser.py
View File

@@ -305,3 +305,137 @@ def test_nexus_php_seeding_list_replaces_profile_fallback_without_double_countin
assert parser.seeding == 1
assert parser.seeding_size == 500 * 1024**3
assert parser.seeding_info == [[7, 500 * 1024**3]]
def test_custom_seeding_endpoints_and_pterclub_navigation_headers():
"""已确认的定制站点直接使用真实入口，猫站请求头不影响其他站点。"""
endpoints = {
"pterclub.net": "getusertorrentlist.php?userid=123&type=seeding",
"pterclub.com": "getusertorrentlist.php?userid=123&type=seeding",
"keepfrds.com": "torrents.php?option-torrents=3&userid=123",
"pt.sjtu.edu.cn": "viewtorrentsdetail.php?id=123",
}
for domain, endpoint in endpoints.items():
parser = _build_parser()
parser._site_domain = domain
parser._parse_site_page('<a href="userdetails.php?id=123">user</a>')
assert parser._torrent_seeding_page == endpoint
if domain.startswith("pterclub"):
assert parser._torrent_seeding_headers["Accept-Language"]
assert parser._torrent_seeding_headers["Accept"].startswith("text/html")
assert parser._torrent_seeding_headers["Sec-Fetch-Dest"] == "document"
assert parser._torrent_seeding_headers["Sec-Fetch-Mode"] == "navigate"
assert parser._torrent_seeding_headers["Sec-Fetch-Site"] == "same-origin"
assert parser._user_detail_headers == parser._torrent_seeding_headers
else:
assert parser._torrent_seeding_headers is None
def test_keepfrds_unmarked_columns_preserve_profile_count(monkeypatch):
"""朋友列表含做种时长，汇总数量与可见明细不一致时保留汇总和真实明细体积。"""
parser = _build_parser()
parser._site_domain = "keepfrds.com"
parser._parse_site_page('<a href="userdetails.php?id=123">user</a>')
parser._parse_user_detail_info('<table><tr><td>当前做种</td><td><a>7</a></td></tr></table>')
page = '''<table class="torrents"><tr><td>类型</td><td>标题</td><td>魔力</td>
<td>时长</td><td>体积</td><td>人数</td></tr>
<tr><td>电影</td><td><a>test</a></td><td>123</td><td>1月7天</td>
<td>7.29GiB</td><td>4</td></tr></table>'''
monkeypatch.setattr(parser, "_get_page_content", lambda **_kwargs: page)
parser._parse_seeding_pages()
assert parser.seeding == 7
assert parser.seeding_size == parser.num_filesize("7.29GiB")
assert parser.seeding_info == [[4, parser.num_filesize("7.29GiB")]]
def test_pterclub_profile_size_overrides_partial_paginated_list(monkeypatch):
"""猫站汇总体积优先于分页明细累加，翻页继续沿用导航头且不重复汇总。"""
parser = _build_parser()
parser._site_domain = "pterclub.net"
parser._parse_site_page('<a href="userdetails.php?id=123">user</a>')
parser._parse_user_detail_info('<table><tr><td>做种大小</td><td>5.522 TB</td></tr></table>')
page = '''<h1>正在做种的种子 (共 470 个)</h1><table class="torrents">
<tr><td>类型</td><td>标题</td><td>时间</td><td>大小</td><td>人数</td></tr>
<tr><td>电影</td><td><a>test</a></td><td>昨天</td><td>1GB</td><td>4</td></tr></table>'''
calls = []
def get_page(**kwargs):
"""回放分页内容并记录请求参数。"""
calls.append(kwargs)
if len(calls) == 1:
return page + '<a href="getusertorrentlist.php?page=1">下一页</a>'
return page
monkeypatch.setattr(parser, "_get_page_content", get_page)
parser._parse_seeding_pages()
assert parser.seeding == 470
assert parser.seeding_size == parser.num_filesize("5.522 TB")
assert len(parser.seeding_info) == 2
assert len(calls) == 2
assert calls[1]["headers"]["Sec-Fetch-Mode"] == "navigate"
assert "userid=123" in calls[1]["url"]
def test_putao_summary_only_does_not_follow_global_torrent_list(monkeypatch):
"""葡萄只取明确做种计数，不推算体积或扫描全站列表。"""
parser = _build_parser()
parser._site_domain = "pt.sjtu.edu.cn"
parser._parse_site_page('<a href="userdetails.php?id=123">user</a>')
calls = []
def get_page(**kwargs):
"""回放葡萄异步汇总响应。"""
calls.append(kwargs)
return '<div>当前做种 21 完成种子数 2 未完成种子数 138</div>'
monkeypatch.setattr(parser, "_get_page_content", get_page)
parser._parse_seeding_pages()
assert len(calls) == 1
assert calls[0]["url"].endswith("viewtorrentsdetail.php?id=123")
assert parser.seeding == 21
assert parser.seeding_size == 0
assert parser.seeding_info == []
def test_nexus_empty_seeding_response_logs_warning(monkeypatch):
"""接口失败为空时应提示可诊断警告，并保留详情页汇总。"""
parser = _build_parser()
parser.userid = "123"
parser.seeding = 7
parser.seeding_size = 1024
warnings = []
monkeypatch.setattr("app.modules.indexer.parser.nexus_php.logger.warning", warnings.append)
monkeypatch.setattr(parser, "_get_page_content", lambda **_kwargs: "")
parser._parse_seeding_pages()
assert parser.seeding == 7
assert parser.seeding_size == 1024
assert any("未返回内容" in warning for warning in warnings)
def test_pterclub_headers_reach_http_transport_and_failures_are_visible(monkeypatch):
"""导航头实际传递给统一 HTTP 客户端，404/403 警告不暴露 Cookie 或用户标识。"""
from types import SimpleNamespace
parser = _build_parser()
parser._site_domain = "pterclub.net"
parser._parse_site_page('<a href="userdetails.php?id=123">user</a>')
calls, warnings = [], []
def get_response(**kwargs):
"""回放 HTTP 失败响应，验证传输层参数。"""
calls.append(kwargs)
return SimpleNamespace(status_code=404, text="File not found.")
monkeypatch.setattr(parser._request_utils, "get_res", get_response)
monkeypatch.setattr("app.modules.indexer.parser.logger.warning", warnings.append)
assert parser._get_page_content(url="https://pterclub.net/getusertorrentlist.php?userid=123",
headers=parser._torrent_seeding_headers) == ""
assert calls[0]["headers"]["Sec-Fetch-Site"] == "same-origin"
assert any("HTTP 404" in warning for warning in warnings)
assert all("userid" not in warning for warning in warnings)
monkeypatch.setattr(parser._request_utils, "get_res",
lambda **_kwargs: SimpleNamespace(status_code=403, text="Forbidden"))
monkeypatch.setattr("app.modules.indexer.parser.RequestUtils.get_decoded_html_content",
lambda *_args: "Forbidden")
parser._get_page_content(url="https://pterclub.net/", headers=parser._torrent_seeding_headers)
assert any("HTTP 403" in warning for warning in warnings)

17
tests/test_unit3d_parser.py
View File

@@ -34,3 +34,20 @@ def test_unit3d_user_name_is_used_as_valid_userid() -> None:
assert parser._user_detail_page == "/users/monica"
assert parser._torrent_seeding_page.startswith("/users/monica/active?")
assert parser.bonus == 1234.5
def test_unit3d_user_bonus_transaction_link():
"""魔力值交易入口仅解析当前用户，忽略其他用户链接。"""
parser = _build_parser()
parser._parse_user_base_info('''<a href="/users/monica/settings">设置</a>
<a href="/users/other/bonus/transactions/create">999</a>
<a href="https://monikadesign.uk/users/monica/bonus/transactions/create">33,035,774</a>''')
assert parser.bonus == 33035774
def test_unit3d_earnings_link_has_priority():
"""保留标准 earnings 入口的解析优先级。"""
parser = _build_parser()
parser._parse_user_base_info(USER_BASE_HTML +
'<a href="/users/monica/bonus/transactions/create">999</a>')
assert parser.bonus == 1234.5