Merge pull request #7298 from jxxghp/fix/issue-7231-cspt-traffic

fix(indexer): 修复修道院新版首页流量解析
This commit is contained in:
jxxghp
2026-11-04 14:36:24 +08:00
committed by GitHub
3 changed files with 171 additions and 16 deletions

View File

@@ -151,26 +151,29 @@ class NexusPhpSiteUserInfo(SiteParserBase):
def _parse_user_traffic_info(self, html_text: str) -> None:
"""
解析用户流量信息。
解析用户流量信息,优先读取修道院概览指标,缺失字段沿用传统页面规则。
"""
html_text = self._prepare_html_text(html_text)
upload_match = re.search(r"[^总]上[传傳]量?[::_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text,
re.IGNORECASE)
self.upload = self.num_filesize(upload_match.group(1).strip()) if upload_match else 0
download_match = re.search(r"[^总子影力]下[载載]量?[::_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text,
re.IGNORECASE)
self.download = self.num_filesize(download_match.group(1).strip()) if download_match else 0
ratio_match = re.search(r"分享率[::_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+)", html_text)
# 计算分享率
calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
# 优先使用页面上的分享率
self.ratio = text_tools.parse_float(ratio_match.group(1)) if (
ratio_match and ratio_match.group(1).strip()) else calc_ratio
leeching_match = re.search(r"(Torrents leeching|下载中)[\u4E00-\u9FA5\D\s]+(\d+)[\s\S]+<", html_text)
self.leeching = text_tools.parse_int(leeching_match.group(2)) if leeching_match and leeching_match.group(
2).strip() else 0
html = etree.HTML(html_text)
try:
metrics = self._traffic_metric_values(html)
upload_match = re.search(r"[^总]上[传傳]量?[::_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text,
re.IGNORECASE)
upload_text = metrics.get("上传量") or (upload_match.group(1).strip() if upload_match else "")
self.upload = self.num_filesize(upload_text)
download_match = re.search(r"[^总子影力]下[载載]量?[::_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", html_text,
re.IGNORECASE)
download_text = metrics.get("下载量") or (download_match.group(1).strip() if download_match else "")
self.download = self.num_filesize(download_text)
ratio_match = re.search(r"分享率[::_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+)", html_text)
ratio_text = metrics.get("分享率") or (ratio_match.group(1).strip() if ratio_match else "")
# 优先使用页面上的分享率,否则按最终解析到的流量计算。
calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3)
self.ratio = text_tools.parse_float(ratio_text) if ratio_text else calc_ratio
leeching_match = re.search(r"(Torrents leeching|下载中)[\u4E00-\u9FA5\D\s]+(\d+)[\s\S]+<", html_text)
self.leeching = text_tools.parse_int(leeching_match.group(2)) if leeching_match and leeching_match.group(
2).strip() else 0
has_ucoin, self.bonus = self._parse_ucoin(html)
if has_ucoin:
return
@@ -221,6 +224,25 @@ class NexusPhpSiteUserInfo(SiteParserBase):
if html is not None:
del html
@staticmethod
def _traffic_metric_values(html: Any) -> dict[str, str]:
"""
读取修道院概览中的账户流量指标,忽略字段名下的中文副标题。
:param html: 首页节点;响应为空时可为 None
:return: 指标名称到显示值的映射;缺失字段不加入结果
"""
metrics: dict[str, str] = {}
if not DomUtils.has_child_elements(html):
return metrics
for metric in html.xpath(
'//dl[contains(concat(" ", normalize-space(@class), " "), " cultivation-metrics ")]/div[dt and dd]'
):
label = metric.xpath('normalize-space(dt/text()[1])')
if label in {"上传量", "下载量", "分享率"}:
metrics[label] = metric.xpath('normalize-space(dd)')
return metrics
@staticmethod
def _parse_ucoin(html: etree._Element) -> tuple[bool, float]:
"""

38
tests/fixtures/cspt_homepage.html vendored Normal file
View File

@@ -0,0 +1,38 @@
<section class="cspt-card cspt-card-profile">
<div class="cspt-info-line cspt-info-line-account">
<div class="cspt-info-toolbar">欢迎回来, <span class="nowrap"><a href="https://example.com/userdetails.php?id=xxxx" class="User_Name"><b>xxxx</b></a> <span class="nexus-custom-title">(练气期)</span><img src="https://img3.pixhost.cc/images/5068/760548593__30.jpg" title="爱莉希雅" class="nexus-username-medal preview" style="max-height: 11px;max-width: 11px;margin-left: 2pt"><img src="https://img3.pixhost.cc/images/5068/760554581_-1.png" title="爱门" class="nexus-username-medal preview" style="max-height: 11px;max-width: 11px;margin-left: 2pt"><img src="https://img3.pixhost.cc/images/5068/760565870_da.png" title="莉可丽丝" class="nexus-username-medal preview" style="max-height: 11px;max-width: 11px;margin-left: 2pt"></span> <span class="cspt-chip"><a href="logout.php">退出</a></span>
<span class="cspt-chip"><a href="usercp.php">控制面板</a></span>
<span class="cspt-chip"><a href="torrents.php?inclbookmarked=1&amp;allsec=1&amp;incldead=0">收藏</a></span>
<font class="color_bonus">法钱 </font><span class="cspt-chip"><a href="mybonus.php">使用</a></span>: 1,585.8 <span class="cspt-chip"><a href="attendance.php" class="">签到已得95, 补签卡: 0</a></span> <span class="cspt-chip"><a href="medal.php">勋章</a></span>
<span class="cspt-chip"><a href="task.php">任务</a></span>
<font class="color_invite">邀请 </font><span class="cspt-chip"><a href="invite.php?id=xxxx">发送</a></span>: 2(0)</div> <div class="cspt-info-stats"><span class="cspt-kv"><font class="color_active">当前活动:</font> <img class="arrowup" alt="Torrents seeding" title="当前做种" src="pic/trans.gif">2 <img class="arrowdown" alt="Torrents leeching" title="当前下载" src="pic/trans.gif">0&nbsp;&nbsp;</span><span class="cspt-kv"><font class="color_connectable">可连接:</font><b><font color="green">是</font></b></span><span class="cspt-kv"><font class="color_slots">连接数:</font>无限制</span><span class="cspt-kv"><font class="color_bonus">认领: </font> [<a href="claim.php?uid=xxxx">0/1000</a>]</span></div> </div>
<section class="cultivation-dashboard cultivation-reveal cultivation-visible" aria-labelledby="cultivation-overview-title" data-sancai="human">
<div class="cultivation-section-heading">
<div><h2 id="cultivation-overview-title">我的概览</h2><p>修持在日常,贡献有记录。</p></div>
<a href="userdetails.php?id=xxxx">个人资料与记录</a>
</div>
<div class="cultivation-identity-line">
<span>境界 <strong>人材(User)</strong></span>
<span class="cultivation-badge" data-stage="mortal">从芸芸黔首中显露出利用价值的人,可被仙门体系吸纳;耗材:已经“有用”,但这种价值更多是对上修而言——可用、可炼、可牺牲。</span> <span>道籍 <strong>普通道籍</strong></span> </div>
<dl class="cultivation-metrics">
<div><dt>上传量 <span>功行</span></dt><dd>105.06 GB</dd></div>
<div><dt>下载量 <span>所耗资粮</span></dt><dd>10.00 GB</dd></div>
<div><dt>分享率 <span>取予有度</span></dt><dd>10.506</dd></div>
<div><dt>做种时长 <span>修持</span></dt><dd>42天18:11:03</dd></div>
<div><dt>魔力值 <span>法钱</span></dt><dd>1,585.8</dd></div>
<div><dt>做种积分 <span>成长积累</span></dt><dd>935</dd></div>
</dl>
</section>
<section class="cultivation-workspace cultivation-reveal" aria-labelledby="cultivation-library-title" data-sancai="earth">
<div class="cultivation-section-heading">
<div><h2 id="cultivation-library-title">资源检索 <span>道藏</span></h2><p>找到所需的版本,让每一份资源持续可达。</p></div>
<a class="cultivation-action" href="upload.php">发布资源</a> </div>
<form class="cultivation-search" method="get" action="torrents.php">
<label for="cultivation-search-input">搜索资源 <span>检索道藏</span></label>
<div><input id="cultivation-search-input" name="search" type="search" placeholder="资源名称或关键词"><button type="submit">搜索</button></div>
</form>
<div class="cultivation-workspace-links"><a href="torrents.php">浏览资源</a><a href="torrents.php?inclbookmarked=1&amp;allsec=1&amp;incldead=0">我的收藏 · 藏经</a><a href="forums.php">社区交流</a><a href="rules.php">站点规则 · 法度</a></div>
</section>
<div class="cspt-info-icons"><span class="medium">
<a href="friends.php"><img class="buddylist" alt="Buddylist" title="社交名单" src="pic/trans.gif"></a> <a href="getrss.php"><img class="rss" alt="RSS" title="获取RSS" src="pic/trans.gif"></a><br><a href="messages.php"><img class="inbox" src="pic/trans.gif" alt="inbox" title="收件箱&nbsp;(无新短讯)"></a> 0 <a href="messages.php?action=viewmailbox&amp;box=-1"><img class="sentbox" alt="sentbox" title="发件箱" src="pic/trans.gif"></a> 0
</span></div> </section>

View File

@@ -25,6 +25,101 @@ def _build_parser() -> NexusPhpSiteUserInfo:
)
def test_nexus_php_cspt_homepage_traffic() -> None:
"""
回放 Issue #7231 的脱敏首页,指标副标题不能阻断上传量、下载量和分享率解析。
"""
homepage = (Path(__file__).parent / "fixtures" / "cspt_homepage.html").read_text(encoding="utf-8")
parser = _build_parser()
parser._parse_site_page(homepage)
parser._parse_user_base_info(homepage)
assert parser.upload == round(105.06 * 1024 ** 3)
assert parser.download == 10 * 1024 ** 3
assert parser.ratio == 10.506
assert parser.userid == "xxxx"
assert parser.username == "xxxx"
parser._parse_user_detail_info(
'<table><tr><td>上传量</td><td>1 GB</td></tr>'
'<tr><td>下载量</td><td>2 GB</td></tr><tr><td>分享率</td><td>0.5</td></tr></table>'
)
assert parser.upload == round(105.06 * 1024 ** 3)
assert parser.download == 10 * 1024 ** 3
assert parser.ratio == 10.506
@pytest.mark.parametrize("metrics, legacy, expected_upload, expected_download, expected_ratio", [
('<div><dt>上传量 <span>功行</span></dt><dd>8 <span>GB</span></dd></div>'
'<div><dt>下载量 <span>所耗资粮</span></dt><dd>2 GB</dd></div>'
'<div><dt>分享率 <span>取予有度</span></dt><dd>4.001</dd></div>',
"上传量: 1 GB 下载量: 1 GB 分享率: 1", 8 * 1024 ** 3, 2 * 1024 ** 3, 4.001),
('<div><dt>上传量 <span>功行</span></dt><dd>8 GB</dd></div>'
'<div><dt>下载量 <span>所耗资粮</span></dt><dd>2 GB</dd></div>',
"", 8 * 1024 ** 3, 2 * 1024 ** 3, 4.0),
('<div><dt>上传量 <span>功行</span></dt><dd>8 GB</dd></div>',
"下载量: 2 GB", 8 * 1024 ** 3, 2 * 1024 ** 3, 4.0),
('<div><dt>上传量 <span>功行</span></dt><dd>0 B</dd></div>'
'<div><dt>下载量 <span>所耗资粮</span></dt><dd>0 B</dd></div>'
'<div><dt>分享率 <span>取予有度</span></dt><dd>0</dd></div>',
"上传量: 1 GB 下载量: 1 GB 分享率: 1", 0, 0, 0.0),
('<div><dt>上传量 <span>功行</span></dt><dd>8 GB</dd></div>'
'<div><dt>下载量 <span>所耗资粮</span></dt><dd>0 B</dd></div>',
"", 8 * 1024 ** 3, 0, 0.0),
('<div><dt>上传量 <span>功行</span></dt><dd></dd></div>',
"上传量: 1 GB 下载量: 2 GB 分享率: 0.501", 1024 ** 3, 2 * 1024 ** 3, 0.501),
], ids=["displayed-ratio", "calculated-ratio", "partial-metrics", "zero-values", "zero-download", "empty-value"])
def test_nexus_php_cspt_metrics_prefer_displayed_values(
metrics: str, legacy: str, expected_upload: int, expected_download: int, expected_ratio: float
) -> None:
"""
概览指标优先取页面显示值,缺失或为空时沿用传统字段及分享率计算。
"""
parser = _build_parser()
parser._parse_user_traffic_info(
f'<html><body>{legacy}<dl class="extra cultivation-metrics">{metrics}</dl></body></html>'
)
assert parser.upload == expected_upload
assert parser.download == expected_download
assert parser.ratio == expected_ratio
@pytest.mark.parametrize("class_name, upload_label", [
("other-metrics", "上传量"),
("cultivation-metrics-other", "上传量"),
("cultivation-metrics", "实际上传量"),
], ids=["unrelated-list", "class-substring", "actual-upload"])
def test_nexus_php_cspt_metrics_ignore_unrelated_fields(class_name: str, upload_label: str) -> None:
"""
不把其他列表或实际流量字段识别为账户总流量。
"""
parser = _build_parser()
parser._parse_user_traffic_info(
f'<html><body><dl class="{class_name}">'
f'<div><dt>{upload_label} <span>功行</span></dt><dd>8 GB</dd></div>'
'</dl></body></html>'
)
assert (parser.upload, parser.download, parser.ratio) == (0, 0, 0.0)
@pytest.mark.parametrize("html_text", ["", "<html></html>"])
def test_nexus_php_empty_traffic_page(html_text: str) -> None:
"""
空响应或空页面没有概览指标时,保持零流量且不抛出异常。
"""
parser = _build_parser()
parser._parse_user_traffic_info(html_text)
assert (parser.upload, parser.download, parser.ratio) == (0, 0, 0.0)
def test_nexus_php_depthstudio_traffic_falls_back_to_details():
"""
首页解析失败后从详情表格补齐流量,不依赖卡片类名或实际流量。