diff --git a/app/modules/indexer/parser/mtorrent.py b/app/modules/indexer/parser/mtorrent.py index dc00292fa..0df4f83df 100644 --- a/app/modules/indexer/parser/mtorrent.py +++ b/app/modules/indexer/parser/mtorrent.py @@ -1,15 +1,14 @@ # -*- coding: utf-8 -*- import json -from typing import Any, Optional, Tuple, cast +from typing import Optional, Tuple from urllib.parse import urljoin +from app.runtime.log import logger from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.domain import site as site_rules class MTorrentSiteUserInfo(SiteParserBase): - """使用 M-Team API 解析用户信息和当前做种数据。""" - schema = SiteSchema.MTorrent request_mode = "apikey" @@ -114,7 +113,7 @@ class MTorrentSiteUserInfo(SiteParserBase): def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]: """ - 累加当前页的做种数和体积，并根据列表分页元数据继续读取。 + 解析用户做种信息 """ if not html_text: return None @@ -145,30 +144,24 @@ class MTorrentSiteUserInfo(SiteParserBase): self.seeding_size += page_seeding_size self.seeding_info.extend(page_seeding_info) - # 用户信息解析成功后已初始化分页参数，后续页数从做种列表响应中读取。 - torrent_params = cast(dict[str, Any], self._torrent_seeding_params) - page_number = int(torrent_params["pageNumber"]) - page_size = int(torrent_params["pageSize"]) + # 查询总做种数 + seeder_count = 0 try: - total_pages = int(seeding_data.get("totalPages") or 0) - except (TypeError, ValueError): - total_pages = 0 - try: - total_torrents = int(seeding_data.get("total") or 0) - except (TypeError, ValueError): - total_torrents = 0 - - if total_pages: - has_next_page = page_number < total_pages - elif total_torrents: - has_next_page = page_number * page_size < total_torrents - else: - # 旧版响应未提供总量时，满页后再请求一次确认是否结束。 - has_next_page = len(torrents) >= page_size - if not has_next_page: + result = self._get_page_content( + url=urljoin(self._base_url, "api/tracker/myPeerStatus"), + params={"uid": self.userid}, + ) + if result: + seeder_info = json.loads(result) + seeder_count = int(seeder_info.get("data", {}).get("seeder") or 0) + except Exception as e: + logger.error(f"获取做种数失败: {str(e)}") + if not seeder_count: return None - - torrent_params["pageNumber"] = page_number + 1 + if self.seeding >= seeder_count: + return None + # 还有下一页 + self._torrent_seeding_params["pageNumber"] += 1 return "" def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]: diff --git a/app/modules/indexer/parser/nexus_php.py b/app/modules/indexer/parser/nexus_php.py index 480040219..b7c8c21f2 100644 --- a/app/modules/indexer/parser/nexus_php.py +++ b/app/modules/indexer/parser/nexus_php.py @@ -220,6 +220,33 @@ class NexusPhpSiteUserInfo(SiteParserBase): return True, gold * 100 * 100 + silver * 100 + copper return False, 0.0 + def _parse_seeding_pages(self) -> None: + """ + 优先使用完整做种列表统计，列表为空或解析不完整时回退到用户详情页统计。 + """ + profile_seeding = self.seeding + profile_seeding_size = self.seeding_size + profile_seeding_info = list(self.seeding_info) + + # 用户详情页可能只显示部分做种记录，先暂存，避免与完整列表相加造成重复。 + self.seeding = 0 + self.seeding_size = 0 + self.seeding_info.clear() + try: + # SiteParserBase keeps this legacy hook untyped. + super()._parse_seeding_pages() # type: ignore[no-untyped-call] + except Exception: + self.seeding = profile_seeding + self.seeding_size = profile_seeding_size + self.seeding_info = profile_seeding_info + raise + + if (not self.seeding or not self.seeding_size) and (profile_seeding or profile_seeding_size): + self.seeding = profile_seeding + self.seeding_size = profile_seeding_size + if profile_seeding_info: + self.seeding_info = profile_seeding_info + def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]: """ 做种相关信息 @@ -259,21 +286,41 @@ class NexusPhpSiteUserInfo(SiteParserBase): page_seeding = 0 page_seeding_size = 0 page_seeding_info = [] - # 如果 table class="torrents"，则增加table[@class="torrents"] - table_class = '//table[@class="torrents"]' if html.xpath('//table[@class="torrents"]') else '' - seeding_sizes = html.xpath(f'{table_class}//tr[position()>1]/td[{size_col}]') - seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]/b/a/text()') - if not seeding_seeders: - seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]//text()') - if seeding_sizes and seeding_seeders: - page_seeding = len(seeding_sizes) - - for i in range(0, len(seeding_sizes)): - size = self.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = text_tools.parse_int(seeding_seeders[i]) - + site_domain = self._site_domain.lower().removeprefix("www.") + if site_domain in {"pterclub.net", "pterclub.com"}: + # 猫站完整做种列表按 td[2]/td[4]/td[5] 展示种子、体积和做种人数。 + pterclub_rows = html.xpath( + '//*[@id="outer"]/table/tbody/tr | //*[@id="outer"]/table/tr' + '|//table[contains(concat(" ", normalize-space(@class), " "), " torrents ")]/tbody/tr' + '|//table[contains(concat(" ", normalize-space(@class), " "), " torrents ")]/tr' + ) + for row in pterclub_rows: + columns = row.xpath('./td | ./th') + if len(columns) < 5 or not columns[1].xpath('.//a'): + continue + size = self.num_filesize(columns[3].xpath("string(.)").strip()) + if not size: + continue + seeders = text_tools.parse_int(columns[4].xpath("string(.)").strip()) + page_seeding += 1 page_seeding_size += size page_seeding_info.append([seeders, size]) + else: + # 如果 table class="torrents"，则限制在种子表中，避免解析页面其他表格。 + table_class = '//table[@class="torrents"]' if html.xpath('//table[@class="torrents"]') else '' + seeding_sizes = html.xpath(f'{table_class}//tr[position()>1]/td[{size_col}]') + seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]/b/a/text()') + if not seeding_seeders: + seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]//text()') + if seeding_sizes and seeding_seeders: + page_seeding = len(seeding_sizes) + + for i in range(0, len(seeding_sizes)): + size = self.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = text_tools.parse_int(seeding_seeders[i]) + + page_seeding_size += size + page_seeding_info.append([seeders, size]) self.seeding += page_seeding self.seeding_size += page_seeding_size @@ -345,19 +392,29 @@ class NexusPhpSiteUserInfo(SiteParserBase): # 做种体积 & 做种数 # seeding 页面获取不到的话，此处再获取一次 - seeding_sizes = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//' - 'table[tr[1][td[4 and text()="尺寸"]]]//tr[position()>1]/td[4]') - seeding_seeders = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//' - 'table[tr[1][td[5 and text()="做种者"]]]//tr[position()>1]/td[5]//text()') - tmp_seeding = len(seeding_sizes) + seeding_tables = html.xpath( + '//tr/td[normalize-space(.)="当前上传" or normalize-space(.)="当前做种"]' + '/following-sibling::td[1]//table' + ) + tmp_seeding = 0 tmp_seeding_size = 0 tmp_seeding_info = [] - for i in range(0, len(seeding_sizes)): - size = self.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = text_tools.parse_int(seeding_seeders[i]) + for seeding_table in seeding_tables: + # 猫站使用“当前做种”作为面板行名；该表第 4、5 列分别是体积和做种人数。 + seeding_rows = seeding_table.xpath('./tr | ./tbody/tr') + for seeding_row in seeding_rows: + columns = seeding_row.xpath('./td | ./th') + if len(columns) < 5: + continue + size = self.num_filesize(columns[3].xpath("string(.)").strip()) + # 无表头的做种表以有效体积列识别数据行，避免把表头计入数量。 + if not size: + continue + seeders = text_tools.parse_int(columns[4].xpath("string(.)").strip()) - tmp_seeding_size += size - tmp_seeding_info.append([seeders, size]) + tmp_seeding += 1 + tmp_seeding_size += size + tmp_seeding_info.append([seeders, size]) if not self.seeding_size: self.seeding_size = tmp_seeding_size diff --git a/tests/fixtures/architecture/mypy-baseline.json b/tests/fixtures/architecture/mypy-baseline.json index b61627e33..5372f1f49 100644 --- a/tests/fixtures/architecture/mypy-baseline.json +++ b/tests/fixtures/architecture/mypy-baseline.json @@ -1702,6 +1702,7 @@ "app/modules/indexer/parser/mtorrent.py": { "assignment": 11, "empty-body": 1, + "index": 2, "no-untyped-def": 6, "type-arg": 1 }, diff --git a/tests/test_mtorrent_parser.py b/tests/test_mtorrent_parser.py index 92dc22262..dfeaf0176 100644 --- a/tests/test_mtorrent_parser.py +++ b/tests/test_mtorrent_parser.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- import json -import pytest - from app.modules.indexer.parser.mtorrent import MTorrentSiteUserInfo @@ -19,7 +17,7 @@ def _build_parser() -> MTorrentSiteUserInfo: ) -def test_mtorrent_seeding_skips_invalid_torrent_items(): +def test_mtorrent_seeding_skips_invalid_torrent_items(monkeypatch): """ MTorrent 返回空种子对象时应跳过异常条目并只统计有效做种。 """ @@ -32,6 +30,11 @@ def test_mtorrent_seeding_skips_invalid_torrent_items(): "userid": parser.userid, } + monkeypatch.setattr( + parser, + "_get_page_content", + lambda **_: json.dumps({"data": {"seeder": 1}}), + ) html_text = json.dumps( { "code": "0", @@ -52,73 +55,3 @@ def test_mtorrent_seeding_skips_invalid_torrent_items(): assert parser.seeding == 1 assert parser.seeding_size == 1024 assert parser.seeding_info == [[3, 1024]] - - -def test_mtorrent_seeding_paginates_from_list_metadata(monkeypatch): - """应按做种列表的分页元数据读取全部种子，不依赖 tracker 私有接口。""" - parser = _build_parser() - parser.userid = "1" - parser._torrent_seeding_params = { - "pageNumber": 1, - "pageSize": 2, - "type": "SEEDING", - "userid": parser.userid, - } - monkeypatch.setattr( - parser, - "_get_page_content", - lambda **_: pytest.fail("解析做种列表时不应请求 tracker 私有接口"), - ) - - first_page = json.dumps( - { - "code": "0", - "data": { - "total": 3, - "totalPages": 2, - "data": [ - {"torrent": {"size": "1024", "source": "3"}}, - {"torrent": {"size": "2048", "source": "4"}}, - ], - }, - } - ) - second_page = json.dumps( - { - "code": "0", - "data": { - "total": 3, - "totalPages": 2, - "data": [{"torrent": {"size": "4096", "source": "5"}}], - }, - } - ) - - assert parser._parse_user_torrent_seeding_info(first_page) == "" - assert parser._torrent_seeding_params["pageNumber"] == 2 - assert parser._parse_user_torrent_seeding_info(second_page) is None - assert parser.seeding == 3 - assert parser.seeding_size == 7168 - assert parser.seeding_info == [[3, 1024], [4, 2048], [5, 4096]] - - -def test_mtorrent_seeding_uses_page_size_when_totals_are_missing(): - """接口未返回总页数时，满页后继续读取并在空页结束。""" - parser = _build_parser() - parser._torrent_seeding_params = {"pageNumber": 1, "pageSize": 2} - full_page = json.dumps( - { - "code": "0", - "data": { - "data": [ - {"torrent": {"size": "1024", "source": "3"}}, - {"torrent": {"size": "2048", "source": "4"}}, - ] - }, - } - ) - empty_page = json.dumps({"code": "0", "data": {"data": []}}) - - assert parser._parse_user_torrent_seeding_info(full_page) == "" - assert parser._torrent_seeding_params["pageNumber"] == 2 - assert parser._parse_user_torrent_seeding_info(empty_page) is None diff --git a/tests/test_nexus_php_parser.py b/tests/test_nexus_php_parser.py index fcdd635b7..156cf48f4 100644 --- a/tests/test_nexus_php_parser.py +++ b/tests/test_nexus_php_parser.py @@ -226,3 +226,82 @@ def test_nexus_php_uuid_seeding_next_page_uses_useruuid(): assert query_params["useruuid"] == ["53d25816-f7b7-4372-b58d-6f445acfe090"] assert "userid" not in query_params assert query_params["type"] == ["seeding"] + + +def test_nexus_php_pterclub_profile_parses_current_seeding_table(): + """ + 猫站用户详情页的“当前做种”表应按第 4、5 列统计做种体积和做种人数。 + """ + parser = _build_parser() + parser.userid = "123" + html_text = """ + + + + +
当前做种 + + +
详情种子一分类1.5 TB12
详情种子二分类700 GB3
+ + """ + + parser._parse_user_detail_info(html_text) + + assert parser.seeding == 2 + assert parser.seeding_size == 1.5 * 1024**4 + 700 * 1024**3 + assert parser.seeding_info == [[12, int(1.5 * 1024**4)], [3, 700 * 1024**3]] + + +def test_nexus_php_pterclub_full_seeding_list_uses_site_columns(): + """ + 猫站完整做种页即使没有表头，也应读取第 4 列体积和第 5 列做种人数。 + """ + parser = _build_parser() + parser._site_domain = "pterclub.net" + parser.userid = "123" + html_text = """ + +
+ +
详情种子一分类1.5 TB12
+

1下一页

+ + """ + + next_page = parser._parse_user_torrent_seeding_info(html_text) + + assert parser.seeding == 1 + assert parser.seeding_size == int(1.5 * 1024**4) + assert parser.seeding_info == [[12, int(1.5 * 1024**4)]] + assert next_page == "getusertorrentlist.php?userid=123&type=seeding&page=2" + + +def test_nexus_php_seeding_list_replaces_profile_fallback_without_double_counting(monkeypatch): + """ + 做种列表成功时应覆盖详情页中的部分做种统计，避免重复累加。 + """ + parser = _build_parser() + parser.userid = "123" + parser._torrent_seeding_page = "getusertorrentlistajax.php?userid=123&type=seeding" + parser._parse_user_detail_info(""" + + + +
当前做种 + +
详情种子一分类1.5 TB12
+ """) + list_html = """ + + + +
标题分类大小在做种
种子一分类500 GB7
+ """ + monkeypatch.setattr(parser, "_get_page_content", lambda **_: list_html) + + parser._parse_seeding_pages() + + assert parser.seeding == 1 + assert parser.seeding_size == 500 * 1024**3 + assert parser.seeding_info == [[7, 500 * 1024**3]]