mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-03 05:44:44 +08:00
fix(download): preserve protocol failures and isolate site updates (#7143)
* fix(download): preserve protocol failures and isolate site updates * test: cover tracker token lifecycle and record dependency facts --------- Co-authored-by: jxxghp <jxxghp@gmail.com>
This commit is contained in:
@@ -157,6 +157,10 @@ class MTorrentSiteUserInfo(SiteParserBase):
|
||||
self.seeding_size += page_seeding_size
|
||||
self.seeding_info.extend(page_seeding_info)
|
||||
|
||||
# 空页或没有有效条目的页面不再推进,汇总可能滞后于实时做种明细。
|
||||
if not page_seeding:
|
||||
return None
|
||||
|
||||
# 查询总做种数
|
||||
seeder_count = 0
|
||||
try:
|
||||
|
||||
@@ -371,16 +371,26 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
|
||||
if not DomUtils.has_child_elements(html):
|
||||
return
|
||||
|
||||
for user_node in html.xpath('//*[@data-uploader-url or @data-uploader-stats]'):
|
||||
userbars = html.xpath(
|
||||
'//*[contains(concat(" ", normalize-space(@class), " "), " site-userbar ")]'
|
||||
)
|
||||
scope = userbars[0] if userbars else html
|
||||
nodes = scope.xpath('.//*[@data-uploader-url or @data-uploader-stats]')
|
||||
# 兼容只有单个用户卡的旧页面;多用户卡且无用户栏时不能猜测当前账号。
|
||||
if not userbars:
|
||||
nodes = [node for node in nodes if self.userid and
|
||||
re.search(rf"userdetails\.php\?id={re.escape(str(self.userid))}(?:&|$)",
|
||||
node.get("data-uploader-url") or "")] or (nodes if len(nodes) == 1 else [])
|
||||
for user_node in nodes:
|
||||
self.__parse_user_identity(user_node)
|
||||
self.__parse_uploader_stats(user_node.get("data-uploader-stats"))
|
||||
|
||||
# data-uploader-stats 不包含分享率,需从 compact metric 的 class 中读取。
|
||||
self.__parse_compact_metric(html, "ratio", "ratio")
|
||||
self.__parse_compact_metric(html, "uploaded", "upload")
|
||||
self.__parse_compact_metric(html, "downloaded", "download")
|
||||
self.__parse_compact_metric(html, "bonus", "bonus")
|
||||
self.__parse_compact_metric(html, "active", "active")
|
||||
self.__parse_compact_metric(scope, "ratio", "ratio")
|
||||
self.__parse_compact_metric(scope, "uploaded", "upload")
|
||||
self.__parse_compact_metric(scope, "downloaded", "download")
|
||||
self.__parse_compact_metric(scope, "bonus", "bonus")
|
||||
self.__parse_compact_metric(scope, "active", "active")
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
@@ -221,7 +221,11 @@ class MTorrentSpider:
|
||||
统一判定搜索响应状态并投影 M-Team 种子结果。
|
||||
"""
|
||||
if res and res.status_code == 200:
|
||||
data = res.json().get('data', {})
|
||||
payload = res.json()
|
||||
if str(payload.get("code")) != "0" or not isinstance(payload.get("data"), dict):
|
||||
logger.warning(f"{self._name} 搜索失败,站点返回业务错误")
|
||||
return True, []
|
||||
data = payload["data"]
|
||||
results = data.get("data") or []
|
||||
# 分页事实只供逐页搜索判断末页,空结果仍按末页处理。
|
||||
page_number, total_pages = data.get("pageNumber"), data.get("totalPages")
|
||||
|
||||
@@ -7,13 +7,13 @@ from typing import Any, List, Optional, Tuple
|
||||
|
||||
from app.adapters.network.http import AsyncRequestUtils, RequestUtils
|
||||
from app.foundation import temporal as time_tools
|
||||
from app.foundation.singleton import SingletonClass
|
||||
from app.foundation.crypto import HashUtils
|
||||
from app.runtime.cache import cached
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
|
||||
class TNodeSpider(metaclass=SingletonClass):
|
||||
class TNodeSpider:
|
||||
"""
|
||||
TNode 页面令牌与种子搜索 API 适配器。
|
||||
"""
|
||||
@@ -47,13 +47,28 @@ class TNodeSpider(metaclass=SingletonClass):
|
||||
if indexer.get('proxy'):
|
||||
self._proxy = get_runtime_setting('PROXY')
|
||||
self._cookie = indexer.get('cookie')
|
||||
self._token_key = HashUtils.md5(repr((self._domain, self._cookie, indexer.get('ua'))))
|
||||
self._ua = indexer.get('ua')
|
||||
self._timeout = indexer.get('timeout') or 15
|
||||
|
||||
@cached(region="indexer_spider", maxsize=1, ttl=60 * 60 * 24, skip_empty=True, shared_key="get_token")
|
||||
def __get_token(self) -> Optional[str]:
|
||||
"""
|
||||
以当前站点和凭据隔离的缓存键获取同步CSRF令牌。
|
||||
"""
|
||||
return self.__cached_token(self._token_key)
|
||||
|
||||
async def __async_get_token(self) -> Optional[str]:
|
||||
"""
|
||||
以当前站点和凭据隔离的缓存键获取异步CSRF令牌。
|
||||
"""
|
||||
return await self.__async_cached_token(self._token_key)
|
||||
|
||||
@cached(region="indexer_spider", maxsize=64, ttl=60 * 60 * 24, skip_empty=True, shared_key="get_token")
|
||||
def __cached_token(self, _identity: str) -> Optional[str]:
|
||||
"""
|
||||
同步请求站点首页并提取 CSRF 令牌。
|
||||
|
||||
:param _identity: 站点和凭据摘要,用于隔离共享令牌缓存
|
||||
"""
|
||||
if not self._domain:
|
||||
return None
|
||||
@@ -63,10 +78,12 @@ class TNodeSpider(metaclass=SingletonClass):
|
||||
timeout=self._timeout).get_res(url=self._domain)
|
||||
return self.__parse_token_response(res)
|
||||
|
||||
@cached(region="indexer_spider", maxsize=1, ttl=60 * 60 * 24, skip_empty=True, shared_key="get_token")
|
||||
async def __async_get_token(self) -> Optional[str]:
|
||||
@cached(region="indexer_spider", maxsize=64, ttl=60 * 60 * 24, skip_empty=True, shared_key="get_token")
|
||||
async def __async_cached_token(self, _identity: str) -> Optional[str]:
|
||||
"""
|
||||
异步请求站点首页并提取 CSRF 令牌。
|
||||
|
||||
:param _identity: 站点和凭据摘要,用于隔离共享令牌缓存
|
||||
"""
|
||||
if not self._domain:
|
||||
return None
|
||||
|
||||
@@ -463,13 +463,12 @@ class QbittorrentModule(_DownloaderModuleBase[Qbittorrent]):
|
||||
:param hashs: 种子哈希或哈希列表
|
||||
:param tags: 需要追加的标签列表
|
||||
:param downloader: 下载器配置名称,省略时使用默认实例
|
||||
:return: 找到实例并调用标签接口后返回 True,不表示写入已经确认;找不到实例时返回 None
|
||||
:return: 客户端接受标签写入时返回 True,RPC失败时返回 False;找不到实例时返回 None
|
||||
"""
|
||||
server = self.get_instance(downloader)
|
||||
if not server:
|
||||
return None
|
||||
server.set_torrents_tag(ids=hashs, tags=tags)
|
||||
return True
|
||||
return server.set_torrents_tag(ids=hashs, tags=tags)
|
||||
|
||||
def update_torrent(
|
||||
self,
|
||||
|
||||
@@ -451,21 +451,22 @@ class Qbittorrent:
|
||||
logger.error(f"移除种子Tag出错:{str(err)}")
|
||||
return False
|
||||
|
||||
def set_torrents_tag(self, ids: Union[str, list[str]], tags: list[str]) -> None:
|
||||
def set_torrents_tag(self, ids: Union[str, list[str]], tags: list[str]) -> bool:
|
||||
"""
|
||||
向指定种子追加标签,保留原有标签。
|
||||
|
||||
未连接时直接返回,请求失败只记录日志;本方法不返回写入确认。
|
||||
|
||||
:param ids: 种子哈希或哈希列表
|
||||
:param tags: 待追加的标签列表
|
||||
:return: RPC成功完成时为True,未连接或失败时为False
|
||||
"""
|
||||
if not self.qbc:
|
||||
return
|
||||
return False
|
||||
try:
|
||||
self.qbc.torrents_add_tags(tags=tags, torrent_hashes=ids)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"设置种子Tag出错:{str(err)}")
|
||||
return False
|
||||
|
||||
def is_force_resume(self) -> Optional[bool]:
|
||||
"""
|
||||
|
||||
@@ -5,6 +5,7 @@ rTorrent 下载模块,负责实例路由、任务投影与下载文件选择
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Set, Tuple, Union, cast
|
||||
|
||||
from app.domain import torrent as torrent_rules
|
||||
from app.domain.metainfo import MetaInfo
|
||||
from app.foundation import size as size_tools
|
||||
from app.foundation import temporal as time_tools
|
||||
@@ -115,6 +116,7 @@ class RtorrentModule(_DownloaderModuleBase[Rtorrent]):
|
||||
if not torrent_from_file and not is_magnet:
|
||||
return None, None, None, "添加种子任务失败:无法读取种子文件"
|
||||
resolved_content = cast(Union[str, bytes], resolved_content)
|
||||
expected_hashes = torrent_rules.download_hashes(resolved_content)
|
||||
|
||||
# 失败提示使用种子名称(磁力链接保留原文),不拼接种子文件的二进制内容
|
||||
torrent_desc = torrent_from_file.name if torrent_from_file else resolved_content
|
||||
@@ -157,12 +159,8 @@ class RtorrentModule(_DownloaderModuleBase[Rtorrent]):
|
||||
if torrents:
|
||||
try:
|
||||
for torrent in torrents:
|
||||
# 名称与大小相等则认为是同一个种子
|
||||
if torrent.get("name") == getattr(
|
||||
torrent_from_file, "name", ""
|
||||
) and torrent.get("total_size") == getattr(
|
||||
torrent_from_file, "total_size", 0
|
||||
):
|
||||
candidate_hash = str(torrent.get("hash") or "").lower()
|
||||
if candidate_hash in expected_hashes:
|
||||
torrent_hash = cast(str, torrent.get("hash"))
|
||||
torrent_tags = [
|
||||
str(t).strip()
|
||||
@@ -208,45 +206,14 @@ class RtorrentModule(_DownloaderModuleBase[Rtorrent]):
|
||||
else:
|
||||
torrent_hash = added_torrent_hash
|
||||
if episodes:
|
||||
torrent_files = server.get_files(torrent_hash)
|
||||
if not torrent_files:
|
||||
return (
|
||||
downloader or self.get_default_config_name(),
|
||||
torrent_hash,
|
||||
torrent_layout,
|
||||
"获取种子文件失败,下载任务可能在暂停状态",
|
||||
)
|
||||
|
||||
# 不需要的文件ID
|
||||
file_ids = []
|
||||
# 需要的集清单
|
||||
sucess_epidised = set()
|
||||
try:
|
||||
for torrent_file in torrent_files:
|
||||
file_id = torrent_file["id"]
|
||||
file_name = torrent_file["name"]
|
||||
meta_info = MetaInfo(file_name)
|
||||
if not meta_info.episode_list or not set(
|
||||
meta_info.episode_list
|
||||
).issubset(episodes):
|
||||
file_ids.append(file_id)
|
||||
else:
|
||||
sucess_epidised.update(meta_info.episode_list)
|
||||
finally:
|
||||
torrent_files.clear()
|
||||
del torrent_files
|
||||
selected_episodes = list(sucess_epidised)
|
||||
if selected_episodes and file_ids:
|
||||
# 设置不需要的文件优先级为0(不下载)
|
||||
server.set_files(
|
||||
torrent_hash=torrent_hash, file_ids=file_ids, priority=0
|
||||
)
|
||||
server.start_torrents(torrent_hash)
|
||||
selected, message = self._select_episode_files(server, torrent_hash, episodes)
|
||||
if not selected:
|
||||
return None, None, None, message
|
||||
return (
|
||||
downloader or self.get_default_config_name(),
|
||||
torrent_hash,
|
||||
torrent_layout,
|
||||
f"添加下载成功,已选择集数:{selected_episodes}",
|
||||
message,
|
||||
)
|
||||
else:
|
||||
return (
|
||||
@@ -256,6 +223,36 @@ class RtorrentModule(_DownloaderModuleBase[Rtorrent]):
|
||||
"添加下载成功",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _select_episode_files(server: Rtorrent, torrent_hash: str, episodes: Set[int]) -> Tuple[bool, str]:
|
||||
"""
|
||||
对暂停添加的任务完成选集,所有RPC均成功才允许下载链确认任务。
|
||||
|
||||
:param server: 当前下载器客户端
|
||||
:param torrent_hash: 已暂停添加的任务Hash
|
||||
:param episodes: 本次允许下载的目标集数
|
||||
:return: 是否确认选集启动成功及结果说明
|
||||
"""
|
||||
torrent_files = server.get_files(torrent_hash)
|
||||
if not torrent_files:
|
||||
return False, "获取种子文件失败,下载任务保留暂停状态,请检查下载器"
|
||||
file_ids, selected_episodes = [], set()
|
||||
for torrent_file in torrent_files:
|
||||
meta = MetaInfo(torrent_file["name"])
|
||||
if meta.episode_list and set(meta.episode_list).issubset(episodes):
|
||||
selected_episodes.update(meta.episode_list)
|
||||
else:
|
||||
file_ids.append(torrent_file["id"])
|
||||
if not selected_episodes:
|
||||
return False, "未匹配到目标集数,下载任务保留暂停状态"
|
||||
if file_ids and not server.set_files(torrent_hash=torrent_hash, file_ids=file_ids, priority=0):
|
||||
return False, "设置选集优先级失败,下载任务保留暂停状态"
|
||||
if not server.start_torrents(torrent_hash):
|
||||
stopped = server.stop_torrents(torrent_hash)
|
||||
message = "启动下载失败,已请求暂停,请检查下载器确认状态" if stopped else "启动下载状态未知,请检查下载器"
|
||||
return False, message
|
||||
return True, f"添加下载成功,已选择集数:{sorted(selected_episodes)}"
|
||||
|
||||
def list_torrents(
|
||||
self,
|
||||
status: Optional[TorrentStatus] = None,
|
||||
@@ -448,6 +445,7 @@ class RtorrentModule(_DownloaderModuleBase[Rtorrent]):
|
||||
return server.delete_torrents(
|
||||
delete_file=delete_file if delete_file is not None else True,
|
||||
ids=hashs,
|
||||
path_mapper=lambda path: self.normalize_return_path(path, downloader),
|
||||
)
|
||||
|
||||
def set_torrents_tag(
|
||||
|
||||
@@ -7,7 +7,7 @@ import io
|
||||
import socket
|
||||
import traceback
|
||||
import xmlrpc.client
|
||||
from collections.abc import Buffer
|
||||
from collections.abc import Buffer, Callable
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union, cast
|
||||
from urllib.parse import urlparse
|
||||
@@ -504,16 +504,18 @@ class Rtorrent:
|
||||
logger.error(f"停止种子出错:{str(err)}")
|
||||
return False
|
||||
|
||||
def delete_torrents(self, delete_file: bool, ids: Union[str, list[str]]) -> bool:
|
||||
def delete_torrents(self, delete_file: bool, ids: Union[str, list[str]],
|
||||
*, path_mapper: Optional[Callable[[Path], str]] = None) -> bool:
|
||||
"""
|
||||
删除指定种子任务,并按选项删除下载文件。
|
||||
|
||||
rTorrent 返回的内容路径在 MoviePilot 所在文件系统中删除。文件清理失败会记录警告;
|
||||
返回 True 只说明任务删除流程完成,不保证文件全部移除。
|
||||
先将rTorrent内容路径映射到MoviePilot本地文件系统,文件清理成功后才移除任务。
|
||||
清理失败或映射为远程存储时保留任务,以便用户核对和重试。
|
||||
|
||||
:param delete_file: 是否同时删除下载文件
|
||||
:param ids: 种子哈希或哈希列表
|
||||
:return: 请求完成时返回 True;未连接或请求失败时返回 False,不代表任务已完成该动作
|
||||
:param path_mapper: 将下载器内容路径映射为宿主本地路径;省略时保留直接客户端调用合同
|
||||
:return: 文件清理及任务删除均完成时返回 True,失败时返回 False
|
||||
"""
|
||||
if not self._proxy:
|
||||
return False
|
||||
@@ -524,23 +526,23 @@ class Rtorrent:
|
||||
ids = [ids]
|
||||
for tid in ids:
|
||||
if delete_file:
|
||||
# 先获取base_path用于删除文件
|
||||
try:
|
||||
base_path = self._proxy.d.base_path(tid)
|
||||
self._proxy.d.erase(tid)
|
||||
if base_path:
|
||||
import shutil
|
||||
|
||||
path = Path(_rpc_text(base_path))
|
||||
if path.is_dir():
|
||||
shutil.rmtree(str(path), ignore_errors=True)
|
||||
elif path.is_file():
|
||||
path.unlink(missing_ok=True)
|
||||
except Exception as e:
|
||||
logger.warning(f"删除种子文件出错:{str(e)}")
|
||||
self._proxy.d.erase(tid)
|
||||
else:
|
||||
self._proxy.d.erase(tid)
|
||||
base_path = self._proxy.d.base_path(tid)
|
||||
if not base_path:
|
||||
logger.warning("下载器未返回文件路径,保留任务以便人工处理")
|
||||
return False
|
||||
path = Path(_rpc_text(base_path))
|
||||
mapped = path_mapper(path) if path_mapper else path.as_posix()
|
||||
# 云存储URI不能交给本地Path删除;下载器任务也保留以便核对。
|
||||
if ":" in mapped and not (len(mapped) > 2 and mapped[1] == ":"):
|
||||
logger.warning("下载器文件映射为远程存储,拒绝本地删除")
|
||||
return False
|
||||
path = Path(mapped)
|
||||
if path.is_dir():
|
||||
import shutil
|
||||
shutil.rmtree(path)
|
||||
elif path.exists():
|
||||
path.unlink()
|
||||
self._proxy.d.erase(tid)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"删除种子出错:{str(err)}")
|
||||
|
||||
@@ -4,6 +4,7 @@ Transmission 客户端连接与下载任务操作封装。
|
||||
|
||||
import time
|
||||
from typing import List, Optional, Sequence, Tuple, Union
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import transmission_rpc
|
||||
from transmission_rpc import Client, File, Torrent
|
||||
@@ -32,7 +33,8 @@ class Transmission:
|
||||
"trackerStats",
|
||||
"leftUntilDone", "rateDownload", "rateUpload", "recheckProgress", "rateDownload", "rateUpload",
|
||||
"peersGettingFromUs", "peersSendingToUs", "uploadRatio", "uploadedEver", "downloadedEver", "downloadDir",
|
||||
"error", "errorString", "doneDate", "queuePosition", "activityDate", "trackers"]
|
||||
"error", "errorString", "doneDate", "queuePosition", "activityDate", "trackers",
|
||||
"downloadLimit", "uploadLimit", "seedRatioLimit", "seedIdleLimit"]
|
||||
|
||||
def __init__(self, host: Optional[str] = None, port: Optional[int] = None,
|
||||
username: Optional[str] = None, password: Optional[str] = None,
|
||||
@@ -51,12 +53,16 @@ class Transmission:
|
||||
self._host: Optional[str] = None
|
||||
self._port: Optional[int] = None
|
||||
self._protocol: object
|
||||
self._path = "/transmission/rpc"
|
||||
if host and port:
|
||||
self._protocol, self._host, self._port = kwargs.get("protocol", "http"), host, port
|
||||
elif host:
|
||||
result = UrlUtils.parse_url_params(url=host)
|
||||
if result:
|
||||
self._protocol, self._host, self._port, path = result
|
||||
self._protocol, self._host, self._port, _path = result
|
||||
self._path = urlsplit(host if "://" in host else f"http://{host}").path or self._path
|
||||
if self._path == "/":
|
||||
self._path = "/transmission/rpc"
|
||||
else:
|
||||
logger.error("Transmission配置不正确!")
|
||||
return
|
||||
@@ -102,6 +108,7 @@ class Transmission:
|
||||
trt = transmission_rpc.Client(protocol=self._protocol, # noqa
|
||||
host=self._host,
|
||||
port=self._port,
|
||||
path=self._path,
|
||||
username=self._username,
|
||||
password=self._password,
|
||||
# transmission-rpc 运行时接受 Requests 的连接与读取超时二元组。
|
||||
@@ -584,14 +591,14 @@ class Transmission:
|
||||
https://github.com/trim21/transmission-rpc/blob/8eb82629492a0eeb0bb565f82c872bf9ccdcb313/transmission_rpc/client.py#L654
|
||||
|
||||
:param hash_string: 种子哈希
|
||||
:param tracker_list: 要写入的 Tracker 地址列表,原样交给客户端
|
||||
:param tracker_list: 要写入的 Tracker 地址列表,每个地址作为独立层级交给客户端
|
||||
:return: 请求完成时返回 True;未连接或请求失败时返回 False,不代表任务已完成该动作
|
||||
"""
|
||||
if not self.trc:
|
||||
return False
|
||||
try:
|
||||
self.trc.change_torrent(ids=hash_string,
|
||||
tracker_list=tracker_list)
|
||||
tracker_list=[[url] for url in tracker_list] if tracker_list is not None else None)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"修改tracker出错:{str(err)}")
|
||||
|
||||
Reference in New Issue
Block a user