mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-02 13:28:32 +08:00
fix(download): preserve protocol failures and isolate site updates (#7143)
* fix(download): preserve protocol failures and isolate site updates * test: cover tracker token lifecycle and record dependency facts --------- Co-authored-by: jxxghp <jxxghp@gmail.com>
This commit is contained in:
@@ -454,6 +454,14 @@ class RssHelper:
|
||||
logger.warning(f"RSS文件过大: {len(raw_data) / 1024 / 1024:.1f}MB,跳过解析")
|
||||
return False
|
||||
|
||||
expired_messages = {
|
||||
"RSS 链接已过期, 您需要获得一个新的!",
|
||||
"RSS Link has expired, You need to get a new one!",
|
||||
"RSS Link has expired, You need to get new!",
|
||||
}
|
||||
if ret.text.strip() in expired_messages:
|
||||
return None
|
||||
|
||||
if raw_data:
|
||||
ret_xml = http_port.decode_xml(
|
||||
ret,
|
||||
@@ -601,14 +609,6 @@ class RssHelper:
|
||||
|
||||
except Exception as e2:
|
||||
logger.error(f"解析RSS失败:{str(e2)} - {traceback.format_exc()}")
|
||||
# RSS过期检查
|
||||
_rss_expired_msg = [
|
||||
"RSS 链接已过期, 您需要获得一个新的!",
|
||||
"RSS Link has expired, You need to get a new one!",
|
||||
"RSS Link has expired, You need to get new!"
|
||||
]
|
||||
if ret_xml in _rss_expired_msg:
|
||||
return None
|
||||
return False
|
||||
finally:
|
||||
if root is not None:
|
||||
|
||||
@@ -22,6 +22,7 @@ from app.application.subscription.contract import (
|
||||
)
|
||||
from app.application.subscription.delete import AsyncUnitOfWork
|
||||
from app.schemas.common import JsonData
|
||||
from app.schemas.exception import SiteDomainConflictError
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
SiteMutationResult: TypeAlias = SiteWriteResult
|
||||
@@ -181,8 +182,12 @@ class SiteMutationCommand:
|
||||
"public": 1 if site_info.get("public") else 0,
|
||||
}
|
||||
)
|
||||
await self._repository.stage_create(SiteMutation(values))
|
||||
await self._commit()
|
||||
try:
|
||||
await self._repository.stage_create(SiteMutation(values))
|
||||
await self._commit()
|
||||
except SiteDomainConflictError:
|
||||
await self._unit_of_work.rollback()
|
||||
return SiteMutationResult(False, f"{domain} 站点已存在")
|
||||
await self._publish_updated({"domain": domain})
|
||||
return SiteMutationResult(True)
|
||||
|
||||
@@ -204,8 +209,12 @@ class SiteMutationCommand:
|
||||
values["url"] = normalized_url
|
||||
values["domain"] = normalized_domain
|
||||
values.pop("id", None)
|
||||
await self._repository.stage_update(site_id, SiteMutation(values))
|
||||
await self._commit()
|
||||
try:
|
||||
await self._repository.stage_update(site_id, SiteMutation(values))
|
||||
await self._commit()
|
||||
except SiteDomainConflictError:
|
||||
await self._unit_of_work.rollback()
|
||||
return SiteMutationResult(False, f"{normalized_domain} 站点已存在")
|
||||
await self._publish_updated(
|
||||
{
|
||||
"site_id": site_id,
|
||||
|
||||
@@ -9,6 +9,7 @@ from app.domain.media import is_music_media_source, normalize_music_type
|
||||
from app.domain.meta.metamusic import MetaMusic
|
||||
from app.domain.metainfo import MetaInfo
|
||||
from app.foundation.crypto import HashUtils
|
||||
from app.runtime.cache import async_cache_mutation
|
||||
from app.schemas.types import MUSIC_ENTITY_RECORDING, MediaSource, MediaType, MusicTargetEntityType
|
||||
|
||||
|
||||
@@ -113,12 +114,24 @@ class TorrentCacheRecognitionService:
|
||||
)
|
||||
else:
|
||||
mediainfo.clear()
|
||||
target_context.media_info = mediainfo
|
||||
|
||||
video_cache, music_cache = self.torrents_chain.split_cache_contexts(cache_data)
|
||||
video_file, music_file = self.torrents_chain.cache_files()
|
||||
await self.torrents_chain.async_save_cache(video_cache, video_file)
|
||||
await self.torrents_chain.async_save_cache(music_cache, music_file)
|
||||
async with async_cache_mutation("torrent-snapshots"):
|
||||
# 远端识别期间不占写锁;提交时重读当前快照,仅修改本次目标。
|
||||
latest = await self.torrents_chain.async_get_torrents()
|
||||
current = next((context for context in latest.get(domain, [])
|
||||
if HashUtils.md5(f"{context.torrent_info.title}{context.torrent_info.description}")
|
||||
== torrent_hash), None)
|
||||
if current is None:
|
||||
return False, "种子缓存已刷新,目标条目不存在", None
|
||||
current.media_info = mediainfo
|
||||
video_cache, music_cache = self.torrents_chain.split_cache_contexts(latest)
|
||||
video_file, music_file = self.torrents_chain.cache_files()
|
||||
# 同一个资源不会跨分类移动;只写其所属缓存,避免无关文件失败或部分提交。
|
||||
is_music_cache = current.torrent_info.category in (MediaType.MUSIC, MediaType.MUSIC.value)
|
||||
cache, filename = (music_cache, music_file) if is_music_cache else (video_cache, video_file)
|
||||
try:
|
||||
await self.torrents_chain.async_save_cache(cache, filename, strict=True)
|
||||
except Exception:
|
||||
return False, "种子缓存保存失败,请稍后重试", None
|
||||
return True, "重新识别完成", {
|
||||
"media_name": mediainfo.title if mediainfo else "",
|
||||
"media_year": mediainfo.year if mediainfo else "",
|
||||
|
||||
@@ -26,6 +26,7 @@ from app.domain.meta.metamusic import audio_quality_tier, normalize_audio_format
|
||||
from app.domain.metainfo import MetaInfo
|
||||
from app.foundation import text as text_tools
|
||||
from app.foundation.crypto import HashUtils
|
||||
from app.foundation.url import UrlUtils
|
||||
from app.runtime.cache import FileCache, TTLCache
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.media import resolve_media_identity
|
||||
@@ -327,6 +328,59 @@ class TorrentHelper:
|
||||
time.sleep(attempt + 1)
|
||||
return None
|
||||
|
||||
def _request_torrent_redirects(
|
||||
self,
|
||||
http_port: TorrentHttpPort,
|
||||
*,
|
||||
url: str,
|
||||
cookie: Optional[str],
|
||||
ua: Optional[str],
|
||||
referer: Optional[str],
|
||||
proxies: Optional[dict[str, Any]],
|
||||
retry: bool,
|
||||
) -> tuple[Optional[TorrentResponsePort], str, Optional[str]]:
|
||||
"""
|
||||
有界处理GET跳转,跨来源后永久移除原站点凭据和Referer。
|
||||
|
||||
:param http_port: 已装配HTTP端口
|
||||
:param url: 当前下载地址
|
||||
:param cookie: 仅供原始来源使用的站点Cookie
|
||||
:param ua: 请求用户代理
|
||||
:param referer: 原始来源页地址
|
||||
:param proxies: 本次请求代理
|
||||
:param retry: 是否对普通GET执行既有瞬时失败重试
|
||||
:return: 最终响应、最终地址及允许在该地址使用的Cookie
|
||||
:raises ValueError: 跳转缺少地址、目标非法、重复或超过上限
|
||||
"""
|
||||
seen = set()
|
||||
for _hop in range(9):
|
||||
if url in seen:
|
||||
raise ValueError("种子下载重定向出现环路")
|
||||
seen.add(url)
|
||||
response = self._request_torrent_get(
|
||||
http_port, retry and not torrent_rules.is_expiring_download_url(url),
|
||||
url=url, cookie=cookie, ua=ua, referer=referer,
|
||||
proxies=proxies, allow_redirects=False,
|
||||
)
|
||||
if response is None or response.status_code not in (301, 302, 303, 307, 308):
|
||||
return response, url, cookie
|
||||
location = response.headers.get("Location") or response.headers.get("location")
|
||||
close = getattr(response, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
if not location:
|
||||
raise ValueError("种子下载重定向缺少目标地址")
|
||||
destination = urljoin(url, location)
|
||||
if destination.startswith("magnet:"):
|
||||
return None, destination, None
|
||||
target = urlsplit(destination)
|
||||
if target.scheme not in ("http", "https") or not target.hostname or target.username or target.password:
|
||||
raise ValueError("种子下载重定向目标无效")
|
||||
if not UrlUtils.same_origin(url, destination):
|
||||
cookie, referer = None, None
|
||||
url = destination
|
||||
raise ValueError("种子下载重定向次数超过上限")
|
||||
|
||||
def _should_retry_torrent_response(self, response: Optional[TorrentResponsePort]) -> bool:
|
||||
"""
|
||||
保留重定向、磁力和首次确认页,明确限流不得消耗瞬时重试预算。
|
||||
@@ -380,19 +434,15 @@ class TorrentHelper:
|
||||
config = get_chain_runtime_config_snapshot()
|
||||
http_port = _require_torrent_port()
|
||||
retry = cache_invalid and not torrent_rules.is_expiring_download_url(url)
|
||||
req = self._request_torrent_get(
|
||||
http_port, retry, url=url, cookie=cookie, ua=ua, referer=referer,
|
||||
proxies=config.proxy if proxy else None, allow_redirects=False,
|
||||
)
|
||||
while req is not None and req.status_code in [301, 302]:
|
||||
url = req.headers['Location']
|
||||
if url and url.startswith("magnet:"):
|
||||
return None, url, "", [], "获取到磁力链接"
|
||||
req = self._request_torrent_get(
|
||||
http_port, retry and not torrent_rules.is_expiring_download_url(url),
|
||||
url=url, cookie=cookie, ua=ua, referer=referer,
|
||||
proxies=config.proxy if proxy else None, allow_redirects=False,
|
||||
try:
|
||||
req, url, cookie = self._request_torrent_redirects(
|
||||
http_port, url=url, cookie=cookie, ua=ua, referer=referer,
|
||||
proxies=config.proxy if proxy else None, retry=retry,
|
||||
)
|
||||
except ValueError as err:
|
||||
return cache_path, None, "", [], str(err)
|
||||
if url.startswith("magnet:"):
|
||||
return None, url, "", [], "获取到磁力链接"
|
||||
if rate_limit_error := _torrent_rate_limit_error(req):
|
||||
return cache_path, None, "", [], rate_limit_error
|
||||
if req is not None and req.status_code == 200:
|
||||
@@ -416,7 +466,7 @@ class TorrentHelper:
|
||||
action, data = confirmation
|
||||
req = http_port.request(
|
||||
method="POST", url=action, cookie=cookie, ua=ua,
|
||||
referer=url, proxies=config.proxy if proxy else None, data=data,
|
||||
referer=url, proxies=config.proxy if proxy else None, data=data, allow_redirects=False,
|
||||
)
|
||||
if rate_limit_error := _torrent_rate_limit_error(req):
|
||||
return cache_path, None, "", [], rate_limit_error
|
||||
|
||||
Reference in New Issue
Block a user