diff --git a/app/adapters/network/http.py b/app/adapters/network/http.py index 6072469a4..2225f5193 100644 --- a/app/adapters/network/http.py +++ b/app/adapters/network/http.py @@ -1,5 +1,5 @@ """ -提供同步和异步 HTTP 请求、Cookie 解析与共享连接池生命周期管理。 +提供同步和异步 HTTP 请求、异常分类、Cookie 解析与共享连接池生命周期管理。 """ import asyncio @@ -33,6 +33,11 @@ urllib3.disable_warnings(InsecureRequestWarning) # 业务模块只依赖网络适配器暴露的异常边界,不直接导入具体 transport。 HttpRequestError = requests.exceptions.RequestException AsyncHttpRequestError = httpx2.RequestError +# 资源等幂等下载调用方通过同一异常边界区分可重试故障,不持有原始 transport。 +HttpConnectionError = requests.exceptions.ConnectionError +HttpTimeoutError = requests.exceptions.Timeout +HttpResponseError = requests.exceptions.HTTPError +HttpStreamError = requests.exceptions.ChunkedEncodingError _default_user_agent: Optional[str] = None diff --git a/app/adapters/system/resource.py b/app/adapters/system/resource.py index 49746043b..bef0b3dde 100644 --- a/app/adapters/system/resource.py +++ b/app/adapters/system/resource.py @@ -9,11 +9,20 @@ import json import platform import sys import sysconfig +import time from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar -from app.adapters.network.http import RequestUtils +from app.adapters.network.http import ( + HttpConnectionError, + HttpRequestError, + HttpResponseError, + HttpStreamError, + HttpTimeoutError, + RequestUtils, +) from app.adapters.system.host import SystemUtils +from app.foundation.url import UrlUtils, mask_url_credentials from app.foundation.version import compare_version from app.runtime.log import logger from app.runtime.settings import get_runtime_setting @@ -22,6 +31,9 @@ if TYPE_CHECKING: from requests import Response ResourceVersionProvider = Callable[[], tuple[str, str]] +_RequestResult = TypeVar("_RequestResult") +# 仅重试幂等读取的暂时性 HTTP 故障;404 等永久错误应直接反馈。 +_RETRYABLE_HTTP_STATUSES = frozenset({408, 429, 500, 502, 503, 504}) def _unavailable_resource_versions() -> tuple[str, str]: @@ -71,7 +83,7 @@ class ResourceHelper: _resource_target = Path("app/application/site") _version_flag = get_runtime_setting("RESOURCE_VERSION_FLAG") _repo = ( - f"{get_runtime_setting('GITHUB_PROXY')}https://raw.githubusercontent.com/" + "https://raw.githubusercontent.com/" f"jxxghp/MoviePilot-Resources/main/package.{_version_flag}.json" ) _files_api = ( @@ -82,11 +94,11 @@ class ResourceHelper: @property def proxies(self) -> Optional[dict[str, str]]: """ - 返回访问 GitHub 资源时应使用的代理配置。 + 仅在存在非空镜像前缀时绕过全局代理,保持空白配置与直连地址一致。 """ proxies: Optional[dict[str, str]] = ( None - if get_runtime_setting("GITHUB_PROXY") + if str(get_runtime_setting("GITHUB_PROXY") or "").strip() else get_runtime_setting("PROXY") ) return proxies @@ -132,14 +144,67 @@ class ResourceHelper: def _load_resource_info(self) -> Optional[Response]: """ - 读取 V3 资源清单。 + 按请求时的代理配置读取 V3 清单,暂时性网络失败最多尝试三次。 + + :raises RuntimeError: 请求失败,包含阶段、HTTP 状态或底层网络原因 """ - response: Optional[Response] = RequestUtils( - proxies=self.proxies, + return self._request_with_retry( + lambda: self._get_response(self._proxied(self._repo), self.proxies, 10), + "读取站点资源清单", + ) + + @staticmethod + def _check_response(response: Optional[Response]) -> Response: + """ + 保留 HTTP 状态及无响应原因,供资源请求统一决定是否重试。 + """ + if response is None: + raise HttpConnectionError("未收到响应") + if response.status_code != 200: + raise HttpResponseError(f"HTTP {response.status_code}", response=response) + return response + + def _get_response(self, url: str, proxies: Optional[dict[str, str]], timeout: int) -> Response: + """ + 读取完整响应并保留传输异常;非成功响应在重试前释放。 + """ + response = RequestUtils( + proxies=proxies, headers=get_runtime_setting("GITHUB_HEADERS"), - timeout=10, - ).get_res(self._repo) - return response if response and response.status_code == 200 else None + timeout=timeout, + ).get_res(url, raise_exception=True) + try: + return self._check_response(response) + except HttpResponseError: + if response is not None: + response.close() + raise + + @staticmethod + def _request_with_retry(action: Callable[[], _RequestResult], label: str) -> _RequestResult: + """ + 对资源幂等请求作三次有界尝试,间隔一秒、两秒,不重试解析或磁盘错误。 + + :param action: 单次请求,流式文件请求须自行回滚失败进度及临时内容 + :param label: 展示给用户的请求阶段,不包含凭据 + :raises RuntimeError: 永久故障或重试耗尽,保留脱敏后的原始原因 + """ + attempt = 0 + while True: + attempt += 1 + try: + return action() + except HttpRequestError as error: + retryable = isinstance(error, (HttpConnectionError, HttpTimeoutError, HttpStreamError)) or ( + isinstance(error, HttpResponseError) + and error.response is not None + and error.response.status_code in _RETRYABLE_HTTP_STATUSES + ) + detail = f"{type(error).__name__}:{mask_url_credentials(str(error))}" + if not retryable or attempt >= 3: + raise RuntimeError(f"{label}失败({attempt} 次请求):{detail}") from error + logger.warning(f"{label}失败,准备第 {attempt + 1}/3 次请求:{detail}") + time.sleep(2 ** (attempt - 1)) def get_update_info( self, @@ -168,6 +233,8 @@ class ResourceHelper: resource_info = json.loads(response.text) except json.JSONDecodeError as error: raise RuntimeError("资源包仓库数据解析失败") from error + finally: + response.close() if not isinstance(resource_info, dict): raise RuntimeError("资源包仓库数据格式异常") @@ -229,20 +296,19 @@ class ResourceHelper: def get_download_files(self, update_info: dict[str, Any]) -> list[dict[str, Any]]: """ - 解析资源目录 API,补齐每个待下载文件的地址和大小。 + 有限重试资源目录 API,补齐待下载地址和大小;API 沿用全局代理。 """ - response = RequestUtils( - proxies=get_runtime_setting("PROXY"), - headers=get_runtime_setting("GITHUB_HEADERS"), - timeout=30, - ).get_res(self._files_api) - if response is None or response.status_code != 200: - raise RuntimeError( - f"连接资源仓库失败:HTTP {getattr(response, 'status_code', '无响应')}" - ) + response = self._request_with_retry( + lambda: self._get_response(self._files_api, get_runtime_setting("PROXY"), 30), + "读取站点资源目录", + ) + try: + remote_files = response.json() + finally: + response.close() files_by_name = { item.get("name"): item - for item in response.json() + for item in remote_files if isinstance(item, dict) and item.get("name") } download_files: list[dict[str, Any]] = [] @@ -268,6 +334,8 @@ class ResourceHelper: ) -> list[dict[str, Any]]: """ 把资源文件下载到暂存目录并返回带 SHA256 的安装清单。 + + :param on_progress: 已写入字节增量及文件总大小;重试前用负增量撤回失败字节 """ destination_dir.mkdir(parents=True, exist_ok=True) prepared: list[dict[str, Any]] = [] @@ -277,41 +345,68 @@ class ResourceHelper: if name.name != resource_name: raise RuntimeError(f"资源文件名不安全:{resource_name}") destination = destination_dir / name.name - temporary = destination.with_suffix(destination.suffix + ".part") - temporary.unlink(missing_ok=True) - current = 0 - digest = hashlib.sha256() - with RequestUtils( - proxies=self.proxies, - headers=get_runtime_setting("GITHUB_HEADERS"), - timeout=180, - ).get_stream(self._proxied(str(resource["download_url"]))) as response: - if response is None or response.status_code != 200: - raise RuntimeError( - f"下载资源文件失败:{resource_name},HTTP {getattr(response, 'status_code', '无响应')}" - ) - with temporary.open("wb") as output: - for chunk in response.iter_content(chunk_size=256 * 1024): - if not chunk: - continue - output.write(chunk) - digest.update(chunk) - current += len(chunk) - if on_progress: - on_progress(len(chunk), int(resource.get("size") or 0)) - temporary.replace(destination) + checksum = self._download_resource(resource, destination, on_progress) prepared.append( { "name": resource_name, "type": resource.get("type"), "version": resource.get("version"), "path": str(destination), - "sha256": digest.hexdigest(), + "sha256": checksum, "target": resource.get("target"), } ) return prepared + def _download_resource( + self, + resource: dict[str, Any], + destination: Path, + on_progress: Callable[[int, int], None] | None, + ) -> str: + """ + 重试单个完整文件,校验长度后原子替换,始终清理未完成内容。 + + :return: 最后一次成功传输的 SHA256;失败尝试不计入摘要与进度 + """ + temporary = destination.with_suffix(destination.suffix + ".part") + total = int(resource.get("size") or 0) + + def download() -> str: + """ + 单次流式传输失败时撤回进度,保证重试从空文件和新摘要开始。 + """ + current = 0 + digest = hashlib.sha256() + try: + with RequestUtils( + proxies=self.proxies, + headers=get_runtime_setting("GITHUB_HEADERS"), + timeout=180, + ).get_stream(self._proxied(str(resource["download_url"])), raise_exception=True) as response: + stream = self._check_response(response) + with temporary.open("wb") as output: + for chunk in stream.iter_content(chunk_size=256 * 1024): + if not chunk: + continue + output.write(chunk) + digest.update(chunk) + current += len(chunk) + if on_progress: + on_progress(len(chunk), total) + if total > 0 and current != total: + raise HttpStreamError(f"资源文件长度不符:预期 {total} 字节,收到 {current} 字节") + temporary.replace(destination) + return digest.hexdigest() + except HttpRequestError: + if on_progress and current: + on_progress(-current, total) + raise + finally: + temporary.unlink(missing_ok=True) + + return self._request_with_retry(download, f"下载资源文件 {destination.name}") + def check( self, *, @@ -334,7 +429,7 @@ class ResourceHelper: @staticmethod def _proxied(url: str) -> str: """ - 为资源直链添加 GitHub 代理前缀。 + 请求时规范化代理前缀,兼容带路径及省略尾斜杠的配置。 """ proxy = str(get_runtime_setting("GITHUB_PROXY") or "").strip() - return f"{proxy}{url}" if proxy else url + return f"{UrlUtils.standardize_base_url(proxy)}{url}" if proxy else url diff --git a/app/adapters/system/update.py b/app/adapters/system/update.py index 5f6514e09..efefed3f7 100644 --- a/app/adapters/system/update.py +++ b/app/adapters/system/update.py @@ -23,6 +23,7 @@ from app.adapters.network.http import RequestUtils from app.adapters.system.resource import ResourceHelper, get_resource_versions from app.foundation.environment import is_docker, is_exe from app.foundation.singleton import SingletonClass +from app.foundation.url import UrlUtils from app.foundation.version import compare_version from app.runtime.dependencies.profile import runtime_sync_arguments from app.runtime.log import logger @@ -82,11 +83,12 @@ class SystemUpdateManager(metaclass=SingletonClass): def __init__(self) -> None: """ - 初始化状态写入锁与当前进程的下载任务标记。 + 初始化状态写入锁、下载任务标记及当前进程已检查的资源清单快照。 """ self._lock = threading.RLock() self._download_active = False self._active_target: SystemUpdateType | None = None + self._resource_update_info: tuple[tuple[str, str], dict[str, Any]] | None = None @property def _root(self) -> Path: @@ -271,9 +273,11 @@ class SystemUpdateManager(metaclass=SingletonClass): def _write_check_result( self, target: SystemUpdateType, *, ready_version: str | None = None, **changes: Any - ) -> None: + ) -> bool: """ 仅允许手动检查替换请求开始时的旧待安装目标,保护交错下载及安装。 + + :return: 检查结果是否写入;调用方仅可为已接受的目标保留下载快照 """ with self._lock: item = self._get_item(self.get_status().model_dump(), target) @@ -285,9 +289,10 @@ class SystemUpdateManager(metaclass=SingletonClass): or changes.get("state") != "available" or compare_version(str(changes.get("version") or ""), ">", ready_version) is not True ): - return + return False self._discard_prepared_target(target) self._write_item(target, **changes) + return True def _persist_state(self, state: dict[str, Any]) -> dict[str, Any]: """ @@ -505,7 +510,7 @@ class SystemUpdateManager(metaclass=SingletonClass): self, target: SystemUpdateType | None = None, *, refresh_ready: bool = False ) -> SystemUpdateStatus: """ - 手动检查可刷新旧待安装目标;定时检查和请求失败保留已下载制品。 + 手动检查可刷新旧待安装目标;检查失败保留已下载制品并标明失败阶段。 """ targets = (target,) if target else _TARGETS for update_target in targets: @@ -523,11 +528,12 @@ class SystemUpdateManager(metaclass=SingletonClass): # 定时检查不得打扰当前任务 except Exception as error: # noqa: BLE001 logger.warning(f"检查 {update_target} 更新失败:{error}") + message = f"检查站点资源更新失败:{error}" if update_target == _RESOURCES else str(error) self._write_check_result( update_target, state="idle", checked_at=self._now(), - error=str(error), + error=message, can_update=False, can_install=False, ) @@ -592,50 +598,38 @@ class SystemUpdateManager(metaclass=SingletonClass): def _check_resources(self, *, ready_version: str | None = None) -> None: """ - 查询当前平台认证资源和索引资源的最新版本。 + 查询最新资源版本并保留清单快照,供随后下载复用同一个目标。 """ current_auth, current_indexer = get_resource_versions() info = ResourceHelper().get_update_info( auth_version=current_auth, indexer_version=current_indexer, ) - if info is None: - self._write_check_result( + resource = info or {} + # 状态与清单快照在同一锁内发布,拒绝过期检查时也不得覆盖在途下载的快照。 + with self._lock: + accepted = self._write_check_result( _RESOURCES, - state="idle", - version=None, - auth_version=None, - indexer_version=None, + ready_version=ready_version, + state="available" if info is not None else "idle", + version=resource.get("package_version") or None, + auth_version=resource.get("auth_version"), + indexer_version=resource.get("indexer_version"), current_auth_version=current_auth, current_indexer_version=current_indexer, checked_at=self._now(), downloaded_bytes=0, total_bytes=0, error=None, - can_update=False, + can_update=info is not None, can_install=False, ) - return - self._write_check_result( - _RESOURCES, - ready_version=ready_version, - state="available", - version=info.get("package_version") or None, - auth_version=info.get("auth_version"), - indexer_version=info.get("indexer_version"), - current_auth_version=current_auth, - current_indexer_version=current_indexer, - checked_at=self._now(), - downloaded_bytes=0, - total_bytes=0, - error=None, - can_update=True, - can_install=False, - ) + if accepted: + self._resource_update_info = ((current_auth, current_indexer), info) if info is not None else None def start_download(self, target: SystemUpdateTarget = _APPLICATION) -> SystemUpdateStatus: """ - 用一个后台任务顺序准备所选更新,合并更新保留已下载且仍为最新的包。 + 资源下载前刷新一次清单并供后台复用;合并更新保留已下载且仍为最新的包。 """ if target not in (*_TARGETS, "all"): raise ValueError(f"未知升级类型:{target}") @@ -650,7 +644,7 @@ class SystemUpdateManager(metaclass=SingletonClass): targets = tuple(item.type for item in state.updates if item.state == "available") else: item = self._get_item(state.model_dump(), target) - if item["state"] not in {"available", "ready"}: + if target == _RESOURCES or item["state"] not in {"available", "ready"}: state = self.check(target) targets = tuple(item.type for item in state.updates if item.type == target and item.state == "available") if not targets: @@ -1381,7 +1375,11 @@ class SystemUpdateManager(metaclass=SingletonClass): # 后台线程必须沉淀为可查询失败 except Exception as error: # noqa: BLE001 logger.error(f"下载 {target} 更新包失败:{error}") - self._write_item(target, state="failed", error=str(error), can_update=True, can_install=False) + message = f"下载站点资源更新包失败:{error}" if target == _RESOURCES else str(error) + self._write_item( + target, state="failed", error=message, + can_update=True, can_install=False, + ) def _simulate_application_download(self) -> None: """ @@ -1488,14 +1486,25 @@ class SystemUpdateManager(metaclass=SingletonClass): def _download_resources(self) -> None: """ - 下载当前平台认证与索引资源到外部暂存目录。 + 复用与当前状态匹配的检查快照下载资源;无匹配快照时才重读清单。 """ current_auth, current_indexer = get_resource_versions() helper = ResourceHelper() - info = helper.get_update_info( - auth_version=current_auth, - indexer_version=current_indexer, - ) + cached = getattr(self, "_resource_update_info", None) + target = self._get_item(self._read_state(), _RESOURCES) + if cached and cached[0] == (current_auth, current_indexer) and all( + cached[1].get(info_key) == target.get(state_key) + for info_key, state_key in ( + ("package_version", "version"), ("auth_version", "auth_version"), + ("indexer_version", "indexer_version"), + ) + ): + info = cached[1] + else: + info = helper.get_update_info( + auth_version=current_auth, + indexer_version=current_indexer, + ) if info is None: raise RuntimeError("站点资源已是最新版本") files = helper.get_download_files(info) @@ -1907,10 +1916,10 @@ class SystemUpdateManager(metaclass=SingletonClass): @staticmethod def _proxied(url: str) -> str: """ - 为 GitHub 下载地址添加配置的代理前缀。 + 规范化 GitHub 代理前缀,避免缺少尾斜杠时拼出无效主机名。 """ proxy = str(get_runtime_setting("GITHUB_PROXY") or "").strip() - return f"{proxy}{url}" if proxy else url + return f"{UrlUtils.standardize_base_url(proxy)}{url}" if proxy else url system_update_manager = SystemUpdateManager() diff --git a/app/schemas/system.py b/app/schemas/system.py index 6d2a85f35..f15f99753 100644 --- a/app/schemas/system.py +++ b/app/schemas/system.py @@ -237,7 +237,7 @@ class SystemUpdateItemStatus(BaseModel): total_bytes: int = Field(default=0, description="更新包总字节数") progress: int = Field(default=0, description="当前完成百分比") error: Optional[str] = Field(default=None, description="错误摘要") - can_update: bool = Field(default=False, description="当前是否有可安装的更新") + can_update: bool = Field(default=False, description="是否允许下载或重试更新;失败阶段以 state 和 error 为准") can_install: bool = Field(default=False, description="当前版本是否可以安装") @@ -280,7 +280,7 @@ class SystemUpdateStatus(BaseModel): total_bytes: int = Field(default=0, description="更新包总字节数") progress: int = Field(default=0, description="当前完成百分比") error: Optional[str] = Field(default=None, description="错误摘要") - can_update: bool = Field(default=False, description="当前是否有可安装的更新") + can_update: bool = Field(default=False, description="是否允许下载或重试更新;失败阶段以 state 和 error 为准") can_install: bool = Field(default=False, description="当前版本是否可以安装") updates: list[SystemUpdateItemStatus] = Field(default_factory=list, description="可用的更新项列表") diff --git a/docker/update.sh b/docker/update.sh index df27e12d6..fb4002054 100644 --- a/docker/update.sh +++ b/docker/update.sh @@ -69,6 +69,20 @@ function set_package_proxy_env() { fi } +# 按调用时配置拼接 GitHub 下载地址;兼容无尾斜杠、路径前缀和首尾空白。 +function github_download_url() { + local proxy + proxy="$(printf '%s' "${GITHUB_PROXY:-}" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -z "${proxy}" ]]; then + printf '%s\n' "$1" + return + fi + if [[ "${proxy}" != http://* && "${proxy}" != https://* ]]; then + proxy="http://${proxy}" + fi + printf '%s\n' "${proxy%/}/$1" +} + # 下载归档到临时目录;失败最多尝试三次,成功后归一化后端根目录名称。 function download_and_unzip() { local retries=0 @@ -326,13 +340,15 @@ function existing_resource_dir() { printf '%s\n' "${resource_source_dir}" } +# 资源只写暂存目录;连接抖动有限重试,耗尽后才回退到当前已安装资源。 function download_staged_resource() { local url="$1" local destination="$2" local fallback="$3" local label="$4" - if curl ${CURL_OPTIONS} --fail "${url}" -o "${destination}" \ + if curl ${CURL_OPTIONS} --fail --connect-timeout 10 --max-time 180 \ + --retry 2 --retry-delay 1 --retry-all-errors "${url}" -o "${destination}" \ && [ -s "${destination}" ]; then return 0 fi @@ -345,6 +361,7 @@ function download_staged_resource() { return 1 } +# 暂存插件与站点资源,规范化镜像前缀后下载匹配当前 Python ABI 的资源。 function stage_runtime_payload() { local stage_app="${TMP_PATH}/App" local stage_plugin_dir="${stage_app}/app/plugins" @@ -402,12 +419,12 @@ function stage_runtime_payload() { fi sites_file="sites.${python_version}-${arch_suffix}.so" download_staged_resource \ - "${GITHUB_PROXY}https://raw.githubusercontent.com/jxxghp/MoviePilot-Resources/main/resources.v3/user.sites.v3.bin" \ + "$(github_download_url "https://raw.githubusercontent.com/jxxghp/MoviePilot-Resources/main/resources.v3/user.sites.v3.bin")" \ "${stage_resource_dir}/user.sites.v3.bin" \ "${resource_source_dir}/user.sites.v3.bin" \ "user.sites.v3.bin" || return 1 download_staged_resource \ - "${GITHUB_PROXY}https://raw.githubusercontent.com/jxxghp/MoviePilot-Resources/main/resources.v3/${sites_file}" \ + "$(github_download_url "https://raw.githubusercontent.com/jxxghp/MoviePilot-Resources/main/resources.v3/${sites_file}")" \ "${stage_resource_dir}/${sites_file}" \ "${resource_source_dir}/${sites_file}" \ "${sites_file}" || return 1 @@ -426,9 +443,9 @@ function swap_staged_payload() { fi } -# 下载程序资源,$1: 后端版本路径 +# 通过规范化镜像地址准备前后端归档;$1 为后端版本路径,全部准备成功后才切换。 function install_backend_and_download_resources() { - if ! download_and_unzip "${GITHUB_PROXY}https://github.com/jxxghp/MoviePilot/archive/refs/${1}" "App"; then + if ! download_and_unzip "$(github_download_url "https://github.com/jxxghp/MoviePilot/archive/refs/${1}")" "App"; then WARN "后端程序下载失败,继续使用旧的程序来启动..." return 1 fi @@ -472,7 +489,7 @@ function install_backend_and_download_resources() { fi INFO "前端版本号:${frontend_version}" fi - if ! download_and_unzip "${GITHUB_PROXY}https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${frontend_version}/dist.zip" "dist"; then + if ! download_and_unzip "$(github_download_url "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${frontend_version}/dist.zip")" "dist"; then WARN "前端程序下载失败,继续使用旧的程序来启动..." return 1 fi @@ -596,12 +613,13 @@ function test_connectivity_package() { esac } -# 按传入模式检测镜像或全局代理;直连模式仅配置参数,不额外探测。 +# 按传入模式检测镜像或全局代理,镜像探测与实际下载共用地址规则。 +# 直连模式仅配置参数,不额外探测。 function test_connectivity_github() { case "$1" in 0) if [[ -n "${GITHUB_PROXY}" ]]; then - if curl -sL --connect-timeout 5 --max-time 10 "${GITHUB_PROXY}https://raw.githubusercontent.com/jxxghp/MoviePilot/main/README.md" > /dev/null 2>&1; then + if curl -sL --connect-timeout 5 --max-time 10 "$(github_download_url "https://raw.githubusercontent.com/jxxghp/MoviePilot/main/README.md")" > /dev/null 2>&1; then GITHUB_LOG="镜像代理模式" return 0 fi diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index 258665747..267ba99f0 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -811,13 +811,13 @@ flowchart LR ### 10.2 当前架构快照 -以下是 2026-10-25 核对的 `tests/fixtures/architecture/` 数据;这是源码合同快照, +以下是 2026-10-26 核对的 `tests/fixtures/architecture/` 数据;这是源码合同快照, 不代表某个已发布镜像或正在运行的实例已经验证。 | 指标 | 当前值 | |---|---:| | Python 模块 | 1139 | -| 内部导入边 | 9,822 | +| 内部导入边 | 9,824 | | 非平凡 SCC | 1(29 个模块的 TMDB vendor 精确 containment) | | Application / Chain 具体 Adapter 直连 | 0 / 0 | | Direct egress | 51(精确 policy 准入,Application / Chain 均为 0) | diff --git a/docs/cli.md b/docs/cli.md index bc89db7de..bae5207f0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -442,6 +442,7 @@ moviepilot version - `MOVIEPILOT_AUTO_UPDATE` 为布尔开关,默认 `false`;只有 `true` 启用后台 Release 检查和版本提醒,保存后定时服务热更新。`AUTO_UPDATE_RESOURCE` 独立控制站点资源检查和提醒;任一开关开启即启用检测服务,且只检查对应目标,两者均关闭才移除服务。`MOVIEPILOT_UPDATE_DEV` 为独立布尔开关,默认 `false`;设为 `true` 时手动 `moviepilot update` 默认使用 DEV 更新模式。旧 `dev/release` 值统一转换为 `MOVIEPILOT_AUTO_UPDATE=true`;旧 `dev` 在未显式配置新开关时迁移为 `MOVIEPILOT_UPDATE_DEV=true` - Release 更新由后台每 6 小时检查 GitHub Release;管理员确认后先静默下载安装包并显示进度,下载完成后再次确认重启,通过专门的内部更新入口安装已下载且通过 SHA-256 校验的包后再启动服务 - Web 更新提示沿用“升级 / 立即重启”入口:同时发现主程序和站点资源更新时,依次准备主程序、后端声明的配套前端和最新站点资源,全部就绪后只重启一次;只有一类更新时保持对应卡片,底部只显示一个升级或重启按钮。重启前主动检查最新版本,可将旧待安装目标替换为更高版本并继续下载;API 仍支持指定单类更新。定时检查、检查失败及无更高版本时仍保留已下载包,下载和安装中不会更换目标。正式 V3 更新按版本号选择最高版本,支持跨多个 V3 小版本直接安装完整包;V2 到 V3 不属于此流程。站点原生扩展随重启加载,普通启动/重启不会检查或下载更新。 +- `GITHUB_PROXY` 镜像前缀可省略尾斜杠,允许带路径;修改配置仍遵循设置项的重启提示。站点资源清单、目录和文件遇到连接失败、超时或暂时性 HTTP 错误时最多尝试三次,间隔一秒、两秒;文件重试从头下载,失败内容不会进入安装清单。下载动作刷新一次清单并在后台复用,避免重复请求或沿用旧检查结果。错误摘要包含检查或下载阶段及具体网络原因;`state=failed` 表示操作失败,`can_update=true` 表示允许重试,只有 `can_install=true` 才能安装。 - 页面中的“稍后”会在当前浏览器暂停提醒 24 小时,“忽略此版本”只屏蔽当前版本;出现更高版本时会重新提示 - 通过系统内置的重启入口触发重启时,本地 CLI 安装模式也会复用同一套前后端进程管理完成重启 - 前端默认监听 `NGINX_PORT`,默认值 `3000` diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index 134173785..26fdbd026 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -959,8 +959,8 @@ "runtime_only": true } }, - "edge_count": 9822, - "edge_sha256": "03e904678af5150034e8f55a64b383c4ce22f6157f21bf2d98e34bb1b6dd3d2d", + "edge_count": 9824, + "edge_sha256": "3f1ac081c43f4139ba631664cea9f91bc3aaccee7b9186b0f57a9334c218b88b", "edges": [ "app -> app.foundation", "app -> app.foundation.environment", @@ -1167,6 +1167,7 @@ "app.adapters.system.resource -> app.adapters.system", "app.adapters.system.resource -> app.adapters.system.host", "app.adapters.system.resource -> app.foundation", + "app.adapters.system.resource -> app.foundation.url", "app.adapters.system.resource -> app.foundation.version", "app.adapters.system.resource -> app.runtime", "app.adapters.system.resource -> app.runtime.log", @@ -1184,6 +1185,7 @@ "app.adapters.system.update -> app.foundation", "app.adapters.system.update -> app.foundation.environment", "app.adapters.system.update -> app.foundation.singleton", + "app.adapters.system.update -> app.foundation.url", "app.adapters.system.update -> app.foundation.version", "app.adapters.system.update -> app.runtime", "app.adapters.system.update -> app.runtime.dependencies", diff --git a/tests/test_github_proxy_update.py b/tests/test_github_proxy_update.py new file mode 100644 index 000000000..4ef82a44f --- /dev/null +++ b/tests/test_github_proxy_update.py @@ -0,0 +1,38 @@ +""" +验证 Python 更新器与 Docker 控制脚本使用相同的 GitHub 代理地址规则。 +""" + +import os +import subprocess +from pathlib import Path + +import pytest + +from app.adapters.system import update as update_module + + +@pytest.mark.parametrize( + ("proxy", "prefix"), + [ + ("", ""), + ("https://proxy.example", "https://proxy.example/"), + ("https://proxy.example/", "https://proxy.example/"), + (" https://proxy.example/path ", "https://proxy.example/path/"), + ("proxy.example/path", "http://proxy.example/path/"), + ], +) +def test_update_proxy_prefix_matches_docker(monkeypatch, tmp_path, proxy, prefix): + """ + 代理首尾空白、路径及尾斜杠兼容规则在两种部署更新入口保持一致。 + """ + url = "https://github.com/jxxghp/MoviePilot/archive/refs/tags/v3.1.9.zip" + monkeypatch.setattr(update_module, "get_runtime_setting", lambda _key: proxy) + updater = Path(__file__).resolve().parents[1] / "docker" / "update.sh" + result = subprocess.run( + ["/bin/bash", "-c", 'source "$1"; github_download_url "$2"', "proxy-test", str(updater), url], + env={**os.environ, "GITHUB_PROXY": proxy, "CONFIG_DIR": str(tmp_path)}, + text=True, capture_output=True, check=True, + ) + + assert result.stdout.strip() == prefix + url + assert update_module.SystemUpdateManager._proxied(url) == prefix + url diff --git a/tests/test_resource_download.py b/tests/test_resource_download.py new file mode 100644 index 000000000..a2814b3bd --- /dev/null +++ b/tests/test_resource_download.py @@ -0,0 +1,220 @@ +""" +站点资源代理地址、有限网络重试及暂存文件完整性的离线回归。 +""" + +import errno +import hashlib +import json +import time +from contextlib import closing +from unittest.mock import Mock + +import pytest +from requests import Response +from requests.exceptions import ChunkedEncodingError, ConnectionError, ReadTimeout + +from app.adapters.system import resource as resource_module + + +@pytest.fixture +def resource_settings(monkeypatch): + """ + 隔离部署配置及退避等待,禁止用例访问实际代理或运行目录。 + """ + values = {"GITHUB_PROXY": "", "PROXY": None, "GITHUB_HEADERS": {}, "AUTO_UPDATE_RESOURCE": True} + monkeypatch.setattr(resource_module, "get_runtime_setting", values.get) + monkeypatch.setattr(resource_module.SystemUtils, "is_frozen", lambda: False) + monkeypatch.setattr(time, "sleep", Mock()) + return values + + +def _response(status=200, content=b"{}"): + """ + 构造可关闭且不访问网络的真实 HTTP 响应。 + """ + response = Response() + response.status_code = status + response._content = content + response._content_consumed = True + response.url = "https://raw.githubusercontent.com/test" + response.close = Mock() + return response + + +@pytest.mark.parametrize("proxy", ["https://proxy.example", "https://proxy.example/", " https://proxy.example/path "]) +def test_manifest_uses_current_normalized_proxy(monkeypatch, resource_settings, proxy): + """ + 导入后改变代理也必须生效,带路径及缺少尾斜杠的前缀均保留目标完整地址。 + """ + request = Mock(return_value=_response()) + monkeypatch.setattr(resource_module.RequestUtils, "get_res", request) + helper = resource_module.ResourceHelper() + resource_settings["GITHUB_PROXY"] = proxy + + helper._load_resource_info() + + assert request.call_args.args[0] == f"{proxy.strip().rstrip('/')}/{helper._repo}" + assert request.call_args.kwargs["raise_exception"] is True + + +@pytest.mark.parametrize("failure", [ConnectionError("DNS lookup failed"), ReadTimeout("read timed out"), 503]) +def test_manifest_retries_transient_failures(monkeypatch, resource_settings, failure): + """ + 清单遭遇 DNS、超时或服务端暂时故障时最多请求三次,成功后停止重试。 + """ + first = _response(failure) if isinstance(failure, int) else failure + request = Mock(side_effect=[first, ConnectionError("connection reset"), _response()]) + monkeypatch.setattr(resource_module.RequestUtils, "get_res", request) + + assert resource_module.ResourceHelper()._load_resource_info().status_code == 200 + assert request.call_count == 3 + assert [item.args for item in time.sleep.call_args_list] == [(1,), (2,)] + if isinstance(first, Response): + first.close.assert_called_once() + + +@pytest.mark.parametrize("status, attempts", [(404, 1), (503, 3)]) +def test_manifest_reports_http_status_with_bounded_attempts(monkeypatch, resource_settings, status, attempts): + """ + 永久 HTTP 错误直接失败,重试耗尽时仍保留阶段及真实状态码。 + """ + request = Mock(side_effect=lambda *args, **kwargs: _response(status)) + monkeypatch.setattr(resource_module.RequestUtils, "get_res", request) + + with pytest.raises(RuntimeError, match=f"清单.*HTTP {status}"): + resource_module.ResourceHelper().get_update_info(auth_version="1", indexer_version="1") + assert request.call_count == attempts + + +def test_manifest_reports_sanitized_connection_reason(monkeypatch, resource_settings): + """ + 连接异常保留 DNS 等诊断线索,同时遮盖异常地址里的凭据。 + """ + request = Mock(side_effect=ConnectionError("DNS failed: https://user:password@proxy.example/?token=private")) + monkeypatch.setattr(resource_module.RequestUtils, "get_res", request) + + with pytest.raises(RuntimeError, match="ConnectionError.*DNS failed") as caught: + resource_module.ResourceHelper()._load_resource_info() + assert "password" not in str(caught.value) + assert "private" not in str(caught.value) + assert request.call_count == 3 + + +def test_manifest_parse_error_closes_response_without_retry(monkeypatch, resource_settings): + """ + 镜像返回无法解析的数据时保留解析错误,不把内容错误当作连接抖动。 + """ + response = _response(content=b"proxy error") + request = Mock(return_value=response) + monkeypatch.setattr(resource_module.RequestUtils, "get_res", request) + + with pytest.raises(RuntimeError, match="解析失败"): + resource_module.ResourceHelper().get_update_info(auth_version="1", indexer_version="1") + request.assert_called_once() + response.close.assert_called_once() + + +@pytest.mark.parametrize("proxy", ["", " "]) +def test_empty_mirror_preserves_global_proxy(monkeypatch, resource_settings, proxy): + """ + 未配置镜像前缀时清单保持原地址,并继续使用全局代理。 + """ + resource_settings.update(GITHUB_PROXY=proxy, PROXY={"https": "http://global.example"}) + request = Mock(return_value=_response()) + factory = Mock(return_value=Mock(get_res=request)) + monkeypatch.setattr(resource_module, "RequestUtils", factory) + + helper = resource_module.ResourceHelper() + helper._load_resource_info() + + assert request.call_args.args[0] == helper._repo + assert factory.call_args.kwargs["proxies"] == resource_settings["PROXY"] + + +def test_directory_request_retries_without_mirror_prefix(monkeypatch, resource_settings): + """ + 目录 API 沿用全局代理并支持重试,不把 GitHub API 改成镜像地址。 + """ + resource_settings.update(GITHUB_PROXY="https://proxy.example", PROXY={"https": "http://global.example"}) + response = _response(content=json.dumps([{"name": "sites.bin", "download_url": "https://raw.example/sites.bin", "size": 6}]).encode()) + request = Mock(side_effect=[ReadTimeout("read timed out"), response]) + factory = Mock(return_value=Mock(get_res=request)) + monkeypatch.setattr(resource_module, "RequestUtils", factory) + + files = resource_module.ResourceHelper().get_download_files({"files": [{"name": "sites.bin"}]}) + + assert files[0]["size"] == 6 + assert request.call_args.args[0].startswith("https://api.github.com/") + assert factory.call_args.kwargs["proxies"] == resource_settings["PROXY"] + response.close.assert_called_once() + + +@pytest.mark.parametrize("failure", [ChunkedEncodingError("reset during body"), None]) +def test_file_retry_discards_partial_content_and_rolls_back_progress(monkeypatch, tmp_path, resource_settings, failure): + """ + 传输中断及长度不足均从头下载,失败字节不累计到进度或最终摘要。 + """ + resource_settings["GITHUB_PROXY"] = "https://proxy.example" + first = _response() + first.iter_content = Mock(return_value=iter([b"bad"])) + if failure: + def interrupted_body(**_kwargs): + """ + 在写入部分内容后模拟网络中断。 + """ + yield b"bad" + raise failure + + first.iter_content.side_effect = interrupted_body + second = _response(content=b"abcdef") + request = Mock(side_effect=[closing(first), closing(second)]) + monkeypatch.setattr(resource_module.RequestUtils, "get_stream", request) + progress = Mock() + + files = resource_module.ResourceHelper().download_files( + [{"name": "sites.bin", "download_url": "https://raw.example/sites.bin", "size": 6}], tmp_path, progress, + ) + + assert (tmp_path / "sites.bin").read_bytes() == b"abcdef" + assert not (tmp_path / "sites.bin.part").exists() + assert files[0]["sha256"] == hashlib.sha256(b"abcdef").hexdigest() + assert [item.args[0] for item in progress.call_args_list] == [3, -3, 6] + assert request.call_count == 2 + assert request.call_args.args[0] == "https://proxy.example/https://raw.example/sites.bin" + first.close.assert_called_once() + second.close.assert_called_once() + + +def test_failed_file_download_preserves_previous_file(monkeypatch, tmp_path, resource_settings): + """ + 重试耗尽不替换旧文件,也不留下可被误当作完整资源的临时文件。 + """ + (tmp_path / "sites.bin").write_bytes(b"old") + request = Mock(side_effect=ConnectionError("connection reset")) + monkeypatch.setattr(resource_module.RequestUtils, "get_stream", request) + + with pytest.raises(RuntimeError, match="sites.bin.*ConnectionError"): + resource_module.ResourceHelper().download_files( + [{"name": "sites.bin", "download_url": "https://raw.example/sites.bin", "size": 6}], tmp_path, + ) + assert (tmp_path / "sites.bin").read_bytes() == b"old" + assert not (tmp_path / "sites.bin.part").exists() + assert request.call_count == 3 + + +def test_file_disk_error_is_not_retried(monkeypatch, tmp_path, resource_settings): + """ + 磁盘写入故障直接反馈,不追加网络请求且仍释放响应、清理临时文件。 + """ + response = _response(content=b"abcdef") + request = Mock(return_value=closing(response)) + monkeypatch.setattr(resource_module.RequestUtils, "get_stream", request) + monkeypatch.setattr(type(tmp_path), "open", Mock(side_effect=OSError(errno.ENOSPC, "disk full"))) + + with pytest.raises(OSError, match="disk full"): + resource_module.ResourceHelper().download_files( + [{"name": "sites.bin", "download_url": "https://raw.example/sites.bin", "size": 6}], tmp_path, + ) + request.assert_called_once() + response.close.assert_called_once() + assert not (tmp_path / "sites.bin.part").exists() diff --git a/tests/test_system_update_batch.py b/tests/test_system_update_batch.py index 3dd733662..6077a83d8 100644 --- a/tests/test_system_update_batch.py +++ b/tests/test_system_update_batch.py @@ -243,6 +243,91 @@ def test_retry_batch_download_reuses_successful_application(monkeypatch, manager assert resources.call_count == 2 +@pytest.mark.parametrize("target", ["resources", "all"]) +def test_resource_download_reuses_checked_manifest(monkeypatch, manager, target): + """ + 一次单类或合并下载只读取一次清单,防止检查成功后重复请求遭遇出口抖动。 + """ + info = {"package_version": "2", "auth_version": "2", "indexer_version": "2", "files": []} + request = Mock(side_effect=[info, RuntimeError("重复清单请求")]) + monkeypatch.setattr(update_module.ResourceHelper, "get_update_info", request) + monkeypatch.setattr(update_module.ResourceHelper, "get_download_files", lambda _self, _info: []) + monkeypatch.setattr(update_module.ResourceHelper, "download_files", lambda _self, *_args: []) + monkeypatch.setattr(manager, "_check_application", lambda **_kwargs: None) + monkeypatch.setattr(update_module, "ThreadHelper", lambda: SimpleNamespace(submit=lambda function, selected: function(selected))) + + status = manager.start_download(target) + + resource = next(item for item in status.updates if item.type == "resources") + assert resource.state == "ready" + assert resource.version == "2" + request.assert_called_once() + + +def test_resource_download_failure_retains_retry_with_explicit_phase(monkeypatch, manager): + """ + 下载失败必须明确阶段,同时保留重试能力并禁止安装。 + """ + manager._write_item("resources", state="available", version="2", indexer_version="2", can_update=True) + monkeypatch.setattr(manager, "_check_resources", lambda: None) + monkeypatch.setattr(manager, "_download_resources", Mock(side_effect=RuntimeError("HTTP 503"))) + monkeypatch.setattr(update_module, "ThreadHelper", lambda: SimpleNamespace(submit=lambda function, target: function(target))) + + status = manager.start_download("resources") + + resource = next(item for item in status.updates if item.type == "resources") + assert resource.state == "failed" + assert "下载" in resource.error and "HTTP 503" in resource.error + assert resource.can_update is True + assert resource.can_install is False + + +def test_resource_download_refreshes_manifest_between_actions(monkeypatch, manager): + """ + 独立检查后远端发布新资源时,下载动作刷新一次清单,避免旧版本号配上新文件。 + """ + request = Mock(side_effect=[ + {"package_version": "2", "indexer_version": "2", "files": []}, + {"package_version": "3", "indexer_version": "3", "files": []}, + RuntimeError("同一次下载不得再次读取清单"), + ]) + monkeypatch.setattr(update_module.ResourceHelper, "get_update_info", request) + monkeypatch.setattr(update_module.ResourceHelper, "get_download_files", lambda _self, _info: []) + monkeypatch.setattr(update_module.ResourceHelper, "download_files", lambda _self, *_args: []) + monkeypatch.setattr(update_module, "ThreadHelper", lambda: SimpleNamespace(submit=lambda function, target: function(target))) + manager.check("resources") + + status = manager.start_download("resources") + + resource = next(item for item in status.updates if item.type == "resources") + assert (resource.state, resource.version) == ("ready", "3") + assert request.call_count == 2 + + +@pytest.mark.parametrize("info", [None, {"package_version": "3", "indexer_version": "3"}]) +def test_rejected_resource_check_preserves_downloading_snapshot(monkeypatch, manager, info): + """ + 过期检查不能替换已经开始下载的资源清单,避免状态和下载目标分离。 + """ + previous = (("1", "1"), {"package_version": "2", "indexer_version": "2"}) + manager._resource_update_info = previous + + def check_response(**_kwargs): + """ + 在检查响应到达前模拟另一个调用者启动下载。 + """ + manager._download_active = True + manager._write_item("resources", state="downloading", version="2", indexer_version="2") + return info + + monkeypatch.setattr(update_module, "ResourceHelper", lambda: SimpleNamespace(get_update_info=check_response)) + + manager.check("resources") + + assert manager._resource_update_info is previous + assert manager.get_status().updates[1].state == "downloading" + + def test_combined_install_writes_one_manifest_and_marks_both_installing(monkeypatch, manager): """ 两个包全部校验后只写一份安装意图,重启失败可一起撤销回到 ready。