mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-22 03:43:45 +08:00
fix(storage): preserve transfer contents and cleanup identities (#7142)
This commit is contained in:
@@ -93,7 +93,7 @@
|
||||
"app/modules/filemanager/storages/u115.py:U115Pan.upload": 33,
|
||||
"app/modules/filemanager/transhandler.py:TransHandler.__resolve_overwrite": 16,
|
||||
"app/modules/filemanager/transhandler.py:TransHandler.__transfer_command": 19,
|
||||
"app/modules/filemanager/transhandler.py:TransHandler.execute_transfer_plan": 20,
|
||||
"app/modules/filemanager/transhandler.py:TransHandler.execute_transfer_plan": 19,
|
||||
"app/modules/filter/__init__.py:FilterModule.__match_rule": 24,
|
||||
"app/modules/imdb/__init__.py:ImdbModule._to_media_info": 17,
|
||||
"app/modules/indexer/spider/__init__.py:SiteSpider.__get_search_url": 24,
|
||||
|
||||
@@ -132,7 +132,7 @@ def test_modified_builtin_skills_have_incremented_versions() -> None:
|
||||
"anysearch": "3",
|
||||
"moviepilot-api": "59",
|
||||
"moviepilot-update": "7",
|
||||
"organize-files": "5",
|
||||
"organize-files": "6",
|
||||
"transfer-failed-retry": "6",
|
||||
"generate-identifiers": "4",
|
||||
"create-moviepilot-plugin": "7",
|
||||
|
||||
214
tests/test_local_transfer_review_regressions.py
Normal file
214
tests/test_local_transfer_review_regressions.py
Normal file
@@ -0,0 +1,214 @@
|
||||
"""
|
||||
验证本地并发暂存、音频链接写入和递归重命名的审查回归。
|
||||
"""
|
||||
|
||||
import errno
|
||||
import os
|
||||
import shutil
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from threading import Barrier, Event
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.adapters.system.fsproxy import fsproxy
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.api.endpoints import storage as storage_endpoint
|
||||
from app.application import audio
|
||||
from app.modules.filemanager.storages.local import LocalStorage
|
||||
from app.schemas.file import FileItem
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["copy", "move"])
|
||||
def test_concurrent_transfers_to_same_target_use_separate_partials(tmp_path, monkeypatch, operation):
|
||||
"""
|
||||
并发同目标复制不能发布另一任务的暂存内容,跨盘移动也不能丢失源内容。
|
||||
"""
|
||||
first, second, target = tmp_path / "a.bin", tmp_path / "b.bin", tmp_path / "target.bin"
|
||||
first.write_bytes(b"A" * 64)
|
||||
second.write_bytes(b"B" * 64)
|
||||
copied_first, copied_second, published_first = Event(), Event(), Event()
|
||||
partials = []
|
||||
original_copy = fsproxy.copy
|
||||
storage = object.__new__(LocalStorage)
|
||||
monkeypatch.setattr(storage, "_try_reflink_copy", lambda *_args: False)
|
||||
monkeypatch.setattr(storage, "_LocalStorage__should_show_progress", lambda *_args: False)
|
||||
|
||||
def copy(source, destination, **kwargs):
|
||||
"""
|
||||
固定两份内容完成复制和提升的先后顺序。
|
||||
"""
|
||||
partials.append(destination)
|
||||
if source == first:
|
||||
result = original_copy(source, destination, **kwargs)
|
||||
copied_first.set()
|
||||
assert copied_second.wait(10)
|
||||
return result
|
||||
assert copied_first.wait(10)
|
||||
result = original_copy(source, destination, **kwargs)
|
||||
copied_second.set()
|
||||
assert published_first.wait(10)
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(fsproxy, "copy", copy)
|
||||
if operation == "move":
|
||||
replace = os.replace
|
||||
|
||||
def cross_device_replace(source, destination):
|
||||
"""
|
||||
只让原始移动跨设备,暂存提升仍使用真实原子替换。
|
||||
"""
|
||||
if source in (first, second):
|
||||
raise OSError(errno.EXDEV, "cross-device")
|
||||
return replace(source, destination)
|
||||
|
||||
monkeypatch.setattr(os, "replace", cross_device_replace)
|
||||
|
||||
def transfer(source):
|
||||
"""
|
||||
返回每个任务确认成功时实际发布的内容。
|
||||
"""
|
||||
try:
|
||||
result = getattr(storage, operation)(FileItem(path=str(source), type="file"), tmp_path, target.name)
|
||||
return result, target.read_bytes()
|
||||
finally:
|
||||
if source == first:
|
||||
published_first.set()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as workers:
|
||||
first_result = workers.submit(transfer, first)
|
||||
second_result = workers.submit(transfer, second)
|
||||
assert first_result.result(20) == (True, b"A" * 64)
|
||||
assert second_result.result(20) == (True, b"B" * 64)
|
||||
assert len(set(partials)) == 2
|
||||
assert not list(tmp_path.glob("*.mp-partial"))
|
||||
|
||||
|
||||
def test_reflink_uses_distinct_private_staging_names(tmp_path, monkeypatch):
|
||||
"""
|
||||
直接调用宿主 reflink 的同目标任务也必须独立分配暂存名称。
|
||||
"""
|
||||
source, target = tmp_path / "source", tmp_path / "target"
|
||||
source.write_bytes(b"data")
|
||||
seen = []
|
||||
copied = Barrier(2)
|
||||
|
||||
def clone(src, dest):
|
||||
"""
|
||||
回放并发克隆,强制两份暂存同时存活。
|
||||
"""
|
||||
seen.append(dest)
|
||||
shutil.copyfile(src, dest)
|
||||
copied.wait(10)
|
||||
return 0, ""
|
||||
|
||||
monkeypatch.setattr(SystemUtils, "_reflink_copy", clone)
|
||||
with ThreadPoolExecutor(max_workers=2) as workers:
|
||||
results = list(workers.map(lambda _index: SystemUtils.reflink(source, target), range(2)))
|
||||
assert results == [(0, ""), (0, "")]
|
||||
assert len(set(seen)) == 2
|
||||
|
||||
|
||||
def test_audio_link_write_commits_despite_unchanged_timestamp_signature(tmp_path, monkeypatch):
|
||||
"""
|
||||
相同长度标签更新即使文件系统时间戳没有变化,也不能丢弃独立副本。
|
||||
"""
|
||||
source, target = tmp_path / "seed.flac", tmp_path / "library.flac"
|
||||
source.write_bytes(b"old-title")
|
||||
os.link(source, target)
|
||||
real_signature = audio._audio_signature
|
||||
signatures = {}
|
||||
|
||||
def signature(path):
|
||||
"""
|
||||
模拟时间粒度不足的文件系统,保持每条路径的首次版本签名。
|
||||
"""
|
||||
signatures.setdefault(path, real_signature(path))
|
||||
return signatures[path]
|
||||
|
||||
monkeypatch.setattr(audio, "_audio_signature", signature)
|
||||
with audio._audio_write_target(target) as writable:
|
||||
writable.write_bytes(b"new-title")
|
||||
assert target.read_bytes() == b"new-title"
|
||||
assert source.read_bytes() == b"old-title"
|
||||
assert not target.samefile(source)
|
||||
|
||||
|
||||
def test_recursive_rename_recognizes_nested_original_paths(tmp_path, monkeypatch):
|
||||
"""
|
||||
递归重命名必须处理季目录内媒体,并用真实子路径进行识别。
|
||||
"""
|
||||
root = tmp_path / "Original"
|
||||
nested = root / "Season1" / "Show.S01E01.mkv"
|
||||
nested.parent.mkdir(parents=True)
|
||||
nested.write_bytes(b"fictional")
|
||||
storage = object.__new__(LocalStorage)
|
||||
recognized = []
|
||||
|
||||
def list_files(item, recursion=False):
|
||||
"""
|
||||
在临时目录提供与存储链一致的递归列表合同。
|
||||
"""
|
||||
directory = Path(item.path)
|
||||
paths = directory.rglob("*") if recursion else directory.iterdir()
|
||||
return [storage.get_item_strict(path) for path in paths]
|
||||
|
||||
def recognize(path, **_kwargs):
|
||||
"""
|
||||
记录实际识别位置,不读取外部媒体或来源。
|
||||
"""
|
||||
recognized.append(path)
|
||||
return SimpleNamespace(meta_info=path, media_info=object())
|
||||
|
||||
monkeypatch.setattr(storage_endpoint, "StorageChain", lambda: SimpleNamespace(
|
||||
list_files=list_files, rename_file=storage.rename,
|
||||
))
|
||||
monkeypatch.setattr(storage_endpoint, "MediaChain", lambda: SimpleNamespace(recognize_by_path=recognize))
|
||||
monkeypatch.setattr(storage_endpoint, "TransferChain", lambda: SimpleNamespace(
|
||||
recommend_name=lambda **_kwargs: "Recognized.mkv"))
|
||||
monkeypatch.setattr(storage_endpoint, "ProgressHelper", Mock())
|
||||
monkeypatch.setattr(storage_endpoint, "get_api_runtime_config_snapshot", lambda: SimpleNamespace(
|
||||
media_extensions=[".mkv"], subtitle_extensions=[], audio_extensions=[]))
|
||||
result = storage_endpoint.rename(storage.get_item_strict(root), "Renamed", recursive=True)
|
||||
assert result.success
|
||||
assert recognized == [nested]
|
||||
assert (tmp_path / "Renamed" / "Season1" / "Recognized.mkv").read_bytes() == b"fictional"
|
||||
|
||||
|
||||
def test_parallel_reflink_permission_probes_are_private(tmp_path, monkeypatch):
|
||||
"""
|
||||
并发权限探针各自持有临时名称,清理另一任务探针不能使克隆退回复制。
|
||||
"""
|
||||
source = tmp_path / "source"
|
||||
source.write_bytes(b"data")
|
||||
probes = []
|
||||
ready = Barrier(2)
|
||||
write_bytes = Path.write_bytes
|
||||
|
||||
def write_probe(path, data):
|
||||
"""
|
||||
强制两个权限探针同时存在,捕获各自名称。
|
||||
"""
|
||||
result = write_bytes(path, data)
|
||||
if path.name.startswith(".mode-probe-"):
|
||||
probes.append(path)
|
||||
ready.wait(10)
|
||||
return result
|
||||
|
||||
def clone(src, dest):
|
||||
"""
|
||||
使用真实复制回放支持 reflink 的文件系统。
|
||||
"""
|
||||
shutil.copyfile(src, dest)
|
||||
return 0, ""
|
||||
|
||||
monkeypatch.setattr(Path, "write_bytes", write_probe)
|
||||
monkeypatch.setattr(SystemUtils, "is_same_disk", lambda *_args: True)
|
||||
monkeypatch.setattr(SystemUtils, "reflink", clone)
|
||||
with ThreadPoolExecutor(max_workers=2) as workers:
|
||||
results = list(workers.map(lambda index: LocalStorage._try_reflink_copy(source, tmp_path / str(index)), range(2)))
|
||||
assert results == [True, True]
|
||||
assert len(set(probes)) == 2
|
||||
assert not list(tmp_path.glob(".mode-probe-*"))
|
||||
165
tests/test_storage_review_regressions.py
Normal file
165
tests/test_storage_review_regressions.py
Normal file
@@ -0,0 +1,165 @@
|
||||
"""
|
||||
验证存储审查发现的分页、凭据和中断下载回归。
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.modules.filemanager.storages import alist as alist_module
|
||||
from app.modules.filemanager.storages.alipan import AliPan
|
||||
from app.modules.filemanager.storages.alist import Alist
|
||||
from app.schemas.file import FileItem
|
||||
|
||||
|
||||
def test_alipan_token_refresh_preserves_drive_configuration(monkeypatch):
|
||||
"""
|
||||
自动刷新只替换令牌字段,保留已授权空间和用户配置。
|
||||
"""
|
||||
storage = object.__new__(AliPan)
|
||||
configuration = {"refresh_token": "old", "access_token": "expired", "expires_in": 1,
|
||||
"refresh_time": 1, "resource_drive_id": "drive", "custom": "preserved"}
|
||||
storage._request_utils = Mock()
|
||||
monkeypatch.setattr(storage, "get_conf", lambda: dict(configuration))
|
||||
monkeypatch.setattr(storage, "set_config", lambda value: (configuration.clear(), configuration.update(value)))
|
||||
monkeypatch.setattr(storage, "_AliPan__refresh_access_token", lambda _token: {
|
||||
"access_token": "fresh", "refresh_token": "next", "expires_in": 7200,
|
||||
})
|
||||
assert storage.access_token == "fresh"
|
||||
assert configuration["resource_drive_id"] == "drive"
|
||||
assert configuration["custom"] == "preserved"
|
||||
assert configuration["refresh_token"] == "next"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response, expected", [(None, False), ({"code": "AccessDenied"}, False), ({}, True)])
|
||||
def test_alipan_delete_requires_success_response(monkeypatch, response, expected):
|
||||
"""
|
||||
删除拒绝和无响应不能落账为成功,空 JSON 是开放接口正常成功响应。
|
||||
"""
|
||||
storage = object.__new__(AliPan)
|
||||
monkeypatch.setattr(storage, "_request_api", lambda *_args, **_kwargs: response)
|
||||
assert storage.delete(FileItem(storage="alipan", fileid="file", drive_id="drive")) is expected
|
||||
|
||||
|
||||
def test_alipan_full_last_page_stops_at_empty_marker(monkeypatch):
|
||||
"""
|
||||
满页但没有下一页标记时只发送一次目录请求。
|
||||
"""
|
||||
storage = object.__new__(AliPan)
|
||||
monkeypatch.setattr(storage, "get_conf", lambda: {"resource_drive_id": "drive"})
|
||||
request = Mock(side_effect=[{"items": [{"file_id": str(i)} for i in range(100)], "next_marker": ""},
|
||||
AssertionError("末页不应重取第一页")])
|
||||
monkeypatch.setattr(storage, "_request_api", request)
|
||||
monkeypatch.setattr(storage, "_AliPan__get_fileitem", lambda info, **_kwargs: FileItem(fileid=info["file_id"]))
|
||||
assert len(storage.list(FileItem(storage="alipan", path="/", type="dir"))) == 100
|
||||
assert request.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["stream", "short", "cancel"])
|
||||
def test_openlist_incomplete_download_is_not_a_transfer_source(tmp_path, monkeypatch, failure):
|
||||
"""
|
||||
流中断、正常提前结束和取消都必须返回失败并删除未完成文件。
|
||||
"""
|
||||
storage = object.__new__(Alist)
|
||||
monkeypatch.setattr(storage, "get_conf", lambda: {"url": "https://openlist.invalid", "token": "secret"})
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"code": 200, "data": {"raw_url": "https://cdn.invalid/file", "sign": ""}}
|
||||
stream = Mock(headers={"Content-Length": "12"})
|
||||
|
||||
def chunks(**_kwargs):
|
||||
"""
|
||||
只提供部分内容,并可模拟连接中断。
|
||||
"""
|
||||
yield b"part"
|
||||
if failure == "stream":
|
||||
raise ConnectionError("stream interrupted")
|
||||
|
||||
stream.iter_content = chunks
|
||||
|
||||
@contextmanager
|
||||
def get_stream(*_args, **_kwargs):
|
||||
"""
|
||||
在网络边界回放下载流。
|
||||
"""
|
||||
yield stream
|
||||
|
||||
request = Mock()
|
||||
request.post_res.return_value = response
|
||||
request.get_stream = get_stream
|
||||
monkeypatch.setattr(alist_module, "RequestUtils", lambda **_kwargs: request)
|
||||
monkeypatch.setattr(alist_module.runtime_stop_state, "consume_transfer_stop", lambda _path: failure == "cancel")
|
||||
item = FileItem(storage="alist", path="/movie.mkv", name="movie.mkv", type="file", size=12)
|
||||
assert storage.download(item, tmp_path) is None
|
||||
assert not list(tmp_path.iterdir())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw_url", ["https://cdn.invalid/file", "https://openlist.invalid/d/file"])
|
||||
def test_openlist_download_never_forwards_access_token(tmp_path, monkeypatch, raw_url):
|
||||
"""
|
||||
文件直链和签名下载地址不携带 OpenList API 令牌,跨源重定向也没有令牌可泄漏。
|
||||
"""
|
||||
storage = object.__new__(Alist)
|
||||
monkeypatch.setattr(storage, "get_conf", lambda: {"url": "https://openlist.invalid", "token": "secret"})
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"code": 200, "data": {"raw_url": raw_url, "sign": ""}}
|
||||
configurations = []
|
||||
stream = Mock(headers={"Content-Length": "4"})
|
||||
stream.iter_content.return_value = [b"data"]
|
||||
|
||||
@contextmanager
|
||||
def get_stream(*_args, **_kwargs):
|
||||
"""
|
||||
返回完整的离线内容响应。
|
||||
"""
|
||||
yield stream
|
||||
|
||||
def request_factory(**kwargs):
|
||||
"""
|
||||
记录 API 和下载各自的请求头。
|
||||
"""
|
||||
configurations.append(kwargs)
|
||||
return SimpleNamespace(post_res=lambda *_args, **_kwargs: response, get_stream=get_stream)
|
||||
|
||||
monkeypatch.setattr(alist_module, "RequestUtils", request_factory)
|
||||
item = FileItem(storage="alist", path="/file.mkv", name="file.mkv", type="file", size=4)
|
||||
result = storage.download(item, tmp_path)
|
||||
assert result.read_bytes() == b"data"
|
||||
assert "Authorization" not in configurations[-1].get("headers", {})
|
||||
|
||||
|
||||
def test_openlist_detail_size_prevents_legacy_move_deleting_truncated_source(tmp_path, monkeypatch):
|
||||
"""
|
||||
旧文件项及无长度 HTTP 响应仍须使用详情大小阻止截断移动删除源。
|
||||
"""
|
||||
from app.modules.filemanager.transhandler import TransHandler
|
||||
storage = object.__new__(Alist)
|
||||
monkeypatch.setattr(storage, "get_conf", lambda: {"url": "https://openlist.invalid", "token": "secret"})
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"code": 200, "data": {
|
||||
"raw_url": "https://cdn.invalid/file", "sign": "", "size": 100,
|
||||
}}
|
||||
stream = Mock(headers={})
|
||||
stream.iter_content.return_value = [b"PART"]
|
||||
|
||||
@contextmanager
|
||||
def get_stream(*_args, **_kwargs):
|
||||
"""
|
||||
模拟正常结束但短于详情大小的无长度下载。
|
||||
"""
|
||||
yield stream
|
||||
|
||||
request = Mock()
|
||||
request.post_res.return_value = response
|
||||
request.get_stream = get_stream
|
||||
monkeypatch.setattr(alist_module, "RequestUtils", lambda **_kwargs: request)
|
||||
deleted = Mock(return_value=True)
|
||||
monkeypatch.setattr(storage, "delete", deleted)
|
||||
source = FileItem(storage="alist", path="/Movie.mkv", name="Movie.mkv", type="file")
|
||||
target = tmp_path / "Movie.mkv"
|
||||
result, _ = TransHandler._TransHandler__download_to_local(source, storage, target, "move")
|
||||
assert result is False
|
||||
deleted.assert_not_called()
|
||||
assert not target.exists()
|
||||
assert not list(tmp_path.glob(".mp-download-*"))
|
||||
@@ -68,13 +68,10 @@ def test_reflink_success_on_current_platform(tmp_path: Path):
|
||||
assert dest.read_bytes() == b"Modified cloned content"
|
||||
|
||||
|
||||
def test_reflink_cleans_preexisting_temp_file(tmp_path: Path):
|
||||
def test_reflink_preserves_unowned_legacy_temp_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
"""
|
||||
测试当目标临时 .mp 文件预先存在时,reflink 能先清理临时文件并成功完成克隆。
|
||||
私有暂存不清理固定旧名称,避免删除其它任务仍在写入的文件。
|
||||
"""
|
||||
if not _reflink_available(tmp_path):
|
||||
pytest.skip("当前文件系统不支持引用链接")
|
||||
|
||||
src = tmp_path / "source.txt"
|
||||
dest = tmp_path / "target.txt"
|
||||
temp_file = dest.with_suffix(dest.suffix + ".mp")
|
||||
@@ -82,11 +79,21 @@ def test_reflink_cleans_preexisting_temp_file(tmp_path: Path):
|
||||
src.write_text("source content", encoding="utf-8")
|
||||
temp_file.write_text("old stale temp content", encoding="utf-8")
|
||||
|
||||
def clone(source: Path, destination: Path):
|
||||
"""
|
||||
在真实文件上回放底层克隆,避免依赖当前文件系统的 CoW 支持。
|
||||
"""
|
||||
shutil.copyfile(source, destination)
|
||||
return 0, ""
|
||||
|
||||
monkeypatch.setattr(SystemUtils, "_reflink_copy", clone)
|
||||
|
||||
code, msg = SystemUtils.reflink(src, dest)
|
||||
assert code == 0
|
||||
assert msg == ""
|
||||
assert dest.read_text(encoding="utf-8") == "source content"
|
||||
assert not temp_file.exists()
|
||||
assert temp_file.read_text(encoding="utf-8") == "old stale temp content"
|
||||
assert not list(tmp_path.glob(".target.txt.*.mp"))
|
||||
|
||||
|
||||
def test_reflink_copy_failure_cleans_temp_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
@@ -95,10 +102,14 @@ def test_reflink_copy_failure_cleans_temp_file(tmp_path: Path, monkeypatch: pyte
|
||||
"""
|
||||
src = tmp_path / "source.txt"
|
||||
dest = tmp_path / "target.txt"
|
||||
temp_file = dest.with_suffix(dest.suffix + ".mp")
|
||||
temporary_files = []
|
||||
src.write_text("source data", encoding="utf-8")
|
||||
|
||||
def _mock_reflink_copy(_src: Path, _dest: Path):
|
||||
"""
|
||||
记录本次实际暂存路径后回放部分内容写入失败。
|
||||
"""
|
||||
temporary_files.append(_dest)
|
||||
# 模拟底层复制过程中生成了临时文件但最终报告失败
|
||||
_dest.write_text("partial data", encoding="utf-8")
|
||||
return -1, "底层克隆失败模拟错误"
|
||||
@@ -109,7 +120,7 @@ def test_reflink_copy_failure_cleans_temp_file(tmp_path: Path, monkeypatch: pyte
|
||||
assert code == -1
|
||||
assert "底层克隆失败模拟错误" in msg
|
||||
# 验证临时文件已被清理且目标未生成
|
||||
assert not temp_file.exists()
|
||||
assert temporary_files and all(not path.exists() for path in temporary_files)
|
||||
assert not dest.exists()
|
||||
|
||||
|
||||
@@ -119,12 +130,11 @@ def test_reflink_source_file_not_found(tmp_path: Path):
|
||||
"""
|
||||
src = tmp_path / "nonexistent_source.txt"
|
||||
dest = tmp_path / "target.txt"
|
||||
temp_file = dest.with_suffix(dest.suffix + ".mp")
|
||||
|
||||
code, msg = SystemUtils.reflink(src, dest)
|
||||
assert code != 0
|
||||
assert msg != ""
|
||||
assert not temp_file.exists()
|
||||
assert not list(tmp_path.glob(".target.txt.*.mp"))
|
||||
assert not dest.exists()
|
||||
|
||||
|
||||
@@ -134,14 +144,21 @@ def test_reflink_unexpected_exception_handling(tmp_path: Path, monkeypatch: pyte
|
||||
"""
|
||||
src = tmp_path / "source.txt"
|
||||
dest = tmp_path / "target.txt"
|
||||
temp_file = dest.with_suffix(dest.suffix + ".mp")
|
||||
temporary_files = []
|
||||
src.write_text("source data", encoding="utf-8")
|
||||
|
||||
def _mock_reflink_copy(_src: Path, _dest: Path):
|
||||
"""
|
||||
记录实际克隆暂存路径供异常清理断言使用。
|
||||
"""
|
||||
temporary_files.append(_dest)
|
||||
_dest.write_text("cloned data", encoding="utf-8")
|
||||
return 0, ""
|
||||
|
||||
def _mock_move(_src, _dest):
|
||||
"""
|
||||
模拟提升克隆文件失败。
|
||||
"""
|
||||
raise RuntimeError("模拟 shutil.move 移动文件异常")
|
||||
|
||||
monkeypatch.setattr(SystemUtils, "_reflink_copy", _mock_reflink_copy)
|
||||
@@ -150,7 +167,7 @@ def test_reflink_unexpected_exception_handling(tmp_path: Path, monkeypatch: pyte
|
||||
code, msg = SystemUtils.reflink(src, dest)
|
||||
assert code == -1
|
||||
assert "模拟 shutil.move 移动文件异常" in msg
|
||||
assert not temp_file.exists()
|
||||
assert temporary_files and all(not path.exists() for path in temporary_files)
|
||||
assert not dest.exists()
|
||||
|
||||
|
||||
|
||||
@@ -403,7 +403,7 @@ class _ImmediateStepRunner:
|
||||
@pytest.mark.parametrize("frozen_time,live_time", [(1.0, 1.0), (1.0, 5.0), (None, 5.0), (1.0, None)])
|
||||
def test_cross_storage_move_materializes_before_independent_source_delete(tmp_path, frozen_time, live_time):
|
||||
"""
|
||||
跨存储 move 先复制再独立删除源,源和目标时间戳变化或缺失均不阻止清理。
|
||||
跨存储 move 先复制再独立删除源,云端时间戳变化或缺失不阻止清理,本地源版本保持稳定。
|
||||
"""
|
||||
source_path = tmp_path / "source.mkv"
|
||||
source_path.write_bytes(b"movie")
|
||||
@@ -413,7 +413,7 @@ def test_cross_storage_move_materializes_before_independent_source_delete(tmp_pa
|
||||
name=source_path.name,
|
||||
type="file",
|
||||
size=source_path.stat().st_size,
|
||||
modify_time=frozen_time,
|
||||
modify_time=1.0,
|
||||
extension="mkv",
|
||||
)
|
||||
target_item = FileItem(
|
||||
@@ -422,12 +422,12 @@ def test_cross_storage_move_materializes_before_independent_source_delete(tmp_pa
|
||||
name="source.mkv",
|
||||
type="file",
|
||||
size=source_item.size,
|
||||
modify_time=source_item.modify_time,
|
||||
modify_time=frozen_time,
|
||||
extension="mkv",
|
||||
)
|
||||
source_oper = Mock()
|
||||
source_oper.delete.return_value = True
|
||||
live_source = source_item.model_copy(update={"modify_time": live_time})
|
||||
live_source = source_item.model_copy()
|
||||
source_oper.get_item_strict.return_value = live_source
|
||||
target_oper = Mock()
|
||||
target_oper.get_item_strict.return_value = target_item.model_copy(update={"modify_time": live_time})
|
||||
|
||||
256
tests/test_transfer_review_regressions.py
Normal file
256
tests/test_transfer_review_regressions.py
Normal file
@@ -0,0 +1,256 @@
|
||||
"""
|
||||
验证整理恢复、目录覆盖和上下文传递的审查回归。
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.api.endpoints.transferhistory import restore_manual_transfer_history_batch
|
||||
from app.application.transfer.execution import TransferExecutionConflictError, TransferExecutionState
|
||||
from app.application.transfer.history import success_fields
|
||||
from app.application.transfer.models import TransferPlanCheckpoint, TransferPlanItem, TransferPlanningInput
|
||||
from app.chain.scraping import ScrapingChain
|
||||
from app.chain.transfer.execution import _TransferManualReviewRequired
|
||||
from app.domain.context import MediaInfo
|
||||
from app.domain.meta.metavideo import MetaVideo
|
||||
from app.modules.filemanager.module import FileManagerModule
|
||||
from app.modules.filemanager.storages.alist import Alist
|
||||
from app.modules.filemanager.storages.local import LocalStorage
|
||||
from app.modules.filemanager.transhandler import TransHandler
|
||||
from app.schemas.file import FileItem
|
||||
from app.schemas.transfer import ManualTransferItem, TransferInfo
|
||||
from app.schemas.types import MediaType
|
||||
from tests.test_transfer_safe_overwrite import _checkpoint, _durable_runner, _execute
|
||||
from tests.test_transfer_safe_overwrite import repository_factory as _repository_factory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repository_factory():
|
||||
"""
|
||||
复用真实 SQLite 整理账本载具,每个用例结束时关闭独立引擎。
|
||||
"""
|
||||
yield from _repository_factory.__wrapped__()
|
||||
|
||||
|
||||
def test_success_history_preserves_episode_group():
|
||||
"""
|
||||
整理成功历史应与失败历史一样保存自定义剧集组。
|
||||
"""
|
||||
media = MediaInfo(type=MediaType.TV, title="Show", episode_group="group")
|
||||
result = success_fields(FileItem(path="/source"), "copy", MetaVideo("Show.S01E01"), media, TransferInfo())
|
||||
assert result["episode_group"] == "group"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("groups, expected", [(["group", "group"], "group"), (["a", "b"], None)])
|
||||
def test_batch_retransfer_restores_only_common_episode_group(groups, expected):
|
||||
"""
|
||||
批量重整只复用共同剧集组,混合批次不误用单条记录。
|
||||
"""
|
||||
histories = [SimpleNamespace(status=True, mode="copy", src_fileitem={"path": f"/{index}"},
|
||||
type="电视剧", media_source="themoviedb", media_id="123",
|
||||
music_type=None, downloader=None, download_hash=None, episode_group=group)
|
||||
for index, group in enumerate(groups)]
|
||||
request = ManualTransferItem(logids=list(range(len(groups))), from_history=True)
|
||||
result = restore_manual_transfer_history_batch(request, SimpleNamespace(get=lambda identifier: histories[identifier]))
|
||||
assert result[-1] is None
|
||||
assert request.episode_group == expected
|
||||
|
||||
|
||||
def test_episode_scraping_preserves_explicit_source(monkeypatch):
|
||||
"""
|
||||
单集重新识别后继续使用本次请求指定的刮削来源。
|
||||
"""
|
||||
media = MediaInfo(type=MediaType.TV, media_source="themoviedb", media_id="123", scrape_source="imdb")
|
||||
recognized = MediaInfo(type=MediaType.TV, media_source="themoviedb", media_id="123")
|
||||
monkeypatch.setattr("app.chain.scraping.MediaChain", lambda: SimpleNamespace(recognize_media=lambda **_kwargs: recognized))
|
||||
chain = object.__new__(ScrapingChain)
|
||||
nfo, images = Mock(), Mock()
|
||||
monkeypatch.setattr(chain, "_scrape_nfo_generic", nfo)
|
||||
monkeypatch.setattr(chain, "_scrape_images_generic", images)
|
||||
chain._handle_tv_episode_file(FileItem(path="/Show.S01E01.mkv"), Path("/Show.S01E01.mkv"), media, None, False)
|
||||
assert nfo.call_args.kwargs["mediainfo"].scrape_source == "imdb"
|
||||
assert images.call_args.kwargs["mediainfo"].scrape_source == "imdb"
|
||||
|
||||
|
||||
def test_local_exists_includes_all_episodes(monkeypatch):
|
||||
"""
|
||||
合并集媒体应将文件中声明的全部剧集加入本地查重结果。
|
||||
"""
|
||||
module = object.__new__(FileManagerModule)
|
||||
monkeypatch.setattr("app.modules.filemanager.module.get_runtime_setting", lambda *_args: True)
|
||||
monkeypatch.setattr(module, "media_files", lambda _media: [FileItem(basename="Show.S01E01-E02")])
|
||||
assert module.media_exists(MediaInfo(type=MediaType.TV, title="Show")).seasons == {1: [1, 2]}
|
||||
|
||||
|
||||
def test_directory_never_overwrite_preserves_library_and_source(tmp_path, monkeypatch):
|
||||
"""
|
||||
原盘目录的 never 策略必须跳过覆盖,且不能误触发移动源清理。
|
||||
"""
|
||||
source, target = tmp_path / "disc", tmp_path / "library"
|
||||
source.mkdir()
|
||||
target.mkdir()
|
||||
leaf, existing = source / "00001.m2ts", target / "00001.m2ts"
|
||||
leaf.write_bytes(b"new")
|
||||
existing.write_bytes(b"old-library")
|
||||
storage = object.__new__(LocalStorage)
|
||||
item = storage.get_item_strict(source).model_dump(mode="json")
|
||||
checkpoint = replace(_checkpoint(leaf, existing, transfer_type="move", mode="never"),
|
||||
planning_input=TransferPlanningInput(source_fileitem=item), final_target_path=str(target),
|
||||
items=(TransferPlanItem(sequence=0, source_fileitem=storage.get_item_strict(leaf).model_dump(mode="json"),
|
||||
target_storage="local", target_path=str(existing)),))
|
||||
result = _execute(checkpoint, monkeypatch, None)
|
||||
assert existing.read_bytes() == b"old-library"
|
||||
assert leaf.read_bytes() == b"new"
|
||||
assert not result.success and result.overwrite_skipped
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["source", "target"])
|
||||
def test_recovery_retains_all_copies_after_same_size_mutation(tmp_path, monkeypatch, repository_factory, mutation):
|
||||
"""
|
||||
提升落账后发生等长原地修改,恢复必须隔离任务并保留源和备份。
|
||||
"""
|
||||
source, target = tmp_path / "download.mkv", tmp_path / "Movie.2026.mkv"
|
||||
source.write_bytes(b"new-version")
|
||||
target.write_bytes(b"old-version")
|
||||
checkpoint = _checkpoint(source, target, transfer_type="move")
|
||||
repository = repository_factory(checkpoint)
|
||||
runner = _durable_runner(repository, checkpoint)
|
||||
real_run = runner.run
|
||||
|
||||
class ProcessStopped(BaseException):
|
||||
"""
|
||||
模拟删除源之前进程退出,不让执行器转译成普通失败。
|
||||
"""
|
||||
|
||||
def stop_before_cleanup(**kwargs):
|
||||
"""
|
||||
在提升已有持久回执后中断。
|
||||
"""
|
||||
if kwargs["kind"] == "delete_overwrite_source":
|
||||
raise ProcessStopped()
|
||||
return real_run(**kwargs)
|
||||
|
||||
monkeypatch.setattr(runner, "run", stop_before_cleanup)
|
||||
with pytest.raises(ProcessStopped):
|
||||
_execute(checkpoint, monkeypatch, runner)
|
||||
backup = next(tmp_path.glob(".mp-backup-*"))
|
||||
changed = source if mutation == "source" else target
|
||||
before = changed.stat()
|
||||
changed.write_bytes(b"NEW-CONTENT" if mutation == "source" else b"CORRUPTION!")
|
||||
os.utime(changed, ns=(before.st_atime_ns, before.st_mtime_ns + 1))
|
||||
with pytest.raises(_TransferManualReviewRequired):
|
||||
_execute(checkpoint, monkeypatch, _durable_runner(repository, checkpoint))
|
||||
assert repository.get_snapshot(task_id="safe-overwrite").state is TransferExecutionState.MANUAL_REVIEW
|
||||
assert source.exists() and backup.exists()
|
||||
assert backup.read_bytes() == b"old-version"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("storage", ["u115", "alipan", "alist"])
|
||||
def test_cloud_cleanup_identity_allows_timestamp_changes(storage):
|
||||
"""
|
||||
云端大小与稳定 ID 相同时保留上传后时间戳变化的兼容合同。
|
||||
"""
|
||||
expected = FileItem(storage=storage, path="/file", type="file", size=10, fileid="id", modify_time=1.0)
|
||||
TransHandler._TransHandler__assert_file_identity(expected, expected.model_copy(update={"modify_time": 5.0}))
|
||||
|
||||
|
||||
def test_local_cleanup_rejects_mutation_with_restored_mtime(tmp_path):
|
||||
"""
|
||||
原地改写后还原 mtime 仍会改变 ctime,本地清理必须拒绝这类版本变化。
|
||||
"""
|
||||
path = tmp_path / "file"
|
||||
path.write_bytes(b"old")
|
||||
local = object.__new__(LocalStorage)
|
||||
expected = local.get_item_strict(path)
|
||||
before = path.stat()
|
||||
time.sleep(0.02)
|
||||
path.write_bytes(b"new")
|
||||
os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns))
|
||||
current = local.get_item_strict(path)
|
||||
assert expected.modify_time_ns == current.modify_time_ns
|
||||
assert expected.change_time_ns != current.change_time_ns
|
||||
with pytest.raises(TransferExecutionConflictError, match="状态时间已变化"):
|
||||
TransHandler._TransHandler__assert_file_identity(expected, current)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["exception", "detail_size_only"])
|
||||
def test_openlist_truncated_move_never_deletes_remote_source(tmp_path, monkeypatch, repository_factory, failure):
|
||||
"""
|
||||
真实下载与持久步骤执行器遇到截断流时保留远端原件,不发布本地损坏文件。
|
||||
"""
|
||||
source = object.__new__(Alist)
|
||||
target = object.__new__(LocalStorage)
|
||||
remote = FileItem(storage="alist", path="/download/Movie.mkv", name="Movie.mkv",
|
||||
basename="Movie", extension="mkv", type="file",
|
||||
size=None if failure == "detail_size_only" else 100, modify_time=1.0)
|
||||
payload = remote.model_dump(mode="json")
|
||||
destination = tmp_path / "Movie.mkv"
|
||||
checkpoint = TransferPlanCheckpoint(
|
||||
planning_input=TransferPlanningInput(source_fileitem=payload), target_storage="local",
|
||||
root_target_path=str(tmp_path), final_target_path=str(destination), resolved_transfer_type="move",
|
||||
overwrite_mode="never", items=(TransferPlanItem(sequence=0, source_fileitem=payload,
|
||||
target_storage="local", target_path=str(destination)),),
|
||||
)
|
||||
repository = repository_factory(checkpoint)
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = {"code": 200, "data": {
|
||||
"raw_url": "https://cdn.invalid/file", "sign": "", "size": 100,
|
||||
}}
|
||||
stream = Mock(headers={} if failure == "detail_size_only" else {"Content-Length": "100"})
|
||||
|
||||
def chunks(**_kwargs):
|
||||
"""
|
||||
在远端下载传输边界回放部分写入后的连接中断。
|
||||
"""
|
||||
yield b"PART"
|
||||
if failure == "exception":
|
||||
raise ConnectionError("interrupted")
|
||||
|
||||
@contextmanager
|
||||
def download_stream(*_args, **_kwargs):
|
||||
"""
|
||||
保留真实下载代码,只替换 HTTP 响应。
|
||||
"""
|
||||
yield stream
|
||||
|
||||
stream.iter_content = chunks
|
||||
monkeypatch.setattr(source, "get_conf", lambda: {"url": "https://openlist.invalid", "token": "secret"})
|
||||
monkeypatch.setattr(RequestUtils, "post_res", lambda *_args, **_kwargs: response)
|
||||
monkeypatch.setattr(RequestUtils, "get_stream", download_stream)
|
||||
deleted = Mock(return_value=True)
|
||||
monkeypatch.setattr(source, "delete", deleted)
|
||||
handler = TransHandler()
|
||||
monkeypatch.setattr(handler, "_TransHandler__intercept_transfer", lambda **_kwargs: (True, ""))
|
||||
with pytest.raises(RuntimeError, match="下载"):
|
||||
handler.execute_transfer_plan(
|
||||
checkpoint, meta=MetaVideo("Movie.2026.mkv"), mediainfo=MediaInfo(type=MediaType.MOVIE, title="Movie"),
|
||||
source_oper=source, target_oper=target, step_runner=_durable_runner(repository, checkpoint),
|
||||
)
|
||||
deleted.assert_not_called()
|
||||
assert not destination.exists()
|
||||
assert not list(tmp_path.glob(".mp-download-*"))
|
||||
assert all(step.kind != "delete_move_source" for step in repository.get_snapshot(task_id="safe-overwrite").steps)
|
||||
|
||||
|
||||
def test_local_timestamp_json_roundtrip_keeps_nanosecond_precision():
|
||||
"""
|
||||
纳秒版本证据超过浏览器整数精度时以字符串无损传输,回传后恢复为 Python 整数。
|
||||
"""
|
||||
stamp = 1_792_691_575_120_037_153
|
||||
item = FileItem(modify_time_ns=stamp, change_time_ns=stamp + 1)
|
||||
payload = item.model_dump(mode="json")
|
||||
assert payload["modify_time_ns"] == str(stamp)
|
||||
assert payload["change_time_ns"] == str(stamp + 1)
|
||||
restored = FileItem.model_validate(payload)
|
||||
assert restored.modify_time_ns == stamp
|
||||
assert restored.change_time_ns == stamp + 1
|
||||
assert restored.model_dump()["modify_time_ns"] == stamp
|
||||
assert FileItem().model_dump(mode="json")["modify_time_ns"] is None
|
||||
@@ -1022,11 +1022,11 @@ def test_completed_promotion_rechecks_live_files_before_destructive_cleanup(
|
||||
assert backup.read_bytes() == expected_backup
|
||||
|
||||
|
||||
def test_completed_promotion_allows_cleanup_after_timestamp_changes(
|
||||
def test_completed_local_promotion_retains_copies_after_timestamp_changes(
|
||||
tmp_path, monkeypatch, repository_factory,
|
||||
):
|
||||
"""
|
||||
提升结果已落账后,即使源、目标和备份的时间戳均变化,恢复仍按大小和身份完成清理。
|
||||
本地提升落账后时间戳改变意味着内容版本不确定,恢复须保留所有副本。
|
||||
"""
|
||||
source, target = tmp_path / "download.mkv", tmp_path / "Movie.2026.mkv"
|
||||
source.write_bytes(b"new-version")
|
||||
@@ -1057,11 +1057,11 @@ def test_completed_promotion_allows_cleanup_after_timestamp_changes(
|
||||
before = path.stat()
|
||||
os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns + 4_000_000_000))
|
||||
|
||||
result = _execute(checkpoint, monkeypatch, _durable_runner(repository, checkpoint))
|
||||
assert result.success
|
||||
with pytest.raises(execution_module._TransferManualReviewRequired):
|
||||
_execute(checkpoint, monkeypatch, _durable_runner(repository, checkpoint))
|
||||
assert target.read_bytes() == b"new-version"
|
||||
assert not source.exists()
|
||||
assert not backup.exists()
|
||||
assert source.read_bytes() == b"new-version"
|
||||
assert backup.read_bytes() == b"old-version"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_evidence", ["source", "target"])
|
||||
|
||||
Reference in New Issue
Block a user