mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-03 22:04:57 +08:00
fix(anilist): persist Chinese titles and use official API
This commit is contained in:
@@ -2,12 +2,16 @@
|
||||
AniList GraphQL 客户端,统一查询计划、中文标题补充及同步异步缓存。
|
||||
"""
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from math import isfinite
|
||||
from time import time
|
||||
from typing import Any, Callable, Optional, Protocol, TypeVar, cast
|
||||
|
||||
from app.adapters.network.http import AsyncRequestUtils, RequestUtils
|
||||
from app.runtime.cache import cached
|
||||
from app.foundation.url import UrlUtils
|
||||
from app.runtime.cache import AsyncFileCache, FileCache, async_cache_mutation, cache_mutation, cached
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
@@ -88,17 +92,21 @@ class _AniListRequestPlan:
|
||||
|
||||
class AniListApi:
|
||||
"""
|
||||
通过中文代理或官方 GraphQL 接口查询 AniList,并补充中文标题。
|
||||
通过官方 GraphQL 接口查询 AniList,并补充本地缓存的中文标题。
|
||||
|
||||
代理失败后该实例改用官方接口;中文数据集在实例内缓存。
|
||||
中文数据集每天更新,下载失败时保留旧数据并在五分钟后重试。
|
||||
"""
|
||||
|
||||
_base_url = "https://trace.moe/anilist/"
|
||||
_official_url = "https://graphql.anilist.co"
|
||||
_translations_url = (
|
||||
"https://raw.githubusercontent.com/soruly/anilist-chinese/"
|
||||
"master/anilist-chinese.json"
|
||||
)
|
||||
# 上游中文数据集每天更新,磁盘和内存沿用同一有效期(秒)。
|
||||
_translations_ttl = 24 * 60 * 60
|
||||
_translations_retry = 5 * 60
|
||||
_translations_region = "anilist"
|
||||
_translations_key = "anilist-chinese.json"
|
||||
_media_summary_fields = """
|
||||
id
|
||||
idMal
|
||||
@@ -245,8 +253,16 @@ class AniListApi:
|
||||
proxies=get_runtime_setting('PROXY'),
|
||||
headers=headers,
|
||||
)
|
||||
self._proxy_available = True
|
||||
self._translations: Optional[dict[object, dict[str, Any]]] = None
|
||||
self._translations_expires_at = 0.0
|
||||
self._file_cache = None
|
||||
self._async_file_cache = None
|
||||
try:
|
||||
cache_path = get_runtime_setting('CACHE_PATH')
|
||||
self._file_cache = FileCache(base=cache_path, local_only=True)
|
||||
self._async_file_cache = AsyncFileCache(base=cache_path, local_only=True)
|
||||
except OSError as err:
|
||||
logger.warning(f"初始化 anilist-chinese 磁盘缓存失败:{err}")
|
||||
|
||||
@staticmethod
|
||||
def _extract_response(response: Optional[_GraphQLResponse]) -> Optional[dict[str, Any]]:
|
||||
@@ -295,43 +311,28 @@ class AniListApi:
|
||||
"""
|
||||
请求 GraphQL 并合并中文标题数据。
|
||||
|
||||
优先使用中文代理;失败后在当前实例内禁用该代理并回退官方接口。
|
||||
|
||||
:param query: GraphQL 查询文本
|
||||
:param variables: 本次查询的变量映射
|
||||
:return: 经中文字段补充的 data 映射;请求或 GraphQL 业务失败时为 None
|
||||
"""
|
||||
plan = self._request_plan(query, variables)
|
||||
payload = {"query": plan.query, "variables": plan.variables}
|
||||
if self._proxy_available:
|
||||
response = self._request.post_res(self._base_url, json=payload)
|
||||
result = self._extract_response(response)
|
||||
if result is not None:
|
||||
return self._project_translated(result, self._translation_map())
|
||||
self._disable_proxy(response)
|
||||
response = self._request.post_res(self._official_url, json=payload)
|
||||
result = self._extract_response(response)
|
||||
if not result:
|
||||
return result
|
||||
return self._project_translated(result, self._translation_map())
|
||||
|
||||
async def _async_invoke(self, query: str, variables: dict[str, Any]) -> Optional[dict[str, Any]]:
|
||||
"""
|
||||
异步请求 GraphQL 并合并中文标题数据。
|
||||
|
||||
优先使用中文代理;失败后在当前实例内禁用该代理并回退官方接口。
|
||||
|
||||
:param query: GraphQL 查询文本
|
||||
:param variables: 本次查询的变量映射
|
||||
:return: 经中文字段补充的 data 映射;请求或 GraphQL 业务失败时为 None
|
||||
"""
|
||||
plan = self._request_plan(query, variables)
|
||||
payload = {"query": plan.query, "variables": plan.variables}
|
||||
if self._proxy_available:
|
||||
response = await self._async_request.post_res(self._base_url, json=payload)
|
||||
result = self._extract_response(response)
|
||||
if result is not None:
|
||||
translations = await self._async_translation_map()
|
||||
return self._project_translated(result, translations)
|
||||
self._disable_proxy(response)
|
||||
response = await self._async_request.post_res(self._official_url, json=payload)
|
||||
result = self._extract_response(response)
|
||||
if not result:
|
||||
@@ -339,19 +340,6 @@ class AniListApi:
|
||||
translations = await self._async_translation_map()
|
||||
return self._project_translated(result, translations)
|
||||
|
||||
def _disable_proxy(self, response: object) -> None:
|
||||
"""
|
||||
标记中文代理不可用,避免当前进程持续请求已失效的上游。
|
||||
|
||||
:param response: 中文代理响应对象
|
||||
"""
|
||||
self._proxy_available = False
|
||||
status_code = getattr(response, "status_code", None)
|
||||
logger.warning(
|
||||
f"anilist-chinese 代理不可用(HTTP {status_code}),"
|
||||
"改用 AniList 官方接口并合并中文数据集"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_translation_map(items: object) -> dict[object, dict[str, Any]]:
|
||||
"""
|
||||
@@ -365,34 +353,137 @@ class AniListApi:
|
||||
return {
|
||||
item.get("id"): item
|
||||
for item in items
|
||||
if isinstance(item, dict) and item.get("id")
|
||||
if isinstance(item, dict)
|
||||
and isinstance(item.get("id"), (int, str)) and item.get("id")
|
||||
and isinstance(item.get("title"), str)
|
||||
and isinstance(item.get("synonyms", []), list)
|
||||
and all(isinstance(alias, str) for alias in item.get("synonyms", []))
|
||||
}
|
||||
|
||||
def _translation_routes(self) -> list[tuple[str, bool]]:
|
||||
"""
|
||||
返回中文数据集的下载地址及直连标志,GitHub 镜像优先且不叠加部署代理。
|
||||
|
||||
:return: 镜像和原地址的有序列表,原地址沿用 PROXY
|
||||
"""
|
||||
mirror = str(get_runtime_setting('GITHUB_PROXY') or '').strip()
|
||||
routes = [(self._translations_url, False)]
|
||||
if mirror:
|
||||
routes.insert(0, (f"{UrlUtils.standardize_base_url(mirror)}{self._translations_url}", True))
|
||||
return routes
|
||||
|
||||
@classmethod
|
||||
def _decode_translation_cache(cls, content: Optional[bytes]) -> tuple[dict[object, dict[str, Any]], float]:
|
||||
"""
|
||||
校验磁盘快照并读取数据和绝对过期时间,损坏快照按未命中处理。
|
||||
"""
|
||||
if content:
|
||||
try:
|
||||
payload = json.loads(content)
|
||||
if isinstance(payload, dict):
|
||||
expires_at = payload.get("expires_at")
|
||||
if isinstance(expires_at, (int, float)) and isfinite(expires_at):
|
||||
return cls._build_translation_map(payload.get("items")), float(expires_at)
|
||||
except (ValueError, TypeError) as err:
|
||||
logger.warning(f"解析 anilist-chinese 磁盘缓存失败:{err}")
|
||||
return {}, 0.0
|
||||
|
||||
def _translation_snapshot(self, items: object) -> Optional[bytes]:
|
||||
"""
|
||||
接收有效下载并生成磁盘快照;无效载荷不得覆盖内存或旧快照。
|
||||
"""
|
||||
translations = self._build_translation_map(items)
|
||||
if not translations:
|
||||
return None
|
||||
self._translations = translations
|
||||
self._translations_expires_at = time() + self._translations_ttl
|
||||
return json.dumps({
|
||||
"expires_at": self._translations_expires_at,
|
||||
"items": list(translations.values()),
|
||||
}, ensure_ascii=False).encode('utf-8')
|
||||
|
||||
def _reuse_translation_cache(self, translations: dict[object, dict[str, Any]], expires_at: float) -> bool:
|
||||
"""
|
||||
恢复磁盘缓存;内存仍有效时优先复用,过期数据仅保留为下载失败的回退。
|
||||
"""
|
||||
if self._translations is not None and time() < self._translations_expires_at:
|
||||
return True
|
||||
if translations and (not self._translations or expires_at > self._translations_expires_at):
|
||||
self._translations = translations
|
||||
self._translations_expires_at = expires_at
|
||||
return time() < expires_at
|
||||
return False
|
||||
|
||||
def _translation_fallback(self) -> dict[object, dict[str, Any]]:
|
||||
"""
|
||||
下载失败时保留旧中文数据,短暂退避后允许重试,不把空结果永久缓存。
|
||||
"""
|
||||
logger.warning("加载 anilist-chinese 中文数据集失败,暂用已有数据,五分钟后重试")
|
||||
self._translations = self._translations or {}
|
||||
self._translations_expires_at = time() + self._translations_retry
|
||||
return self._translations
|
||||
|
||||
def _translation_map(self) -> dict[object, dict[str, Any]]:
|
||||
"""
|
||||
同步加载并复用 anilist-chinese 中文标题数据。
|
||||
同步复用中文数据快照,仅在缺失或过期时下载;并发调用共用缓存写锁。
|
||||
|
||||
:return: 中文标题数据索引
|
||||
"""
|
||||
if self._translations is None:
|
||||
items = self._request.get_json(self._translations_url)
|
||||
self._translations = self._build_translation_map(items)
|
||||
if not self._translations:
|
||||
logger.warning("加载 anilist-chinese 中文数据集失败")
|
||||
return self._translations
|
||||
with cache_mutation("anilist-translations"):
|
||||
if self._reuse_translation_cache({}, 0.0):
|
||||
return self._translations or {}
|
||||
content = None
|
||||
try:
|
||||
if self._file_cache is not None:
|
||||
content = self._file_cache.get(self._translations_key, region=self._translations_region)
|
||||
except OSError as err:
|
||||
logger.warning(f"读取 anilist-chinese 磁盘缓存失败:{err}")
|
||||
if self._reuse_translation_cache(*self._decode_translation_cache(content)):
|
||||
return self._translations or {}
|
||||
for url, direct in self._translation_routes():
|
||||
request = RequestUtils(ua=get_runtime_setting('NORMAL_USER_AGENT')) if direct else self._request
|
||||
snapshot = self._translation_snapshot(request.get_json(url))
|
||||
if snapshot is None:
|
||||
continue
|
||||
try:
|
||||
if self._file_cache is not None:
|
||||
self._file_cache.set(self._translations_key, snapshot, region=self._translations_region)
|
||||
except OSError as err:
|
||||
logger.warning(f"写入 anilist-chinese 磁盘缓存失败:{err}")
|
||||
return self._translations or {}
|
||||
return self._translation_fallback()
|
||||
|
||||
async def _async_translation_map(self) -> dict[object, dict[str, Any]]:
|
||||
"""
|
||||
异步加载并复用 anilist-chinese 中文标题数据。
|
||||
异步复用同一磁盘快照和写锁,文件读写不阻塞事件循环。
|
||||
|
||||
:return: 中文标题数据索引
|
||||
"""
|
||||
if self._translations is None:
|
||||
items = await self._async_request.get_json(self._translations_url)
|
||||
self._translations = self._build_translation_map(items)
|
||||
if not self._translations:
|
||||
logger.warning("加载 anilist-chinese 中文数据集失败")
|
||||
return self._translations
|
||||
async with async_cache_mutation("anilist-translations"):
|
||||
if self._reuse_translation_cache({}, 0.0):
|
||||
return self._translations or {}
|
||||
content = None
|
||||
try:
|
||||
if self._async_file_cache is not None:
|
||||
content = await self._async_file_cache.get(self._translations_key, region=self._translations_region)
|
||||
except OSError as err:
|
||||
logger.warning(f"读取 anilist-chinese 磁盘缓存失败:{err}")
|
||||
if self._reuse_translation_cache(*self._decode_translation_cache(content)):
|
||||
return self._translations or {}
|
||||
for url, direct in self._translation_routes():
|
||||
request = AsyncRequestUtils(ua=get_runtime_setting('NORMAL_USER_AGENT')) if direct else self._async_request
|
||||
snapshot = self._translation_snapshot(await request.get_json(url))
|
||||
if snapshot is None:
|
||||
continue
|
||||
try:
|
||||
if self._async_file_cache is not None:
|
||||
await self._async_file_cache.set(
|
||||
self._translations_key, snapshot, region=self._translations_region,
|
||||
)
|
||||
except OSError as err:
|
||||
logger.warning(f"写入 anilist-chinese 磁盘缓存失败:{err}")
|
||||
return self._translations or {}
|
||||
return self._translation_fallback()
|
||||
|
||||
@classmethod
|
||||
def _inject_chinese(cls, value: _Result, translations: dict[object, dict[str, Any]]) -> _Result:
|
||||
@@ -567,7 +658,7 @@ class AniListApi:
|
||||
|
||||
def _medias_by_ids(self, media_ids: list[int]) -> list[dict[str, Any]]:
|
||||
"""
|
||||
通过根级 Page.media 批量查询媒体,使中文代理能够注入标题。
|
||||
通过根级 Page.media 批量查询媒体并补充中文标题。
|
||||
|
||||
:param media_ids: AniList 媒体 ID 列表
|
||||
:return: 按输入顺序排列的媒体列表
|
||||
@@ -583,7 +674,7 @@ class AniListApi:
|
||||
|
||||
async def _async_medias_by_ids(self, media_ids: list[int]) -> list[dict[str, Any]]:
|
||||
"""
|
||||
异步通过根级 Page.media 批量查询媒体,使中文代理能够注入标题。
|
||||
异步通过根级 Page.media 批量查询媒体并补充中文标题。
|
||||
|
||||
:param media_ids: AniList 媒体 ID 列表
|
||||
:return: 按输入顺序排列的媒体列表
|
||||
|
||||
@@ -817,7 +817,7 @@ flowchart LR
|
||||
| 指标 | 当前值 |
|
||||
|---|---:|
|
||||
| Python 模块 | 1144 |
|
||||
| 内部导入边 | 9,887 |
|
||||
| 内部导入边 | 9,889 |
|
||||
| 非平凡 SCC | 1(29 个模块的 TMDB vendor 精确 containment) |
|
||||
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
|
||||
| Direct egress | 51(精确 policy 准入,Application / Chain 均为 0) |
|
||||
|
||||
@@ -87,6 +87,13 @@ AppPush 的所有用户范围由 `tests/test_notification_routing_parity.py` 验
|
||||
|
||||
## 外部依赖:一律 mock,零真实网络
|
||||
|
||||
AniList 直接访问官方 GraphQL 接口,不再探测 `trace.moe/anilist/`。中文数据集保存在
|
||||
`<CONFIG_DIR>/cache/anilist/anilist-chinese.json`,有效期 24 小时,同步、异步查询和重建客户端
|
||||
共用快照;配置 `GITHUB_PROXY` 时先直连镜像,失败再按 `PROXY` 访问原始 GitHub 地址。
|
||||
下载失败保留过期数据并在五分钟后重试,目录或文件读写失败时仍可使用已下载的内存数据。
|
||||
`tests/test_anilist_translations.py` 使用临时目录和下载替身验证重建客户端后的缓存复用、到期刷新、
|
||||
坏缓存、磁盘故障、镜像回退、同步异步并发去重及取消后释放锁;不据此推断真实代理耗时。
|
||||
|
||||
高清杜比换域回归由 `tests/test_hddolby_spider.py`、`tests/test_hddolby_parser.py` 和
|
||||
`tests/test_site_connectivity.py` 验证:网页使用 `www.uhddolby.com` 时,搜索、用户资料和
|
||||
做种统计仍可按资源的 `api_url` 访问 `api.hddolby.com`,并保留站点代理和认证配置。
|
||||
|
||||
@@ -959,8 +959,8 @@
|
||||
"runtime_only": true
|
||||
}
|
||||
},
|
||||
"edge_count": 9887,
|
||||
"edge_sha256": "a6ce492847e6336c039524d2b4624ea88b8acdab99db9ab0825cdae99e703f66",
|
||||
"edge_count": 9889,
|
||||
"edge_sha256": "99db09bea0a36fa387eb552e2f97ca96ab6c289ebbab2305c7931f53fa0796f1",
|
||||
"edges": [
|
||||
"app -> app.foundation",
|
||||
"app -> app.foundation.environment",
|
||||
@@ -7301,6 +7301,8 @@
|
||||
"app.modules.anilist.anilist -> app.adapters",
|
||||
"app.modules.anilist.anilist -> app.adapters.network",
|
||||
"app.modules.anilist.anilist -> app.adapters.network.http",
|
||||
"app.modules.anilist.anilist -> app.foundation",
|
||||
"app.modules.anilist.anilist -> app.foundation.url",
|
||||
"app.modules.anilist.anilist -> app.runtime",
|
||||
"app.modules.anilist.anilist -> app.runtime.cache",
|
||||
"app.modules.anilist.anilist -> app.runtime.log",
|
||||
|
||||
@@ -1,17 +1,31 @@
|
||||
"""
|
||||
验证 AniList 浏览筛选、中文代理回退及季节参数映射。
|
||||
验证 AniList 浏览筛选、官方查询的中文合并及季节参数映射。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api.endpoints import anilist as anilist_endpoint
|
||||
from app.domain.context import MediaInfo
|
||||
from app.modules.anilist import AniListModule
|
||||
from app.modules.anilist import anilist as anilist_client
|
||||
from app.modules.anilist.anilist import AniListApi
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_translations(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""
|
||||
每个用例使用独立缓存目录及原始下载地址,防止中文数据跨用例串扰。
|
||||
"""
|
||||
runtime_setting = anilist_client.get_runtime_setting
|
||||
values = {"CACHE_PATH": tmp_path, "GITHUB_PROXY": None}
|
||||
monkeypatch.setattr(anilist_client, "get_runtime_setting", lambda name: values[name] if name in values else runtime_setting(name))
|
||||
|
||||
|
||||
def _media_info(anilist_id: int = 154587) -> dict:
|
||||
"""
|
||||
构造 AniList 榜单测试媒体。
|
||||
@@ -37,9 +51,9 @@ def _media_info(anilist_id: int = 154587) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def test_anilist_client_uses_chinese_proxy_and_forwards_discover_filters() -> None:
|
||||
def test_anilist_client_forwards_discover_filters() -> None:
|
||||
"""
|
||||
AniList 探索应通过中文代理并原样传递组合过滤条件。
|
||||
AniList 探索应原样传递组合过滤条件。
|
||||
"""
|
||||
client = AniListApi()
|
||||
client._invoke = Mock(return_value={"Page": {"media": [_media_info()]}})
|
||||
@@ -57,7 +71,6 @@ def test_anilist_client_uses_chinese_proxy_and_forwards_discover_filters() -> No
|
||||
sort="SCORE_DESC",
|
||||
)
|
||||
|
||||
assert client._base_url == "https://trace.moe/anilist/"
|
||||
assert medias[0]["id"] == 154587
|
||||
variables = client._invoke.call_args.args[1]
|
||||
assert variables == {
|
||||
@@ -76,11 +89,10 @@ def test_anilist_client_uses_chinese_proxy_and_forwards_discover_filters() -> No
|
||||
assert "synonyms" in client._page_query
|
||||
|
||||
|
||||
def test_anilist_client_falls_back_and_merges_chinese_dataset() -> None:
|
||||
def test_anilist_client_queries_official_and_merges_chinese_dataset() -> None:
|
||||
"""
|
||||
中文代理不可用时应回退官方接口并合并项目中文数据。
|
||||
直接请求官方接口并合并项目中文数据,避免探测已失效的中文代理。
|
||||
"""
|
||||
proxy_response = Mock(status_code=403)
|
||||
official_media = _media_info()
|
||||
official_media["title"].pop("chinese")
|
||||
official_media["synonyms"] = ["Official Alias"]
|
||||
@@ -90,7 +102,7 @@ def test_anilist_client_falls_back_and_merges_chinese_dataset() -> None:
|
||||
}
|
||||
client = AniListApi()
|
||||
client._request = Mock()
|
||||
client._request.post_res.side_effect = [proxy_response, official_response]
|
||||
client._request.post_res.return_value = official_response
|
||||
client._request.get_json.return_value = [
|
||||
{
|
||||
"id": 154587,
|
||||
@@ -107,18 +119,15 @@ def test_anilist_client_falls_back_and_merges_chinese_dataset() -> None:
|
||||
"Frieren at the Funeral",
|
||||
]
|
||||
assert [call.args[0] for call in client._request.post_res.call_args_list] == [
|
||||
client._base_url,
|
||||
client._official_url,
|
||||
]
|
||||
client._request.get_json.assert_called_once_with(client._translations_url)
|
||||
assert client._proxy_available is False
|
||||
|
||||
|
||||
def test_anilist_async_client_falls_back_without_real_network() -> None:
|
||||
def test_anilist_async_client_queries_official_without_real_network() -> None:
|
||||
"""
|
||||
异步中文代理失败时也应使用官方接口和同一中文数据集。
|
||||
异步查询也应直接使用官方接口和同一中文数据集。
|
||||
"""
|
||||
proxy_response = Mock(status_code=403)
|
||||
official_media = _media_info()
|
||||
official_media["title"].pop("chinese")
|
||||
official_response = Mock(status_code=200)
|
||||
@@ -128,7 +137,7 @@ def test_anilist_async_client_falls_back_without_real_network() -> None:
|
||||
client = AniListApi()
|
||||
client._async_request = Mock()
|
||||
client._async_request.post_res = AsyncMock(
|
||||
side_effect=[proxy_response, official_response]
|
||||
return_value=official_response
|
||||
)
|
||||
client._async_request.get_json = AsyncMock(
|
||||
return_value=[
|
||||
@@ -140,7 +149,6 @@ def test_anilist_async_client_falls_back_without_real_network() -> None:
|
||||
|
||||
assert result["Page"]["media"][0]["title"]["chinese"] == "葬送的芙莉莲"
|
||||
assert [call.args[0] for call in client._async_request.post_res.call_args_list] == [
|
||||
client._base_url,
|
||||
client._official_url,
|
||||
]
|
||||
client._async_request.get_json.assert_awaited_once_with(client._translations_url)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
AniList 的无效 GraphQL 响应不得阻止官方回退与中文合并。
|
||||
AniList 的无效 GraphQL 响应应返回失败,中文合并保留原始身份与别名顺序。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -11,39 +11,31 @@ from app.modules.anilist.anilist import AniListApi
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("proxy_payload", [
|
||||
@pytest.mark.parametrize("payload", [
|
||||
None, [], [1], "invalid", 7, False,
|
||||
{"data": []}, {"data": [1]}, {"data": "invalid"}, {"data": 7},
|
||||
{"data": False}, {"errors": ["unavailable"], "data": {"Media": {"id": 1}}},
|
||||
])
|
||||
def test_invalid_proxy_payload_preserves_official_fallback(asynchronous, proxy_payload):
|
||||
def test_invalid_official_payload_does_not_download_translations(asynchronous: bool, payload: object) -> None:
|
||||
"""
|
||||
无效对象、无效 data 和 GraphQL 错误都应触发既有官方查询路径。
|
||||
无效对象、无效 data 和 GraphQL 错误不得触发数据集下载或再次查询。
|
||||
"""
|
||||
proxy = Mock(status_code=200)
|
||||
proxy.json.return_value = proxy_payload
|
||||
official = Mock(status_code=200)
|
||||
media = {"id": 12, "title": {"native": "Original"}, "synonyms": ["Alias"]}
|
||||
data = {"Media": media}
|
||||
official.json.return_value = {"data": data}
|
||||
request = AsyncMock(side_effect=[proxy, official]) if asynchronous else Mock(side_effect=[proxy, official])
|
||||
translations = [{"id": 12, "title": "中文标题", "synonyms": ["Alias", "Chinese Alias"]}]
|
||||
response = Mock(status_code=200)
|
||||
response.json.return_value = payload
|
||||
request = AsyncMock(return_value=response) if asynchronous else Mock(return_value=response)
|
||||
client = AniListApi()
|
||||
transport = Mock()
|
||||
transport.post_res = request
|
||||
transport.get_json = AsyncMock(return_value=translations) if asynchronous else Mock(return_value=translations)
|
||||
transport.get_json = AsyncMock() if asynchronous else Mock()
|
||||
if asynchronous:
|
||||
client._async_request = transport
|
||||
result = asyncio.run(client._async_invoke("query", {"id": 12}))
|
||||
else:
|
||||
client._request = transport
|
||||
result = client._invoke("query", {"id": 12})
|
||||
assert result is data
|
||||
assert media["title"]["chinese"] == "中文标题"
|
||||
assert media["synonyms"] == ["Alias", "Chinese Alias"]
|
||||
assert client._proxy_available is False
|
||||
assert [call.args[0] for call in request.call_args_list] == [client._base_url, client._official_url]
|
||||
assert all(call.kwargs["json"] == {"query": "query", "variables": {"id": 12}} for call in request.call_args_list)
|
||||
assert result is None
|
||||
request.assert_called_once_with(client._official_url, json={"query": "query", "variables": {"id": 12}})
|
||||
transport.get_json.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", [{}, {"Media": None}, {"Page": {"media": []}}])
|
||||
|
||||
324
tests/test_anilist_translations.py
Normal file
324
tests/test_anilist_translations.py
Normal file
@@ -0,0 +1,324 @@
|
||||
"""
|
||||
验证 AniList 中文数据集的跨实例持久化、到期刷新、镜像出口及失败回退。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.modules.anilist import anilist as anilist_client
|
||||
from app.modules.anilist.anilist import AniListApi
|
||||
from app.runtime import cache as runtime_cache
|
||||
|
||||
_TranslationEnvironment = tuple[dict[str, Any], list[float]]
|
||||
_Translations = list[dict[str, Any]]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def translation_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> _TranslationEnvironment:
|
||||
"""
|
||||
隔离数据集缓存、下载出口和时钟,使用真实文件后端且禁止路由至 Redis。
|
||||
"""
|
||||
values = {"CACHE_PATH": tmp_path, "GITHUB_PROXY": None}
|
||||
runtime_setting = anilist_client.get_runtime_setting
|
||||
monkeypatch.setattr(anilist_client, "get_runtime_setting", lambda name: values[name] if name in values else runtime_setting(name))
|
||||
monkeypatch.setattr(runtime_cache, "_backend_type_provider", lambda: "redis")
|
||||
now = [100_000.0]
|
||||
monkeypatch.setattr(anilist_client, "time", lambda: now[0])
|
||||
return values, now
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def translations() -> _Translations:
|
||||
"""
|
||||
提供含数字和字符串标识的中文数据,确保 JSON 快照不会改变标识类型。
|
||||
"""
|
||||
return [
|
||||
{"id": 12, "title": "中文标题", "synonyms": ["中文别名"]},
|
||||
{"id": "12", "title": "字符串标题", "synonyms": []},
|
||||
]
|
||||
|
||||
|
||||
def _download(client: AniListApi, asynchronous: bool, payload: object) -> Mock:
|
||||
"""
|
||||
替换指定传输的下载边界,不访问真实网络。
|
||||
"""
|
||||
request = client._async_request if asynchronous else client._request
|
||||
request.get_json = AsyncMock(return_value=payload) if asynchronous else Mock(return_value=payload)
|
||||
return request.get_json
|
||||
|
||||
|
||||
def _load(client: AniListApi, asynchronous: bool) -> dict[object, dict[str, Any]]:
|
||||
"""
|
||||
运行同步或异步加载入口并取得最终标题索引。
|
||||
"""
|
||||
return asyncio.run(client._async_translation_map()) if asynchronous else client._translation_map()
|
||||
|
||||
|
||||
def _cache_path(values: dict[str, Any]) -> Path:
|
||||
"""
|
||||
返回当前用例对应的持久化快照路径。
|
||||
"""
|
||||
return values["CACHE_PATH"] / "anilist" / "anilist-chinese.json"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_async", [False, True])
|
||||
@pytest.mark.parametrize("second_async", [False, True])
|
||||
def test_new_client_reuses_disk_snapshot(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations,
|
||||
first_async: bool, second_async: bool,
|
||||
) -> None:
|
||||
"""
|
||||
首次下载落盘,重建客户端和切换同步异步入口均应复用快照而无需重新下载。
|
||||
"""
|
||||
values, _ = translation_environment
|
||||
first = AniListApi()
|
||||
download = _download(first, first_async, translations)
|
||||
expected = _load(first, first_async)
|
||||
assert _load(first, first_async) == expected
|
||||
download.assert_called_once_with(first._translations_url)
|
||||
assert expected[12]["title"] == "中文标题"
|
||||
assert expected["12"]["title"] == "字符串标题"
|
||||
assert json.loads(_cache_path(values).read_bytes())["items"] == translations
|
||||
|
||||
second = AniListApi()
|
||||
download = _download(second, second_async, None)
|
||||
assert _load(second, second_async) == expected
|
||||
download.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_expired_snapshot_and_memory_refresh(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations, asynchronous: bool,
|
||||
) -> None:
|
||||
"""
|
||||
磁盘和内存到期均应重新下载,更新后的快照能由另一个客户端立即读取。
|
||||
"""
|
||||
values, now = translation_environment
|
||||
path = _cache_path(values)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(json.dumps({"expires_at": now[0] - 1, "items": [{"id": 12, "title": "旧标题"}]}))
|
||||
client = AniListApi()
|
||||
download = _download(client, asynchronous, translations)
|
||||
assert _load(client, asynchronous)[12]["title"] == "中文标题"
|
||||
download.assert_called_once()
|
||||
|
||||
now[0] += client._translations_ttl + 1
|
||||
updated = [{"id": 12, "title": "更新标题", "synonyms": []}]
|
||||
download = _download(client, asynchronous, updated)
|
||||
assert _load(client, asynchronous)[12]["title"] == "更新标题"
|
||||
download.assert_called_once()
|
||||
restored = AniListApi()
|
||||
download = _download(restored, not asynchronous, None)
|
||||
assert _load(restored, not asynchronous)[12]["title"] == "更新标题"
|
||||
download.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("payload", [None, {}, [], [{"id": []}], [{"id": 12, "title": []}]])
|
||||
def test_failed_refresh_keeps_stale_snapshot_and_retries(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations,
|
||||
asynchronous: bool, payload: object,
|
||||
) -> None:
|
||||
"""
|
||||
下载失败或载荷无效不得覆盖旧快照;退避后仍能恢复新中文数据。
|
||||
"""
|
||||
values, now = translation_environment
|
||||
path = _cache_path(values)
|
||||
path.parent.mkdir(parents=True)
|
||||
content = json.dumps({"expires_at": now[0] - 1, "items": translations}).encode()
|
||||
path.write_bytes(content)
|
||||
client = AniListApi()
|
||||
download = _download(client, asynchronous, payload)
|
||||
assert _load(client, asynchronous)[12]["title"] == "中文标题"
|
||||
assert _load(client, asynchronous)[12]["title"] == "中文标题"
|
||||
download.assert_called_once()
|
||||
assert path.read_bytes() == content
|
||||
|
||||
now[0] += client._translations_retry + 1
|
||||
download = _download(client, asynchronous, [{"id": 12, "title": "恢复标题"}])
|
||||
assert _load(client, asynchronous)[12]["title"] == "恢复标题"
|
||||
download.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_initial_failure_is_not_cached_forever(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations, asynchronous: bool,
|
||||
) -> None:
|
||||
"""
|
||||
初次下载失败时不落盘空数据,五分钟后可在同一实例内重试成功。
|
||||
"""
|
||||
values, now = translation_environment
|
||||
client = AniListApi()
|
||||
download = _download(client, asynchronous, None)
|
||||
assert _load(client, asynchronous) == {}
|
||||
assert _load(client, asynchronous) == {}
|
||||
download.assert_called_once()
|
||||
assert not _cache_path(values).exists()
|
||||
now[0] += client._translations_retry + 1
|
||||
download = _download(client, asynchronous, translations)
|
||||
assert _load(client, asynchronous)[12]["title"] == "中文标题"
|
||||
download.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("content", [
|
||||
b"{", b"\xff", b"[]", b"{}",
|
||||
b'{"expires_at": "invalid", "items": []}',
|
||||
b'{"expires_at": NaN, "items": []}',
|
||||
b'{"expires_at": 999999, "items": {}}',
|
||||
])
|
||||
def test_corrupt_cache_is_redownloaded(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations,
|
||||
asynchronous: bool, content: bytes,
|
||||
) -> None:
|
||||
"""
|
||||
损坏 JSON、无效编码、时间或条目结构都按缓存未命中处理。
|
||||
"""
|
||||
values, _ = translation_environment
|
||||
path = _cache_path(values)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_bytes(content)
|
||||
client = AniListApi()
|
||||
download = _download(client, asynchronous, translations)
|
||||
assert _load(client, asynchronous)[12]["title"] == "中文标题"
|
||||
download.assert_called_once()
|
||||
assert json.loads(path.read_bytes())["items"] == translations
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("operation", ["get", "set", "init"])
|
||||
def test_disk_errors_preserve_downloaded_titles(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations,
|
||||
asynchronous: bool, operation: str, monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
缓存目录不可写或文件读写失败时,仍能使用成功下载的数据并在内存复用。
|
||||
"""
|
||||
if operation == "init":
|
||||
monkeypatch.setattr(anilist_client, "FileCache", Mock(side_effect=PermissionError("readonly")))
|
||||
client = AniListApi()
|
||||
if operation != "init":
|
||||
cache = client._async_file_cache if asynchronous else client._file_cache
|
||||
error = AsyncMock(side_effect=OSError("disk error")) if asynchronous else Mock(side_effect=OSError("disk error"))
|
||||
monkeypatch.setattr(cache, operation, error)
|
||||
download = _download(client, asynchronous, translations)
|
||||
assert _load(client, asynchronous)[12]["title"] == "中文标题"
|
||||
assert _load(client, asynchronous)[12]["title"] == "中文标题"
|
||||
download.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("mirror_success", [False, True])
|
||||
def test_github_mirror_precedes_proxied_raw_url(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations,
|
||||
asynchronous: bool, mirror_success: bool, monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
镜像前缀支持路径和缺失尾斜杠,镜像直连,失败后回到已配置 PROXY 的原始客户端。
|
||||
"""
|
||||
values, _ = translation_environment
|
||||
values["GITHUB_PROXY"] = " https://mirror.example/prefix "
|
||||
values["PROXY"] = {"https": "socks5://127.0.0.1:1080"}
|
||||
client = AniListApi()
|
||||
raw = _download(client, asynchronous, translations)
|
||||
transport = Mock()
|
||||
payload = translations if mirror_success else {"error": "unavailable"}
|
||||
transport.get_json = AsyncMock(return_value=payload) if asynchronous else Mock(return_value=payload)
|
||||
constructor = Mock(return_value=transport)
|
||||
monkeypatch.setattr(anilist_client, "AsyncRequestUtils" if asynchronous else "RequestUtils", constructor)
|
||||
assert _load(client, asynchronous)[12]["title"] == "中文标题"
|
||||
constructor.assert_called_once_with(ua=anilist_client.get_runtime_setting("NORMAL_USER_AGENT"))
|
||||
transport.get_json.assert_called_once_with(f"https://mirror.example/prefix/{client._translations_url}")
|
||||
if mirror_success:
|
||||
raw.assert_not_called()
|
||||
else:
|
||||
raw.assert_called_once_with(client._translations_url)
|
||||
request = client._async_request if asynchronous else client._request
|
||||
assert request._proxies
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
def test_failed_write_keeps_newer_memory_on_next_refresh(
|
||||
translation_environment: _TranslationEnvironment, asynchronous: bool, monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
新数据落盘失败后再遇下载失败,旧磁盘快照不得覆盖更新的内存数据。
|
||||
"""
|
||||
values, now = translation_environment
|
||||
path = _cache_path(values)
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(json.dumps({"expires_at": now[0] - 1, "items": [{"id": 12, "title": "旧标题"}]}))
|
||||
client = AniListApi()
|
||||
cache = client._async_file_cache if asynchronous else client._file_cache
|
||||
error = AsyncMock(side_effect=OSError("readonly")) if asynchronous else Mock(side_effect=OSError("readonly"))
|
||||
monkeypatch.setattr(cache, "set", error)
|
||||
_download(client, asynchronous, [{"id": 12, "title": "新标题"}])
|
||||
assert _load(client, asynchronous)[12]["title"] == "新标题"
|
||||
now[0] += client._translations_ttl + 1
|
||||
_download(client, asynchronous, None)
|
||||
assert _load(client, asynchronous)[12]["title"] == "新标题"
|
||||
|
||||
|
||||
def test_mixed_concurrent_clients_download_once(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations,
|
||||
) -> None:
|
||||
"""
|
||||
同步线程和多个异步客户端共用加载锁,等待者从磁盘取得同一快照。
|
||||
"""
|
||||
sync_client = AniListApi()
|
||||
sync_download = _download(sync_client, False, translations)
|
||||
clients = [AniListApi() for _ in range(6)]
|
||||
downloads = [_download(client, True, translations) for client in clients]
|
||||
|
||||
async def load_all() -> list[dict[object, dict[str, Any]]]:
|
||||
"""
|
||||
同时发起跨线程和协程加载,覆盖两种入口的缓存竞争。
|
||||
"""
|
||||
return await asyncio.wait_for(asyncio.gather(
|
||||
asyncio.to_thread(sync_client._translation_map),
|
||||
*(client._async_translation_map() for client in clients),
|
||||
), timeout=5)
|
||||
|
||||
results = asyncio.run(load_all())
|
||||
assert all(result[12]["title"] == "中文标题" for result in results)
|
||||
assert sync_download.call_count + sum(download.call_count for download in downloads) == 1
|
||||
|
||||
|
||||
def test_cancelled_loader_releases_shared_lock(
|
||||
translation_environment: _TranslationEnvironment, translations: _Translations,
|
||||
) -> None:
|
||||
"""
|
||||
异步下载取消后释放共享锁,后续实例仍能重新下载并持久化。
|
||||
"""
|
||||
client = AniListApi()
|
||||
successor = AniListApi()
|
||||
download = _download(successor, True, translations)
|
||||
|
||||
async def cancel_and_reload() -> dict[object, dict[str, Any]]:
|
||||
"""
|
||||
在下载边界取消首个持锁者,并限定后续加载的完成时间。
|
||||
"""
|
||||
entered = asyncio.Event()
|
||||
|
||||
async def blocked_download(_url: str) -> None:
|
||||
"""
|
||||
挂起数据集下载直到调用方取消,避免依赖真实网络时序。
|
||||
"""
|
||||
entered.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
client._async_request.get_json = blocked_download
|
||||
task = asyncio.create_task(client._async_translation_map())
|
||||
await asyncio.wait_for(entered.wait(), timeout=2)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
return await asyncio.wait_for(successor._async_translation_map(), timeout=2)
|
||||
|
||||
assert asyncio.run(cancel_and_reload())[12]["title"] == "中文标题"
|
||||
download.assert_awaited_once()
|
||||
Reference in New Issue
Block a user