mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-09 23:30:13 +08:00
598 lines
21 KiB
Python
598 lines
21 KiB
Python
"""缺失标准分类事实的可选跨来源补充服务。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import hashlib
|
||
import inspect
|
||
import json
|
||
import time
|
||
from collections.abc import Callable, Mapping, Sequence
|
||
from concurrent.futures import FIRST_COMPLETED, Future, wait
|
||
from dataclasses import dataclass
|
||
from enum import Enum
|
||
from typing import Any, Protocol, TypeAlias, cast
|
||
|
||
from pydantic import ValidationError
|
||
|
||
from app.domain.classification.conditions import condition_field_ids
|
||
from app.domain.classification.evaluator import read_fact
|
||
from app.domain.classification.fields import (
|
||
classification_fact_matches_definition,
|
||
field_definition_map,
|
||
)
|
||
from app.schemas.category import (
|
||
ClassificationEnrichmentRequest,
|
||
ClassificationEnrichmentResponse,
|
||
ClassificationFacts,
|
||
ClassificationFactSource,
|
||
ClassificationFieldDefinition,
|
||
ClassificationPolicy,
|
||
)
|
||
from app.schemas.types import MediaSource
|
||
|
||
_ENRICHMENT_METHOD = "get_media_classification_facts"
|
||
_DEFAULT_TIMEOUT_SECONDS = 3.0
|
||
_DEFAULT_MAX_CONCURRENCY = 3
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class ClassificationEnrichmentProvider:
|
||
"""绑定一个运行中模块的稳定身份、来源范围和补充方法。"""
|
||
|
||
provider_id: str
|
||
provider_name: str
|
||
media_sources: tuple[str, ...]
|
||
callback: Callable[..., object]
|
||
|
||
|
||
class ClassificationEnrichmentProviderCatalog(Protocol):
|
||
"""按调用时运行状态返回补充 provider 快照。"""
|
||
|
||
def providers(self) -> tuple[ClassificationEnrichmentProvider, ...]:
|
||
"""返回按稳定优先级排列的当前 provider。"""
|
||
...
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class ClassificationEnrichmentCacheEntry:
|
||
"""区分缓存未命中、空响应和有效响应。"""
|
||
|
||
response: ClassificationEnrichmentResponse | None
|
||
|
||
|
||
class ClassificationEnrichmentCache(Protocol):
|
||
"""补充结果 TTL 缓存的最小同步端口。"""
|
||
|
||
def get(self, key: str) -> ClassificationEnrichmentCacheEntry | None:
|
||
"""读取缓存项；不存在时返回空值。"""
|
||
...
|
||
|
||
def set(self, key: str, entry: ClassificationEnrichmentCacheEntry) -> None:
|
||
"""写入已校验响应或确定性空响应。"""
|
||
...
|
||
|
||
|
||
ProviderSubmit: TypeAlias = Callable[..., Future[Any]]
|
||
"""向进程托管线程池提交同步 provider 的端口。"""
|
||
|
||
ClassificationEnrichmentLogger: TypeAlias = Callable[[str], None]
|
||
"""接收不会阻断分类流程的补充诊断。"""
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class _ProviderCall:
|
||
"""保存一个 provider 的裁剪请求和缓存身份。"""
|
||
|
||
index: int
|
||
provider: ClassificationEnrichmentProvider
|
||
request: ClassificationEnrichmentRequest
|
||
cache_key: str
|
||
|
||
|
||
class _NoopEnrichmentCache:
|
||
"""未注入缓存时保持接口语义的空实现。"""
|
||
|
||
@staticmethod
|
||
def get(key: str) -> None:
|
||
"""始终返回未命中。"""
|
||
del key
|
||
return None
|
||
|
||
@staticmethod
|
||
def set(key: str, entry: ClassificationEnrichmentCacheEntry) -> None:
|
||
"""忽略写入。"""
|
||
del key, entry
|
||
|
||
|
||
class ClassificationEnrichmentService:
|
||
"""在有界预算内并发补充缺失事实，并保持主身份与已有事实不变。"""
|
||
|
||
def __init__(
|
||
self,
|
||
catalog: ClassificationEnrichmentProviderCatalog,
|
||
*,
|
||
submit: ProviderSubmit,
|
||
cache: ClassificationEnrichmentCache | None = None,
|
||
timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS,
|
||
max_concurrency: int = _DEFAULT_MAX_CONCURRENCY,
|
||
logger: ClassificationEnrichmentLogger | None = None,
|
||
) -> None:
|
||
"""保存延迟 provider 目录、线程端口和有界执行参数。"""
|
||
if timeout_seconds <= 0:
|
||
raise ValueError("分类事实补充总超时必须大于 0")
|
||
if max_concurrency <= 0:
|
||
raise ValueError("分类事实补充并发上限必须大于 0")
|
||
self._catalog = catalog
|
||
self._submit = submit
|
||
self._cache = cache or _NoopEnrichmentCache()
|
||
self._timeout_seconds = timeout_seconds
|
||
self._max_concurrency = max_concurrency
|
||
self._logger = logger
|
||
|
||
def enrich(
|
||
self,
|
||
policy: ClassificationPolicy,
|
||
facts: ClassificationFacts,
|
||
media: object,
|
||
) -> ClassificationFacts:
|
||
"""同步执行补全；默认模式或没有缺失引用时不发现 provider。"""
|
||
calls = self._build_calls(policy, facts, media)
|
||
if not calls:
|
||
return facts
|
||
responses = self._run_sync(calls)
|
||
return self._merge_responses(facts, calls, responses)
|
||
|
||
async def async_enrich(
|
||
self,
|
||
policy: ClassificationPolicy,
|
||
facts: ClassificationFacts,
|
||
media: object,
|
||
) -> ClassificationFacts:
|
||
"""异步执行相同补全合同，同步 provider 在线程中有界并发运行。"""
|
||
calls = self._build_calls(policy, facts, media)
|
||
if not calls:
|
||
return facts
|
||
responses = await self._run_async(calls)
|
||
return self._merge_responses(facts, calls, responses)
|
||
|
||
def _build_calls(
|
||
self,
|
||
policy: ClassificationPolicy,
|
||
facts: ClassificationFacts,
|
||
media: object,
|
||
) -> tuple[_ProviderCall, ...]:
|
||
"""按策略、缺失事实和 provider 来源能力裁剪独立请求。"""
|
||
if policy.enrichment_mode != "enrich_missing":
|
||
return ()
|
||
missing_fields = _referenced_missing_standard_fields(policy, facts)
|
||
if not missing_fields:
|
||
return ()
|
||
providers = self._catalog.providers()
|
||
if not providers:
|
||
return ()
|
||
definitions = field_definition_map()
|
||
external_ids = _external_identities(media)
|
||
calls: list[_ProviderCall] = []
|
||
for provider in providers:
|
||
eligible_fields = _provider_fields(
|
||
provider,
|
||
missing_fields,
|
||
definitions,
|
||
primary_source=facts.identity.media_source,
|
||
)
|
||
if not eligible_fields:
|
||
continue
|
||
request = ClassificationEnrichmentRequest(
|
||
identity=facts.identity.model_copy(deep=True),
|
||
media_type=facts.media.type,
|
||
missing_fields=list(eligible_fields),
|
||
external_ids=external_ids,
|
||
policy_revision=policy.revision,
|
||
timeout_seconds=self._timeout_seconds,
|
||
)
|
||
calls.append(
|
||
_ProviderCall(
|
||
index=len(calls),
|
||
provider=provider,
|
||
request=request,
|
||
cache_key=_cache_key(provider, request),
|
||
)
|
||
)
|
||
return tuple(calls)
|
||
|
||
def _run_sync(
|
||
self,
|
||
calls: Sequence[_ProviderCall],
|
||
) -> dict[int, ClassificationEnrichmentResponse | None]:
|
||
"""按并发上限滚动提交同步 provider，并在总预算到期时停止等待。"""
|
||
responses: dict[int, ClassificationEnrichmentResponse | None] = {}
|
||
queue: list[_ProviderCall] = []
|
||
for call in calls:
|
||
cached = self._cache.get(call.cache_key)
|
||
if cached is not None:
|
||
responses[call.index] = cached.response
|
||
else:
|
||
queue.append(call)
|
||
if not queue:
|
||
return responses
|
||
|
||
deadline = time.monotonic() + self._timeout_seconds
|
||
pending: dict[Future[Any], _ProviderCall] = {}
|
||
next_index = 0
|
||
|
||
def submit_next() -> None:
|
||
"""在还有执行容量时提交下一个稳定顺序 provider。"""
|
||
nonlocal next_index
|
||
if next_index >= len(queue):
|
||
return
|
||
call = queue[next_index]
|
||
next_index += 1
|
||
pending[self._submit(self._invoke, call, deadline)] = call
|
||
|
||
for _ in range(min(self._max_concurrency, len(queue))):
|
||
submit_next()
|
||
while pending:
|
||
remaining = deadline - time.monotonic()
|
||
if remaining <= 0:
|
||
break
|
||
done, _ = wait(
|
||
tuple(pending),
|
||
timeout=remaining,
|
||
return_when=FIRST_COMPLETED,
|
||
)
|
||
if not done:
|
||
break
|
||
for future in done:
|
||
call = pending.pop(future)
|
||
self._record_future(call, future, responses)
|
||
submit_next()
|
||
for future, call in pending.items():
|
||
future.cancel()
|
||
self._warn(
|
||
f"classification_enrichment_timeout provider={call.provider.provider_id}"
|
||
)
|
||
return responses
|
||
|
||
async def _run_async(
|
||
self,
|
||
calls: Sequence[_ProviderCall],
|
||
) -> dict[int, ClassificationEnrichmentResponse | None]:
|
||
"""包装宿主管理的线程 Future，并以 asyncio 总超时封口。"""
|
||
responses: dict[int, ClassificationEnrichmentResponse | None] = {}
|
||
queue: list[_ProviderCall] = []
|
||
for call in calls:
|
||
cached = self._cache.get(call.cache_key)
|
||
if cached is not None:
|
||
responses[call.index] = cached.response
|
||
else:
|
||
queue.append(call)
|
||
if not queue:
|
||
return responses
|
||
|
||
deadline = time.monotonic() + self._timeout_seconds
|
||
loop = asyncio.get_running_loop()
|
||
pending: dict[asyncio.Future[Any], _ProviderCall] = {}
|
||
next_index = 0
|
||
|
||
def submit_next() -> None:
|
||
"""在宿主管理线程池还有执行容量时提交下一个 provider。"""
|
||
nonlocal next_index
|
||
if next_index >= len(queue):
|
||
return
|
||
call = queue[next_index]
|
||
next_index += 1
|
||
future = self._submit(self._invoke, call, deadline)
|
||
pending[asyncio.wrap_future(future, loop=loop)] = call
|
||
|
||
for _ in range(min(self._max_concurrency, len(queue))):
|
||
submit_next()
|
||
while pending:
|
||
remaining = deadline - time.monotonic()
|
||
if remaining <= 0:
|
||
break
|
||
done, _ = await asyncio.wait(
|
||
tuple(pending),
|
||
timeout=remaining,
|
||
return_when=asyncio.FIRST_COMPLETED,
|
||
)
|
||
if not done:
|
||
break
|
||
for future in done:
|
||
call = pending.pop(future)
|
||
self._record_async_future(call, future, responses)
|
||
submit_next()
|
||
for future in pending:
|
||
future.cancel()
|
||
future.add_done_callback(_consume_task_result)
|
||
if pending:
|
||
self._warn(
|
||
f"classification_enrichment_timeout providers={len(pending)}"
|
||
)
|
||
return responses
|
||
|
||
def _invoke(
|
||
self,
|
||
call: _ProviderCall,
|
||
deadline: float,
|
||
) -> ClassificationEnrichmentResponse | None:
|
||
"""调用并严格校验一个同步 provider 响应。"""
|
||
remaining = max(0.001, deadline - time.monotonic())
|
||
request = call.request.model_copy(
|
||
deep=True,
|
||
update={"timeout_seconds": remaining},
|
||
)
|
||
raw = call.provider.callback(request=request)
|
||
if inspect.isawaitable(raw):
|
||
close = getattr(raw, "close", None)
|
||
if callable(close):
|
||
close()
|
||
raise TypeError(
|
||
f"{_ENRICHMENT_METHOD} 必须是同步方法；异步识别会在线程中调用它"
|
||
)
|
||
if raw is None:
|
||
return None
|
||
try:
|
||
response = ClassificationEnrichmentResponse.model_validate(raw)
|
||
except ValidationError as error:
|
||
raise ValueError("分类事实 provider 响应结构无效") from error
|
||
return _validated_response(call.provider, request, response)
|
||
|
||
def _record_future(
|
||
self,
|
||
call: _ProviderCall,
|
||
future: Future[Any],
|
||
responses: dict[int, ClassificationEnrichmentResponse | None],
|
||
) -> None:
|
||
"""隔离一个同步 Future 的错误并缓存已校验终态。"""
|
||
try:
|
||
response = cast(ClassificationEnrichmentResponse | None, future.result())
|
||
except Exception as error: # noqa: BLE001 provider 故障必须隔离
|
||
self._warn(
|
||
f"classification_enrichment_error provider={call.provider.provider_id} error={error}"
|
||
)
|
||
return
|
||
responses[call.index] = response
|
||
self._cache.set(
|
||
call.cache_key,
|
||
ClassificationEnrichmentCacheEntry(response=response),
|
||
)
|
||
|
||
def _record_async_future(
|
||
self,
|
||
call: _ProviderCall,
|
||
future: asyncio.Future[Any],
|
||
responses: dict[int, ClassificationEnrichmentResponse | None],
|
||
) -> None:
|
||
"""隔离异步包装 Future 的错误并缓存已校验终态。"""
|
||
try:
|
||
response = cast(ClassificationEnrichmentResponse | None, future.result())
|
||
except Exception as error: # noqa: BLE001 provider 故障必须隔离
|
||
self._warn(
|
||
f"classification_enrichment_error provider={call.provider.provider_id} error={error}"
|
||
)
|
||
return
|
||
responses[call.index] = response
|
||
self._cache.set(
|
||
call.cache_key,
|
||
ClassificationEnrichmentCacheEntry(response=response),
|
||
)
|
||
|
||
@staticmethod
|
||
def _merge_responses(
|
||
facts: ClassificationFacts,
|
||
calls: Sequence[_ProviderCall],
|
||
responses: Mapping[int, ClassificationEnrichmentResponse | None],
|
||
) -> ClassificationFacts:
|
||
"""按 provider 目录顺序只填充仍缺失的事实并记录提供来源。"""
|
||
merged = facts.model_copy(deep=True)
|
||
for call in calls:
|
||
response = responses.get(call.index)
|
||
if response is None:
|
||
continue
|
||
for field_id, value in response.facts.items():
|
||
_, missing = read_fact(merged, field_id)
|
||
if not missing:
|
||
continue
|
||
if not _set_standard_fact(merged, field_id, value):
|
||
continue
|
||
merged.field_sources[field_id] = ClassificationFactSource(
|
||
media_source=response.media_source,
|
||
provider_id=call.provider.provider_id,
|
||
provider_name=call.provider.provider_name,
|
||
)
|
||
return cast(ClassificationFacts, merged)
|
||
|
||
def _warn(self, message: str) -> None:
|
||
"""尽力记录诊断，日志端口失败也不得影响识别。"""
|
||
if self._logger is None:
|
||
return
|
||
try:
|
||
self._logger(message)
|
||
except Exception: # noqa: BLE001 诊断不能成为分类硬依赖
|
||
return
|
||
|
||
|
||
def _referenced_missing_standard_fields(
|
||
policy: ClassificationPolicy,
|
||
facts: ClassificationFacts,
|
||
) -> tuple[str, ...]:
|
||
"""返回当前媒体实际适用规则中引用且缺失的标准事实。"""
|
||
definitions = field_definition_map()
|
||
media_type = facts.media.type
|
||
media_source = facts.identity.media_source
|
||
fields: list[str] = []
|
||
for rule in policy.rules:
|
||
if not rule.enabled or media_type not in rule.media_types:
|
||
continue
|
||
if rule.sources and media_source not in rule.sources:
|
||
continue
|
||
for field_id in condition_field_ids(rule.when):
|
||
if field_id.startswith(("identity.", "extensions.")):
|
||
continue
|
||
definition = definitions.get(field_id)
|
||
if definition is None or media_type not in definition.media_types:
|
||
continue
|
||
_, missing = read_fact(facts, field_id)
|
||
if missing and field_id not in fields:
|
||
fields.append(field_id)
|
||
return tuple(fields)
|
||
|
||
|
||
def _provider_fields(
|
||
provider: ClassificationEnrichmentProvider,
|
||
missing_fields: Sequence[str],
|
||
definitions: Mapping[str, ClassificationFieldDefinition],
|
||
*,
|
||
primary_source: str,
|
||
) -> tuple[str, ...]:
|
||
"""按 provider 来源能力过滤不能跨源提供的字段。"""
|
||
candidate_sources = tuple(
|
||
source for source in provider.media_sources if source != primary_source
|
||
)
|
||
if provider.media_sources and not candidate_sources:
|
||
return ()
|
||
fields: list[str] = []
|
||
for field_id in missing_fields:
|
||
definition = definitions[field_id]
|
||
if not candidate_sources:
|
||
fields.append(field_id)
|
||
continue
|
||
if any(
|
||
definition.source_support.get(source) != "unavailable"
|
||
for source in candidate_sources
|
||
):
|
||
fields.append(field_id)
|
||
return tuple(fields)
|
||
|
||
|
||
def _validated_response(
|
||
provider: ClassificationEnrichmentProvider,
|
||
request: ClassificationEnrichmentRequest,
|
||
response: ClassificationEnrichmentResponse,
|
||
) -> ClassificationEnrichmentResponse:
|
||
"""验证来源所有权、同媒体证明、请求字段和值类型。"""
|
||
source_id = response.media_source.strip()
|
||
if not source_id or source_id == request.identity.media_source:
|
||
raise ValueError("补充 provider 必须返回不同于主身份的有效媒体来源")
|
||
if provider.media_sources and source_id not in provider.media_sources:
|
||
raise ValueError("补充 provider 返回了未声明拥有的媒体来源")
|
||
_validate_match(request, response)
|
||
|
||
definitions = field_definition_map()
|
||
accepted: dict[str, Any] = {}
|
||
for field_id, value in response.facts.items():
|
||
if field_id not in request.missing_fields or field_id.startswith(
|
||
("identity.", "extensions.")
|
||
):
|
||
continue
|
||
definition = definitions.get(field_id)
|
||
if definition is None or request.media_type not in definition.media_types:
|
||
continue
|
||
support = definition.source_support.get(source_id)
|
||
if support == "unavailable":
|
||
continue
|
||
if value is None or not classification_fact_matches_definition(
|
||
value, definition
|
||
):
|
||
continue
|
||
accepted[field_id] = value
|
||
return cast(
|
||
ClassificationEnrichmentResponse,
|
||
response.model_copy(deep=True, update={"facts": accepted}),
|
||
)
|
||
|
||
|
||
def _validate_match(
|
||
request: ClassificationEnrichmentRequest,
|
||
response: ClassificationEnrichmentResponse,
|
||
) -> None:
|
||
"""只接受已知外部 ID 或对主身份的明确映射证明。"""
|
||
match = response.match
|
||
if match.kind == "external_id":
|
||
expected = request.external_ids.get(match.media_source)
|
||
if expected is None or expected != match.media_id:
|
||
raise ValueError("补充结果没有命中请求中已知的外部 ID")
|
||
return
|
||
if (
|
||
match.media_source != request.identity.media_source
|
||
or match.media_id != request.identity.media_id
|
||
):
|
||
raise ValueError("明确映射必须回指完整主媒体身份")
|
||
|
||
|
||
def _external_identities(media: object) -> dict[str, str]:
|
||
"""从标准媒体对象读取可独立验证的来源专用外部 ID。"""
|
||
fields = (
|
||
(MediaSource.TMDB.value, "tmdb_id"),
|
||
(MediaSource.IMDb.value, "imdb_id"),
|
||
(MediaSource.TVDB.value, "tvdb_id"),
|
||
(MediaSource.Douban.value, "douban_id"),
|
||
(MediaSource.Bangumi.value, "bangumi_id"),
|
||
(MediaSource.AniList.value, "anilist_id"),
|
||
)
|
||
identities: dict[str, str] = {}
|
||
for source, attribute in fields:
|
||
value = getattr(media, attribute, None)
|
||
text = _enum_text(value)
|
||
if text:
|
||
identities[source] = text
|
||
if isrc := _enum_text(getattr(media, "isrc", None)):
|
||
identities["isrc"] = isrc
|
||
return identities
|
||
|
||
|
||
def _set_standard_fact(
|
||
facts: ClassificationFacts,
|
||
field_id: str,
|
||
value: object,
|
||
) -> bool:
|
||
"""把已校验值写入 media 或 music 标准事实的一层字段。"""
|
||
namespace, separator, attribute = field_id.partition(".")
|
||
if not separator or namespace not in {"media", "music"} or not attribute:
|
||
return False
|
||
target = getattr(facts, namespace, None)
|
||
if target is None or not hasattr(target, attribute):
|
||
return False
|
||
setattr(target, attribute, value)
|
||
return True
|
||
|
||
|
||
def _cache_key(
|
||
provider: ClassificationEnrichmentProvider,
|
||
request: ClassificationEnrichmentRequest,
|
||
) -> str:
|
||
"""生成包含策略、身份、字段集合和 provider 的稳定缓存键。"""
|
||
payload = {
|
||
"provider_id": provider.provider_id,
|
||
"identity": request.identity.model_dump(mode="json"),
|
||
"media_type": request.media_type,
|
||
"missing_fields": sorted(request.missing_fields),
|
||
"external_ids": dict(sorted(request.external_ids.items())),
|
||
"policy_revision": request.policy_revision,
|
||
}
|
||
serialized = json.dumps(
|
||
payload,
|
||
ensure_ascii=True,
|
||
sort_keys=True,
|
||
separators=(",", ":"),
|
||
)
|
||
return hashlib.sha256(serialized.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _enum_text(value: object) -> str:
|
||
"""把字符串枚举或普通值转换为稳定文本。"""
|
||
if isinstance(value, Enum):
|
||
value = value.value
|
||
return str(value or "").strip()
|
||
|
||
|
||
def _consume_task_result(future: asyncio.Future[Any]) -> None:
|
||
"""消费超时后线程 Future 包装的终态，避免未读取异常告警。"""
|
||
if future.cancelled():
|
||
return
|
||
try:
|
||
future.exception()
|
||
except (asyncio.CancelledError, Exception):
|
||
return
|