mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-10-29 11:27:43 +08:00
343 lines
12 KiB
Python
343 lines
12 KiB
Python
"""
|
||
同步数据库短事务的异步执行器。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import concurrent.futures
|
||
import threading
|
||
import time
|
||
from concurrent.futures import Future, ThreadPoolExecutor
|
||
from contextvars import copy_context
|
||
from dataclasses import dataclass
|
||
from typing import Callable, Optional, TypeVar, cast
|
||
|
||
from app.runtime.observability import record_metric
|
||
from app.runtime.priority import (
|
||
INTERACTIVE_DATABASE_CAPACITY,
|
||
INTERACTIVE_DATABASE_WORKERS,
|
||
is_interactive,
|
||
)
|
||
from app.schemas.exception import (
|
||
DatabaseWorkerClosedError,
|
||
DatabaseWorkerOverloadedError,
|
||
)
|
||
|
||
T = TypeVar("T")
|
||
|
||
DATABASE_WORKER_MAX_WORKERS = 4
|
||
DATABASE_WORKER_CAPACITY = 32
|
||
|
||
|
||
@dataclass(frozen=True, slots=True)
|
||
class DatabaseWorkerStats:
|
||
"""
|
||
单个数据库执行组当前的容量和任务数量快照。
|
||
|
||
:param max_workers: 最大工作线程数
|
||
:param capacity: 包含排队和运行中任务的总容量上限
|
||
:param queued: 当前等待线程执行的任务数
|
||
:param running: 当前正在工作线程内执行的任务数
|
||
:param rejected: 执行器累计因过载拒绝的任务数
|
||
:param closing: 是否已经开始关闭，不再接受新任务
|
||
"""
|
||
|
||
max_workers: int
|
||
capacity: int
|
||
queued: int
|
||
running: int
|
||
rejected: int
|
||
closing: bool
|
||
|
||
|
||
@dataclass(slots=True)
|
||
class _WorkItem:
|
||
"""
|
||
记录一个数据库短事务的排队和开始执行时间。
|
||
|
||
:param submitted_at: 任务提交时的单调时钟读数，单位为秒
|
||
:param started_at: 工作线程开始执行的单调时钟读数；None 表示尚未开始
|
||
"""
|
||
|
||
submitted_at: float
|
||
started_at: float | None = None
|
||
|
||
|
||
class DatabaseWorker:
|
||
"""
|
||
以有限线程和队列执行不能原生异步化的数据库短事务。
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
max_workers: int = DATABASE_WORKER_MAX_WORKERS,
|
||
capacity: int = DATABASE_WORKER_CAPACITY,
|
||
reserve_interactive: bool = True,
|
||
) -> None:
|
||
"""
|
||
保存容量配置，线程只在显式启动后创建。
|
||
|
||
:param max_workers: 普通数据库事务的工作线程上限
|
||
:param capacity: 普通事务的运行和排队总容量
|
||
:param reserve_interactive: 是否额外保留独立的交互短事务 worker 和队列
|
||
"""
|
||
if max_workers < 1:
|
||
raise ValueError("数据库 worker 线程数必须大于 0")
|
||
if capacity < max_workers:
|
||
raise ValueError("数据库 worker 总容量不能小于线程数")
|
||
self._max_workers = max_workers
|
||
self._capacity = capacity
|
||
self._state_lock = threading.Lock()
|
||
self._loop: asyncio.AbstractEventLoop | None = None
|
||
self._executor: ThreadPoolExecutor | None = None
|
||
self._futures: dict[
|
||
Future[object], tuple[asyncio.Future[object], _WorkItem]
|
||
] = {}
|
||
self._queued = 0
|
||
self._running = 0
|
||
self._reported_queued = 0
|
||
self._reported_running = 0
|
||
self._rejected = 0
|
||
self._closing = False
|
||
self._interactive_worker = (
|
||
DatabaseWorker(
|
||
max_workers=INTERACTIVE_DATABASE_WORKERS,
|
||
capacity=INTERACTIVE_DATABASE_CAPACITY,
|
||
reserve_interactive=False,
|
||
) if reserve_interactive else None
|
||
)
|
||
|
||
async def start(self) -> None:
|
||
"""
|
||
绑定当前事件循环并准备专属线程池。
|
||
"""
|
||
if self._executor is not None:
|
||
if self._loop is not asyncio.get_running_loop():
|
||
raise RuntimeError("数据库 worker 不能跨事件循环复用")
|
||
return
|
||
self._loop = asyncio.get_running_loop()
|
||
self._executor = ThreadPoolExecutor(
|
||
max_workers=self._max_workers,
|
||
thread_name_prefix="moviepilot-db",
|
||
)
|
||
self._closing = False
|
||
if self._interactive_worker is not None:
|
||
await self._interactive_worker.start()
|
||
self._record_depth()
|
||
|
||
def snapshot(self) -> DatabaseWorkerStats:
|
||
"""
|
||
返回普通执行组的低基数快照；交互保留组通过 interactive_snapshot 单独读取。
|
||
"""
|
||
with self._state_lock:
|
||
return DatabaseWorkerStats(
|
||
max_workers=self._max_workers,
|
||
capacity=self._capacity,
|
||
queued=self._queued,
|
||
running=self._running,
|
||
rejected=self._rejected,
|
||
closing=self._closing,
|
||
)
|
||
|
||
def interactive_snapshot(self) -> Optional[DatabaseWorkerStats]:
|
||
"""
|
||
返回交互保留组的容量和负载；未配置保留组时返回 None。
|
||
"""
|
||
return self._interactive_worker.snapshot() if self._interactive_worker is not None else None
|
||
|
||
async def run(self, operation: Callable[[], T]) -> T:
|
||
"""
|
||
执行短事务；结果发布和取消返回均晚于容量回收，取消仍等待已开始事务收口。
|
||
"""
|
||
loop = asyncio.get_running_loop()
|
||
executor = self._executor
|
||
if executor is None or self._loop is not loop or self._closing:
|
||
raise DatabaseWorkerClosedError("数据库 worker 当前不可接收任务")
|
||
if self._interactive_worker is not None and is_interactive():
|
||
return await self._interactive_worker.run(operation)
|
||
|
||
with self._state_lock:
|
||
if self._queued + self._running >= self._capacity:
|
||
self._rejected += 1
|
||
record_metric("db.worker.rejected")
|
||
raise DatabaseWorkerOverloadedError(
|
||
f"数据库 worker 容量已用尽（上限 {self._capacity}）"
|
||
)
|
||
self._queued += 1
|
||
|
||
item = _WorkItem(submitted_at=time.perf_counter())
|
||
context = copy_context()
|
||
try:
|
||
future = executor.submit(self._execute, item, context.run, operation)
|
||
except BaseException:
|
||
with self._state_lock:
|
||
self._queued -= 1
|
||
self._record_depth()
|
||
raise
|
||
|
||
# 不能直接包装原始 Future：其结果转发可能先于完成记账，晚注册回调还会立即执行。
|
||
wrapped: asyncio.Future[object] = loop.create_future()
|
||
with self._state_lock:
|
||
self._futures[future] = (wrapped, item)
|
||
future.add_done_callback(
|
||
lambda completed: self._schedule_completion(completed, item)
|
||
)
|
||
self._record_depth()
|
||
|
||
try:
|
||
return cast(T, await asyncio.shield(wrapped))
|
||
except asyncio.CancelledError:
|
||
future.cancel()
|
||
await self._wait_until_done(wrapped)
|
||
if wrapped.done() and not wrapped.cancelled():
|
||
wrapped.exception()
|
||
raise
|
||
|
||
def _execute(
|
||
self,
|
||
item: _WorkItem,
|
||
context_run: Callable[..., T],
|
||
operation: Callable[[], T],
|
||
) -> T:
|
||
"""
|
||
在线程中标记任务开始并保留提交时的上下文。
|
||
"""
|
||
item.started_at = time.perf_counter()
|
||
loop = self._loop
|
||
if loop is not None:
|
||
try:
|
||
loop.call_soon_threadsafe(self._mark_running, item)
|
||
except RuntimeError:
|
||
pass
|
||
return context_run(operation)
|
||
|
||
def _mark_running(self, item: _WorkItem) -> None:
|
||
"""
|
||
把任务从排队状态移入运行状态。
|
||
"""
|
||
with self._state_lock:
|
||
self._queued -= 1
|
||
self._running += 1
|
||
started_at = item.started_at or time.perf_counter()
|
||
record_metric("db.worker.wait", started_at - item.submitted_at)
|
||
self._record_depth()
|
||
|
||
def _schedule_completion(
|
||
self,
|
||
future: Future[object],
|
||
item: _WorkItem,
|
||
) -> None:
|
||
"""
|
||
把线程完成通知安全地回投到所属事件循环。
|
||
"""
|
||
loop = self._loop
|
||
if loop is None:
|
||
return
|
||
try:
|
||
loop.call_soon_threadsafe(self._complete, future, item)
|
||
except RuntimeError:
|
||
pass
|
||
|
||
def _complete(self, future: Future[object], item: _WorkItem) -> None:
|
||
"""
|
||
先释放 admission，再发布最终结果；同一事件循环回调保证调用方观察顺序。
|
||
"""
|
||
with self._state_lock:
|
||
wrapped, _ = self._futures.pop(future)
|
||
if future.running() or future.done() and not future.cancelled():
|
||
self._running -= 1
|
||
else:
|
||
self._queued -= 1
|
||
outcome = self._publish_result(future, wrapped)
|
||
started_at = item.started_at or item.submitted_at
|
||
record_metric(
|
||
"db.worker.duration",
|
||
time.perf_counter() - started_at,
|
||
outcome=outcome,
|
||
)
|
||
self._record_depth()
|
||
|
||
@staticmethod
|
||
def _publish_result(future: Future[object], wrapped: asyncio.Future[object]) -> str:
|
||
"""
|
||
保留原 wrap_future 的取消和异常翻译合同，不调用 asyncio 私有实现。
|
||
"""
|
||
if future.cancelled():
|
||
wrapped.cancel()
|
||
return "cancelled"
|
||
try:
|
||
result = future.result()
|
||
except BaseException as error:
|
||
exception_types: dict[type[BaseException], type[BaseException]] = {
|
||
concurrent.futures.CancelledError: asyncio.CancelledError,
|
||
concurrent.futures.InvalidStateError: asyncio.InvalidStateError,
|
||
}
|
||
exception_type = exception_types.get(type(error))
|
||
if exception_type is not None:
|
||
error = exception_type(*error.args).with_traceback(error.__traceback__)
|
||
wrapped.set_exception(error)
|
||
return "error"
|
||
wrapped.set_result(result)
|
||
return "success"
|
||
|
||
async def _wait_until_done(
|
||
self,
|
||
future: asyncio.Future[object],
|
||
*,
|
||
interruptible: bool = False,
|
||
) -> None:
|
||
"""
|
||
等待线程内事务结束，并按调用场景决定是否响应外层取消。
|
||
"""
|
||
while not future.done():
|
||
try:
|
||
await asyncio.shield(future)
|
||
except asyncio.CancelledError:
|
||
if interruptible:
|
||
raise
|
||
continue
|
||
except BaseException:
|
||
break
|
||
|
||
async def shutdown(self) -> None:
|
||
"""
|
||
拒绝新任务，取消排队任务并等待运行中事务结束。
|
||
"""
|
||
executor = self._executor
|
||
if executor is None:
|
||
return
|
||
if self._loop is not asyncio.get_running_loop():
|
||
raise RuntimeError("数据库 worker 必须在所属事件循环中停止")
|
||
self._closing = True
|
||
if self._interactive_worker is not None:
|
||
await self._interactive_worker.shutdown()
|
||
with self._state_lock:
|
||
futures = tuple(self._futures.items())
|
||
for future, _state in futures:
|
||
future.cancel()
|
||
for future, (wrapped, _item) in futures:
|
||
if not future.cancelled():
|
||
# 关停超时必须能返回并保留 owner；已开始的数据库事务继续由线程完成。
|
||
await self._wait_until_done(wrapped, interruptible=True)
|
||
executor.shutdown(wait=True, cancel_futures=True)
|
||
while self.snapshot().queued or self.snapshot().running:
|
||
await asyncio.sleep(0)
|
||
self._executor = None
|
||
self._record_depth()
|
||
|
||
def _record_depth(self) -> None:
|
||
"""
|
||
以状态变化量记录队列和运行中的任务数量。
|
||
"""
|
||
stats = self.snapshot()
|
||
queued_delta = stats.queued - self._reported_queued
|
||
running_delta = stats.running - self._reported_running
|
||
if queued_delta:
|
||
record_metric("db.worker.queue.depth", queued_delta)
|
||
if running_delta:
|
||
record_metric("db.worker.active", running_delta)
|
||
self._reported_queued = stats.queued
|
||
self._reported_running = stats.running
|