mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-09 23:30:13 +08:00
f3ad81b39e
Run `ruff check app tests scripts --fix` with the configured rule set (E4/E7/E9/F/I): import sorting (I001), f-strings without placeholders (F541), lambda/variable/unused-import cleanups that ruff marks safe. Nine files whose only ruff debt sits next to pre-existing pylint import-resolution errors (lazy exports, sys.path script imports) or whose "unused" imports are reached through attribute access by tests are left untouched so the changed-file pylint gate and the benchmark tests stay green; their diagnostics remain in the baseline. app/startup/lifecycle/__init__.py and app/main.py import business modules after cache composition, import-path and stdio setup on purpose (the source comments document the contract), so declare E402 per-file-ignores for them instead of adding per-line noqa. Ruff ratchet watermark drops from 498 to 100; sync the baseline fixture and the optimization checklist figure. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
302 lines
10 KiB
Python
302 lines
10 KiB
Python
import asyncio
|
||
import inspect
|
||
import threading
|
||
import time
|
||
from concurrent.futures import Executor, Future, ThreadPoolExecutor, wait
|
||
from contextvars import Context, copy_context
|
||
from functools import partial, wraps
|
||
from typing import Any, Callable, TypeVar, cast
|
||
|
||
from anyio.to_thread import run_sync
|
||
|
||
from app.runtime.gc import flush_thread_allocator_cache
|
||
from app.schemas.exception import ImmediateException
|
||
|
||
TaskResult = TypeVar("TaskResult")
|
||
ExecutorResult = TypeVar("ExecutorResult")
|
||
|
||
|
||
def submit_with_context(
|
||
executor: Executor,
|
||
func: Callable[..., ExecutorResult],
|
||
/,
|
||
*args: Any,
|
||
**kwargs: Any,
|
||
) -> Future[ExecutorResult]:
|
||
"""从空线程上下文提交任务，并在执行时恢复调用方的独立上下文快照。"""
|
||
context = copy_context()
|
||
|
||
def submit() -> Future[ExecutorResult]:
|
||
return executor.submit(context.run, func, *args, **kwargs)
|
||
|
||
return Context().run(submit)
|
||
|
||
|
||
class IdleCacheReleasingThreadPoolExecutor(ThreadPoolExecutor):
|
||
"""
|
||
worker 执行完任务、且队列里暂无待办时，由该 worker 交还自己的分配器线程缓存。
|
||
|
||
标准库线程池只按需增长、从不收缩，高峰期建出的 worker 此后长期空闲；jemalloc 只在线程
|
||
继续分配时才回收其线程缓存，空闲 worker 在最后一个任务里囤积的空闲块会一直计入 RSS。
|
||
刷新只发生在即将转入空闲时：连续任务之间不刷新，不影响突发期的分配性能。
|
||
"""
|
||
|
||
def submit(
|
||
self,
|
||
fn: Callable[..., ExecutorResult],
|
||
/,
|
||
*args: Any,
|
||
**kwargs: Any,
|
||
) -> Future[ExecutorResult]:
|
||
"""提交任务，任务结束后在同一 worker 线程里按需交还线程缓存。"""
|
||
return super().submit(self._run_then_release_cache, fn, *args, **kwargs)
|
||
|
||
def _run_then_release_cache(
|
||
self,
|
||
fn: Callable[..., ExecutorResult],
|
||
/,
|
||
*args: Any,
|
||
**kwargs: Any,
|
||
) -> ExecutorResult:
|
||
"""在 worker 线程中执行任务；队列已空说明该 worker 即将等待，此时刷新其线程缓存。"""
|
||
try:
|
||
return fn(*args, **kwargs)
|
||
finally:
|
||
# _work_queue 是标准库线程池的待办队列，自 Python 3.2 起稳定存在
|
||
if self._work_queue.empty():
|
||
flush_thread_allocator_cache()
|
||
|
||
|
||
class OwnedThreadPoolExecutor(IdleCacheReleasingThreadPoolExecutor):
|
||
"""
|
||
追踪已接受 Future，并提供可重试的有界关闭合同。
|
||
|
||
该 owner 不取消排队任务，保持 ``ThreadPoolExecutor.shutdown(wait=True)``
|
||
的历史完成语义；区别仅在于调用方可以在预算耗尽后保留同一实例继续收敛。
|
||
"""
|
||
|
||
def __init__(self, max_workers: int | None = None) -> None:
|
||
"""初始化线程池、提交准入状态和 Future owner 集合。"""
|
||
super().__init__(max_workers=max_workers)
|
||
self._ownership_lock = threading.RLock()
|
||
self._accepting = True
|
||
self._owned_futures: set[Future[Any]] = set()
|
||
|
||
@property
|
||
def accepting(self) -> bool:
|
||
"""返回执行器是否仍允许提交新任务。"""
|
||
with self._ownership_lock:
|
||
return self._accepting
|
||
|
||
def submit(
|
||
self,
|
||
fn: Callable[..., ExecutorResult],
|
||
/,
|
||
*args: Any,
|
||
**kwargs: Any,
|
||
) -> Future[ExecutorResult]:
|
||
"""提交任务并在其达到终态前保留 owner。"""
|
||
with self._ownership_lock:
|
||
future = super().submit(fn, *args, **kwargs)
|
||
self._owned_futures.add(future)
|
||
future.add_done_callback(self._discard_future)
|
||
return future
|
||
|
||
def shutdown(
|
||
self,
|
||
wait: bool = True,
|
||
*,
|
||
cancel_futures: bool = False,
|
||
) -> None:
|
||
"""封口提交准入并保留标准库 shutdown 的调用语义。"""
|
||
with self._ownership_lock:
|
||
self._accepting = False
|
||
# 先在锁内封口；真正等待必须在锁外进行，否则 worker 的完成回调无法释放 owner。
|
||
super().shutdown(wait=False, cancel_futures=cancel_futures)
|
||
if wait:
|
||
super().shutdown(wait=True, cancel_futures=cancel_futures)
|
||
|
||
def _discard_future(self, future: Future[Any]) -> None:
|
||
"""任务达到终态后释放 owner 记录。"""
|
||
with self._ownership_lock:
|
||
self._owned_futures.discard(future)
|
||
|
||
def shutdown_bounded(self, timeout: float) -> bool:
|
||
"""
|
||
封口新提交并有限等待全部已接受任务。
|
||
|
||
:param timeout: 等待 Future 达到终态的最长秒数
|
||
:return: 所有任务与 worker 均已终止时返回 True，否则返回 False
|
||
"""
|
||
deadline = time.monotonic() + max(0.0, timeout)
|
||
with self._ownership_lock:
|
||
self._accepting = False
|
||
# 不取消排队工作，保持历史 shutdown(wait=True) 的完成语义。
|
||
super().shutdown(wait=False)
|
||
owned_futures = tuple(self._owned_futures)
|
||
if owned_futures:
|
||
_, pending_futures = wait(
|
||
owned_futures,
|
||
timeout=max(0.0, deadline - time.monotonic()),
|
||
)
|
||
if pending_futures:
|
||
return False
|
||
# 标准库只提供无界 wait=True；Future 又会先标记完成再执行 done callback，
|
||
# 因此封口后读取稳定 worker 集合，复用同一 deadline 做有限 join。
|
||
worker_threads = tuple(cast(set[threading.Thread], self._threads))
|
||
current_thread = threading.current_thread()
|
||
for worker_thread in worker_threads:
|
||
if worker_thread is current_thread:
|
||
continue
|
||
worker_thread.join(
|
||
timeout=max(0.0, deadline - time.monotonic()),
|
||
)
|
||
return all(not worker_thread.is_alive() for worker_thread in worker_threads)
|
||
|
||
|
||
async def await_task_to_terminal(
|
||
task: asyncio.Future[TaskResult],
|
||
) -> TaskResult:
|
||
"""忽略当前调用方的重复取消，直到受保护任务进入真实终态。"""
|
||
while not task.done():
|
||
try:
|
||
await asyncio.shield(task)
|
||
except asyncio.CancelledError:
|
||
continue
|
||
except BaseException:
|
||
break
|
||
return task.result()
|
||
|
||
|
||
async def run_in_threadpool(
|
||
func: Callable[..., Any],
|
||
*args: Any,
|
||
**kwargs: Any,
|
||
) -> Any:
|
||
"""在线程中执行同步函数，保持 FastAPI 旧帮助函数的参数语义。"""
|
||
if kwargs:
|
||
func = partial(func, **kwargs)
|
||
context = copy_context()
|
||
return await run_sync(context.run, func, *args)
|
||
|
||
|
||
async def run_in_threadpool_to_completion(
|
||
func: Callable[..., Any],
|
||
*args: Any,
|
||
**kwargs: Any,
|
||
) -> Any:
|
||
"""在线程调用取得终态后传播取消，避免提前释放仍在使用的执行容量。"""
|
||
worker_task = asyncio.create_task(run_in_threadpool(func, *args, **kwargs))
|
||
cancellation: asyncio.CancelledError | None = None
|
||
while not worker_task.done():
|
||
try:
|
||
await asyncio.wait({worker_task})
|
||
except asyncio.CancelledError as error:
|
||
cancellation = cancellation or error
|
||
continue
|
||
try:
|
||
result = worker_task.result()
|
||
except Exception as error:
|
||
if cancellation is not None:
|
||
raise cancellation from error
|
||
raise
|
||
if cancellation is not None:
|
||
raise cancellation
|
||
return result
|
||
|
||
|
||
def retry(ExceptionToCheck: Any,
|
||
tries: int = 3, delay: int = 3, backoff: int = 2, logger: Any = None):
|
||
"""
|
||
:param ExceptionToCheck: 需要捕获的异常
|
||
:param tries: 重试次数
|
||
:param delay: 延迟时间
|
||
:param backoff: 延迟倍数
|
||
:param logger: 日志对象
|
||
"""
|
||
|
||
def deco_retry(f):
|
||
def f_retry(*args, **kwargs):
|
||
mtries, mdelay = tries, delay
|
||
while mtries > 1:
|
||
try:
|
||
return f(*args, **kwargs)
|
||
except ImmediateException:
|
||
raise
|
||
except ExceptionToCheck as e:
|
||
msg = f"{str(e)}, {mdelay} 秒后重试 ..."
|
||
if logger:
|
||
logger.warn(msg)
|
||
else:
|
||
print(msg)
|
||
time.sleep(mdelay)
|
||
mtries -= 1
|
||
mdelay *= backoff
|
||
return f(*args, **kwargs)
|
||
|
||
async def async_f_retry(*args, **kwargs):
|
||
mtries, mdelay = tries, delay
|
||
while mtries > 1:
|
||
try:
|
||
return await f(*args, **kwargs)
|
||
except ImmediateException:
|
||
raise
|
||
except ExceptionToCheck as e:
|
||
msg = f"{str(e)}, {mdelay} 秒后重试 ..."
|
||
if logger:
|
||
logger.warn(msg)
|
||
else:
|
||
print(msg)
|
||
await asyncio.sleep(mdelay)
|
||
mtries -= 1
|
||
mdelay *= backoff
|
||
return await f(*args, **kwargs)
|
||
|
||
# 根据函数类型返回相应的包装器
|
||
if inspect.iscoroutinefunction(f):
|
||
return async_f_retry
|
||
else:
|
||
return f_retry
|
||
|
||
return deco_retry
|
||
|
||
|
||
def log_execution_time(logger: Any = None):
|
||
"""
|
||
记录函数执行时间的装饰器
|
||
:param logger: 日志记录器对象，用于记录异常信息
|
||
"""
|
||
|
||
def decorator(func: Callable):
|
||
@wraps(func)
|
||
def wrapper(*args, **kwargs):
|
||
start_time = time.time()
|
||
result = func(*args, **kwargs)
|
||
end_time = time.time()
|
||
msg = f"{func.__name__} execution time: {end_time - start_time:.2f} seconds"
|
||
if logger:
|
||
logger.debug(msg)
|
||
else:
|
||
print(msg)
|
||
return result
|
||
|
||
@wraps(func)
|
||
async def async_wrapper(*args, **kwargs):
|
||
start_time = time.time()
|
||
result = await func(*args, **kwargs)
|
||
end_time = time.time()
|
||
msg = f"{func.__name__} execution time: {end_time - start_time:.2f} seconds"
|
||
if logger:
|
||
logger.debug(msg)
|
||
else:
|
||
print(msg)
|
||
return result
|
||
|
||
# 根据函数类型返回相应的包装器
|
||
if inspect.iscoroutinefunction(func):
|
||
return async_wrapper
|
||
else:
|
||
return wrapper
|
||
|
||
return decorator
|