nas/Auto_Bangumi
1
0
Fork 0
mirror of https://github.com/EstrellaXD/Auto_Bangumi.git synced 2026-07-18 12:41:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): stop blocking the event loop in OpenAI parsing and static routing

Browse Source 
OpenAIParser now uses AsyncOpenAI — the previous sync SDK call ran directly
on the event loop (submitting to a ThreadPoolExecutor and immediately
blocking on .result() serialized it anyway). TitleParser.raw_parser and its
callers become async accordingly.

The SPA catch-all route listed dist/ on every request; snapshot it once at
startup. The module-level bangumi TTL cache is now lock-guarded — it is
written from asyncio.to_thread workers in notification paths while the event
loop reads it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014w1Z6Nxy6XTRgkFXqPr9Zh
This commit is contained in:
Estrella Pan
2026-07-02 11:39:10 +02:00
parent 6c39227aed
commit fa24ec4e2a
8 changed files with 43 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/src/main.py
View File

@@ -83,10 +83,13 @@ if VERSION != "DEV_VERSION":
# app.mount("/icons", StaticFiles(directory="dist/icons"), name="icons")
templates = Jinja2Templates(directory="dist")
# dist/ is immutable inside the container — snapshot once instead of
# hitting the filesystem on every request.
_DIST_FILES = frozenset(os.listdir("dist"))
@app.get("/{path:path}")
def html(request: Request, path: str):
files = os.listdir("dist")
if path in files:
if path in _DIST_FILES:
return FileResponse(f"dist/{path}")
else:
context = {"request": request}

29
backend/src/module/database/bangumi.py
View File

@@ -1,6 +1,7 @@
import json
import logging
import re
import threading
import time
from typing import Optional
@@ -64,16 +65,20 @@ def _set_aliases_list(bangumi: Bangumi, aliases: list[str]) -> None:
bangumi.title_aliases = json.dumps(unique_aliases, ensure_ascii=False)
# Module-level TTL cache for search_all results
# Module-level TTL cache for search_all results.
# Guarded by a lock: notification paths read/write it from asyncio.to_thread
# worker threads while the event loop uses it concurrently.
_bangumi_cache: list[Bangumi] | None = None
_bangumi_cache_time: float = 0
_BANGUMI_CACHE_TTL: float = 300.0 # 5 minutes - extended from 60s to reduce DB queries
_bangumi_cache_lock = threading.Lock()
def _invalidate_bangumi_cache():
global _bangumi_cache, _bangumi_cache_time
_bangumi_cache = None
_bangumi_cache_time = 0
with _bangumi_cache_lock:
_bangumi_cache = None
_bangumi_cache_time = 0
class BangumiDatabase:
@@ -365,11 +370,12 @@ class BangumiDatabase:
def search_all(self) -> list[Bangumi]:
global _bangumi_cache, _bangumi_cache_time
now = time.time()
if (
_bangumi_cache is not None
and (now - _bangumi_cache_time) < _BANGUMI_CACHE_TTL
):
return _bangumi_cache
with _bangumi_cache_lock:
if (
_bangumi_cache is not None
and (now - _bangumi_cache_time) < _BANGUMI_CACHE_TTL
):
return _bangumi_cache
statement = select(Bangumi)
result = self.session.execute(statement)
bangumis = list(result.scalars().all())
@@ -377,9 +383,10 @@ class BangumiDatabase:
# cached objects are accessed from a different session/request context
for b in bangumis:
self.session.expunge(b)
_bangumi_cache = bangumis
_bangumi_cache_time = now
return _bangumi_cache
with _bangumi_cache_lock:
_bangumi_cache = bangumis
_bangumi_cache_time = now
return bangumis
def search_id(self, _id: int) -> Optional[Bangumi]:
statement = select(Bangumi).where(Bangumi.id == _id)

16
backend/src/module/parser/analyser/openai.py
View File

@@ -1,9 +1,8 @@
import json
import logging
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Optional
from openai import AzureOpenAI, OpenAI
from openai import AsyncAzureOpenAI, AsyncOpenAI
from pydantic import BaseModel
from module.models import Bangumi
@@ -62,19 +61,19 @@ class OpenAIParser:
if not api_key:
raise ValueError("API key is required.")
if api_type == "azure":
self.client = AzureOpenAI(
self.client = AsyncAzureOpenAI(
api_key=api_key,
base_url=api_base,
azure_deployment=kwargs.get("deployment_id", ""),
api_version=kwargs.get("api_version", "2023-05-15"),
)
else:
self.client = OpenAI(api_key=api_key, base_url=api_base)
self.client = AsyncOpenAI(api_key=api_key, base_url=api_base)
self.model = model
self.openai_kwargs = kwargs
def parse(
async def parse(
self, text: str, prompt: str | None = None, asdict: bool = True
) -> dict | str:
"""parse text with openai
@@ -96,11 +95,8 @@ class OpenAIParser:
params = self._prepare_params(text, prompt)
with ThreadPoolExecutor(max_workers=1) as worker:
future = worker.submit(self.client.beta.chat.completions.parse, **params)
resp = future.result()
result = resp.choices[0].message.parsed
resp = await self.client.beta.chat.completions.parse(**params)
result = resp.choices[0].message.parsed
if asdict:
if hasattr(result, "model_dump"):

4
backend/src/module/parser/title_parser.py
View File

@@ -57,14 +57,14 @@ class TitleParser:
logger.warning("Please change bangumi info manually.")
@staticmethod
def raw_parser(raw: str) -> Bangumi | None:
async def raw_parser(raw: str) -> Bangumi | None:
language = settings.rss_parser.language
try:
# use OpenAI ChatGPT to parse raw title and get structured data
if settings.experimental_openai.enable:
kwargs = settings.experimental_openai.dict(exclude={"enable"})
gpt = OpenAIParser(**kwargs)
episode_dict = gpt.parse(raw, asdict=True)
episode_dict = await gpt.parse(raw, asdict=True)
episode = Episode(**episode_dict)
else:
episode = raw_parser(raw)

4
backend/src/module/rss/analyser.py
View File

@@ -49,7 +49,7 @@ class RSSAnalyser(TitleParser):
new_data = []
seen_titles: set[str] = set()
for torrent in torrents:
bangumi = self.raw_parser(raw=torrent.name)
bangumi = await self.raw_parser(raw=torrent.name)
if bangumi and bangumi.title_raw not in seen_titles:
await self.official_title_parser(bangumi=bangumi, rss=rss, torrent=torrent)
if not full_parse:
@@ -60,7 +60,7 @@ class RSSAnalyser(TitleParser):
return new_data
async def torrent_to_data(self, torrent: Torrent, rss: RSSItem) -> Bangumi:
bangumi = self.raw_parser(raw=torrent.name)
bangumi = await self.raw_parser(raw=torrent.name)
if bangumi:
await self.official_title_parser(bangumi=bangumi, rss=rss, torrent=torrent)
bangumi.rss_link = rss.url

8
backend/src/test/test_issue_bugs.py
View File

@@ -340,11 +340,11 @@ class TestIssue990NumberPrefixTitle:
assert result.resolution == "1080P"
assert result.group == "ANi"
def test_title_parser_returns_bangumi_for_number_prefix_title(self):
async def test_title_parser_returns_bangumi_for_number_prefix_title(self):
"""TitleParser.raw_parser returns a valid Bangumi for number-prefixed titles."""
from module.parser.title_parser import TitleParser
result = TitleParser.raw_parser(self.PROBLEM_TITLE)
result = await TitleParser.raw_parser(self.PROBLEM_TITLE)
assert result is not None
assert result.official_title == "29 岁单身中坚冒险家的日常"
assert result.title_raw == "29 岁单身中坚冒险家的日常"
@@ -504,11 +504,11 @@ class TestIssue992NonEpisodicAttributeError:
]
@pytest.mark.parametrize("title", NON_EPISODIC_TITLES)
def test_title_parser_returns_none_for_non_episodic(self, title):
async def test_title_parser_returns_none_for_non_episodic(self, title):
"""TitleParser.raw_parser should return None instead of crashing."""
from module.parser.title_parser import TitleParser
result = TitleParser.raw_parser(title)
result = await TitleParser.raw_parser(title)
assert result is None
def test_raw_parser_returns_none_for_unparseable(self):

4
backend/src/test/test_openai.py
View File

@@ -51,7 +51,7 @@ class TestOpenAIParser:
params = azure_parser._prepare_params(text, DEFAULT_PROMPT)
assert expected == params
def test_parse(self):
async def test_parse(self):
text = "[梦蓝字幕组]New Doraemon 哆啦A梦新番[747][2023.02.25][AVC][1080P][GB_JP][MP4]"
expected = {
"group": "梦蓝字幕组",
@@ -69,5 +69,5 @@ class TestOpenAIParser:
with mock.patch("module.parser.analyser.OpenAIParser.parse") as mocker:
mocker.return_value = json.dumps(expected)
result = self.parser.parse(text=text, asdict=False)
result = await self.parser.parse(text=text, asdict=False)
assert json.loads(result) == expected

8
backend/src/test/test_title_parser.py
View File

@@ -4,9 +4,9 @@ from module.parser.title_parser import TitleParser
class TestTitleParser:
def test_parse_without_openai(self):
async def test_parse_without_openai(self):
text = "[梦蓝字幕组]New Doraemon 哆啦A梦新番[747][2023.02.25][AVC][1080P][GB_JP][MP4]"
result = TitleParser.raw_parser(text)
result = await TitleParser.raw_parser(text)
assert result.group_name == "梦蓝字幕组"
assert result.title_raw == "New Doraemon"
assert result.dpi == "1080P"
@@ -17,9 +17,9 @@ class TestTitleParser:
not settings.experimental_openai.enable,
reason="OpenAI is not enabled in settings",
)
def test_parse_with_openai(self):
async def test_parse_with_openai(self):
text = "[梦蓝字幕组]New Doraemon 哆啦A梦新番[747][2023.02.25][AVC][1080P][GB_JP][MP4]"
result = TitleParser.raw_parser(text)
result = await TitleParser.raw_parser(text)
assert result.group_name == "梦蓝字幕组"
assert result.title_raw == "New Doraemon"
assert result.dpi == "1080P"