From a5d6062aa8d26e3c54e6532bf3f26ca24f206950 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Mon, 23 Dec 2024 01:32:17 +0800 Subject: [PATCH 1/4] feat(recommend): add job to refresh recommend cache --- app/chain/recommend.py | 3 ++- app/scheduler.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/app/chain/recommend.py b/app/chain/recommend.py index 3783ca45..dae271d7 100644 --- a/app/chain/recommend.py +++ b/app/chain/recommend.py @@ -62,6 +62,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): self.tmdb_tvs() self.tmdb_trending() self.bangumi_calendar() + self.douban_movie_showing() self.douban_movies() self.douban_tvs() self.douban_movie_top250() @@ -88,7 +89,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): @log_execution_time(logger=logger) @cached_with_empty_check def tmdb_tvs(self, sort_by: str = "popularity.desc", with_genres: str = "", - with_original_language: str = "", page: int = 1) -> Any: + with_original_language: str = "zh|en|ja|ko", page: int = 1) -> Any: """ TMDB热门电视剧 """ diff --git a/app/scheduler.py b/app/scheduler.py index b3cf42f8..dc67e777 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -11,6 +11,7 @@ from apscheduler.schedulers.background import BackgroundScheduler from app import schemas from app.chain import ChainBase from app.chain.mediaserver import MediaServerChain +from app.chain.recommend import RecommendChain from app.chain.site import SiteChain from app.chain.subscribe import SubscribeChain from app.chain.tmdb import TmdbChain @@ -121,6 +122,11 @@ class Scheduler(metaclass=Singleton): "name": "站点数据刷新", "func": SiteChain().refresh_userdatas, "running": False, + }, + "recommend_refresh": { + "name": "推荐缓存", + "func": RecommendChain().refresh_recommend, + "running": False, } } @@ -310,6 +316,19 @@ class Scheduler(metaclass=Singleton): } ) + # 推荐缓存 + self._scheduler.add_job( + self.start, + "interval", + id="recommend_refresh", + name="推荐缓存", + hours=6, + next_run_time=datetime.now(pytz.timezone(settings.TZ)) + timedelta(seconds=3), + kwargs={ + 'job_id': 'recommend_refresh' + } + ) + self.init_plugin_jobs() # 打印服务 From 6fec16d78a2433db166dde7c982f937998a7b0fd Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Mon, 23 Dec 2024 01:39:34 +0800 Subject: [PATCH 2/4] fix(cache): include method name and default parameters in cache key --- app/chain/recommend.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/app/chain/recommend.py b/app/chain/recommend.py index dae271d7..f946e218 100644 --- a/app/chain/recommend.py +++ b/app/chain/recommend.py @@ -1,3 +1,4 @@ +import inspect from functools import wraps from typing import Any, Callable @@ -29,8 +30,18 @@ def cached_with_empty_check(func: Callable): @wraps(func) def wrapper(*args, **kwargs): + signature = inspect.signature(func) + resolved_kwargs = {} + # 获取默认值并结合传递的参数(如果有) + for param, value in signature.parameters.items(): + if param in kwargs: + # 使用显式传递的参数 + resolved_kwargs[param] = kwargs[param] + elif value.default is not inspect.Parameter.empty: + # 没有传递参数时使用默认值 + resolved_kwargs[param] = value.default # 使用 cachetools 缓存,构造缓存键 - cache_key = hashkey(*args, **kwargs) + cache_key = f"{func.__name__}_{hashkey(*args, **resolved_kwargs)}" if cache_key in recommend_cache: return recommend_cache[cache_key] result = func(*args, **kwargs) @@ -58,6 +69,9 @@ class RecommendChain(ChainBase, metaclass=Singleton): """ 刷新推荐 """ + logger.debug("Starting to refresh Recommend data.") + recommend_cache.clear() + logger.debug("Recommend Cache has been cleared.") self.tmdb_movies() self.tmdb_tvs() self.tmdb_trending() @@ -71,6 +85,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): self.douban_tv_animation() self.douban_movie_hot() self.douban_tv_hot() + logger.debug("Recommend data refresh completed.") @log_execution_time(logger=logger) @cached_with_empty_check From 33de1c36189127bc664816ae6557b83926e51c76 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Mon, 23 Dec 2024 02:51:23 +0800 Subject: [PATCH 3/4] feat(recommend): add semaphore to limit concurrent requests --- app/api/endpoints/mediaserver.py | 40 +++++++++++--------- app/api/endpoints/subscribe.py | 64 +++++++++++++++++--------------- 2 files changed, 58 insertions(+), 46 deletions(-) diff --git a/app/api/endpoints/mediaserver.py b/app/api/endpoints/mediaserver.py index 1f1e90b8..fbfa7909 100644 --- a/app/api/endpoints/mediaserver.py +++ b/app/api/endpoints/mediaserver.py @@ -1,3 +1,4 @@ +import asyncio from typing import Any, List, Dict from fastapi import APIRouter, Depends @@ -42,8 +43,12 @@ def play_item(itemid: str, _: schemas.TokenPayload = Depends(verify_token)) -> s return schemas.Response(success=False, message="未找到播放地址") +# 控制最大并发数 +semaphore = asyncio.Semaphore(10) + + @router.get("/exists", summary="查询本地是否存在(数据库)", response_model=schemas.Response) -def exists_local(title: str = None, +async def exists_local(title: str = None, year: int = None, mtype: str = None, tmdbid: int = None, @@ -53,22 +58,23 @@ def exists_local(title: str = None, """ 判断本地是否存在 """ - meta = MetaInfo(title) - if not season: - season = meta.begin_season - # 返回对象 - ret_info = {} - # 本地数据库是否存在 - exist: MediaServerItem = MediaServerOper(db).exists( - title=meta.name, year=year, mtype=mtype, tmdbid=tmdbid, season=season - ) - if exist: - ret_info = { - "id": exist.item_id - } - return schemas.Response(success=True if exist else False, data={ - "item": ret_info - }) + async with semaphore: + meta = MetaInfo(title) + if not season: + season = meta.begin_season + # 返回对象 + ret_info = {} + # 本地数据库是否存在 + exist: MediaServerItem = MediaServerOper(db).exists( + title=meta.name, year=year, mtype=mtype, tmdbid=tmdbid, season=season + ) + if exist: + ret_info = { + "id": exist.item_id + } + return schemas.Response(success=True if exist else False, data={ + "item": ret_info + }) @router.post("/exists_remote", summary="查询已存在的剧集信息(媒体服务器)", response_model=Dict[int, list]) diff --git a/app/api/endpoints/subscribe.py b/app/api/endpoints/subscribe.py index b19c36bf..69df8b90 100644 --- a/app/api/endpoints/subscribe.py +++ b/app/api/endpoints/subscribe.py @@ -1,3 +1,4 @@ +import asyncio from typing import List, Any import cn2an @@ -146,8 +147,12 @@ def update_subscribe_status( return schemas.Response(success=True) +# 控制最大并发数 +semaphore = asyncio.Semaphore(10) + + @router.get("/media/{mediaid}", summary="查询订阅", response_model=schemas.Subscribe) -def subscribe_mediaid( +async def subscribe_mediaid( mediaid: str, season: int = None, title: str = None, @@ -156,35 +161,36 @@ def subscribe_mediaid( """ 根据 TMDBID/豆瓣ID/BangumiId 查询订阅 tmdb:/douban: """ - result = None - title_check = False - if mediaid.startswith("tmdb:"): - tmdbid = mediaid[5:] - if not tmdbid or not str(tmdbid).isdigit(): - return Subscribe() - result = Subscribe.exists(db, tmdbid=int(tmdbid), season=season) - elif mediaid.startswith("douban:"): - doubanid = mediaid[7:] - if not doubanid: - return Subscribe() - result = Subscribe.get_by_doubanid(db, doubanid) - if not result and title: - title_check = True - elif mediaid.startswith("bangumi:"): - bangumiid = mediaid[8:] - if not bangumiid or not str(bangumiid).isdigit(): - return Subscribe() - result = Subscribe.get_by_bangumiid(db, int(bangumiid)) - if not result and title: - title_check = True - # 使用名称检查订阅 - if title_check and title: - meta = MetaInfo(title) - if season: - meta.begin_season = season - result = Subscribe.get_by_title(db, title=meta.name, season=meta.begin_season) + async with semaphore: + result = None + title_check = False + if mediaid.startswith("tmdb:"): + tmdbid = mediaid[5:] + if not tmdbid or not str(tmdbid).isdigit(): + return Subscribe() + result = Subscribe.exists(db, tmdbid=int(tmdbid), season=season) + elif mediaid.startswith("douban:"): + doubanid = mediaid[7:] + if not doubanid: + return Subscribe() + result = Subscribe.get_by_doubanid(db, doubanid) + if not result and title: + title_check = True + elif mediaid.startswith("bangumi:"): + bangumiid = mediaid[8:] + if not bangumiid or not str(bangumiid).isdigit(): + return Subscribe() + result = Subscribe.get_by_bangumiid(db, int(bangumiid)) + if not result and title: + title_check = True + # 使用名称检查订阅 + if title_check and title: + meta = MetaInfo(title) + if season: + meta.begin_season = season + result = Subscribe.get_by_title(db, title=meta.name, season=meta.begin_season) - return result if result else Subscribe() + return result if result else Subscribe() @router.get("/refresh", summary="刷新订阅", response_model=schemas.Response) From 544119c49fb96bc884794d514697c6c1ae726f28 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:29:37 +0800 Subject: [PATCH 4/4] Revert "feat(recommend): add semaphore to limit concurrent requests" This reverts commit 33de1c36189127bc664816ae6557b83926e51c76. --- app/api/endpoints/mediaserver.py | 40 +++++++++----------- app/api/endpoints/subscribe.py | 64 +++++++++++++++----------------- 2 files changed, 46 insertions(+), 58 deletions(-) diff --git a/app/api/endpoints/mediaserver.py b/app/api/endpoints/mediaserver.py index fbfa7909..1f1e90b8 100644 --- a/app/api/endpoints/mediaserver.py +++ b/app/api/endpoints/mediaserver.py @@ -1,4 +1,3 @@ -import asyncio from typing import Any, List, Dict from fastapi import APIRouter, Depends @@ -43,12 +42,8 @@ def play_item(itemid: str, _: schemas.TokenPayload = Depends(verify_token)) -> s return schemas.Response(success=False, message="未找到播放地址") -# 控制最大并发数 -semaphore = asyncio.Semaphore(10) - - @router.get("/exists", summary="查询本地是否存在(数据库)", response_model=schemas.Response) -async def exists_local(title: str = None, +def exists_local(title: str = None, year: int = None, mtype: str = None, tmdbid: int = None, @@ -58,23 +53,22 @@ async def exists_local(title: str = None, """ 判断本地是否存在 """ - async with semaphore: - meta = MetaInfo(title) - if not season: - season = meta.begin_season - # 返回对象 - ret_info = {} - # 本地数据库是否存在 - exist: MediaServerItem = MediaServerOper(db).exists( - title=meta.name, year=year, mtype=mtype, tmdbid=tmdbid, season=season - ) - if exist: - ret_info = { - "id": exist.item_id - } - return schemas.Response(success=True if exist else False, data={ - "item": ret_info - }) + meta = MetaInfo(title) + if not season: + season = meta.begin_season + # 返回对象 + ret_info = {} + # 本地数据库是否存在 + exist: MediaServerItem = MediaServerOper(db).exists( + title=meta.name, year=year, mtype=mtype, tmdbid=tmdbid, season=season + ) + if exist: + ret_info = { + "id": exist.item_id + } + return schemas.Response(success=True if exist else False, data={ + "item": ret_info + }) @router.post("/exists_remote", summary="查询已存在的剧集信息(媒体服务器)", response_model=Dict[int, list]) diff --git a/app/api/endpoints/subscribe.py b/app/api/endpoints/subscribe.py index 69df8b90..b19c36bf 100644 --- a/app/api/endpoints/subscribe.py +++ b/app/api/endpoints/subscribe.py @@ -1,4 +1,3 @@ -import asyncio from typing import List, Any import cn2an @@ -147,12 +146,8 @@ def update_subscribe_status( return schemas.Response(success=True) -# 控制最大并发数 -semaphore = asyncio.Semaphore(10) - - @router.get("/media/{mediaid}", summary="查询订阅", response_model=schemas.Subscribe) -async def subscribe_mediaid( +def subscribe_mediaid( mediaid: str, season: int = None, title: str = None, @@ -161,36 +156,35 @@ async def subscribe_mediaid( """ 根据 TMDBID/豆瓣ID/BangumiId 查询订阅 tmdb:/douban: """ - async with semaphore: - result = None - title_check = False - if mediaid.startswith("tmdb:"): - tmdbid = mediaid[5:] - if not tmdbid or not str(tmdbid).isdigit(): - return Subscribe() - result = Subscribe.exists(db, tmdbid=int(tmdbid), season=season) - elif mediaid.startswith("douban:"): - doubanid = mediaid[7:] - if not doubanid: - return Subscribe() - result = Subscribe.get_by_doubanid(db, doubanid) - if not result and title: - title_check = True - elif mediaid.startswith("bangumi:"): - bangumiid = mediaid[8:] - if not bangumiid or not str(bangumiid).isdigit(): - return Subscribe() - result = Subscribe.get_by_bangumiid(db, int(bangumiid)) - if not result and title: - title_check = True - # 使用名称检查订阅 - if title_check and title: - meta = MetaInfo(title) - if season: - meta.begin_season = season - result = Subscribe.get_by_title(db, title=meta.name, season=meta.begin_season) + result = None + title_check = False + if mediaid.startswith("tmdb:"): + tmdbid = mediaid[5:] + if not tmdbid or not str(tmdbid).isdigit(): + return Subscribe() + result = Subscribe.exists(db, tmdbid=int(tmdbid), season=season) + elif mediaid.startswith("douban:"): + doubanid = mediaid[7:] + if not doubanid: + return Subscribe() + result = Subscribe.get_by_doubanid(db, doubanid) + if not result and title: + title_check = True + elif mediaid.startswith("bangumi:"): + bangumiid = mediaid[8:] + if not bangumiid or not str(bangumiid).isdigit(): + return Subscribe() + result = Subscribe.get_by_bangumiid(db, int(bangumiid)) + if not result and title: + title_check = True + # 使用名称检查订阅 + if title_check and title: + meta = MetaInfo(title) + if season: + meta.begin_season = season + result = Subscribe.get_by_title(db, title=meta.name, season=meta.begin_season) - return result if result else Subscribe() + return result if result else Subscribe() @router.get("/refresh", summary="刷新订阅", response_model=schemas.Response)