From 69cb07c527bc1e51778e4d4f13959bc4ffcc26ca Mon Sep 17 00:00:00 2001 From: jxxghp Date: Wed, 20 Aug 2025 09:16:30 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BC=93=E5=AD=98=E6=9C=BA?= =?UTF-8?q?=E5=88=B6=EF=BC=8C=E6=94=AF=E6=8C=81Redis=E5=92=8C=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E7=BC=93=E5=AD=98=E7=9A=84=E5=88=87=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/chain/__init__.py | 1 + app/modules/douban/douban_cache.py | 221 ++++++++++++++++----------- app/modules/themoviedb/tmdb_cache.py | 196 +++++++++++++++--------- 3 files changed, 256 insertions(+), 162 deletions(-) diff --git a/app/chain/__init__.py b/app/chain/__init__.py index a44814e6..fb585500 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -54,6 +54,7 @@ class ChainBase(metaclass=ABCMeta): try: self._redis_helper = RedisHelper(redis_url=settings.CACHE_BACKEND_URL) except RuntimeError as e: + self._redis_helper = None logger.warning(f"Redis缓存初始化失败，将使用本地缓存: {e}") def load_cache(self, filename: str) -> Any: diff --git a/app/modules/douban/douban_cache.py b/app/modules/douban/douban_cache.py index 95fc7511..370b3dd4 100644 --- a/app/modules/douban/douban_cache.py +++ b/app/modules/douban/douban_cache.py @@ -9,9 +9,10 @@ from typing import Optional from app.core.config import settings from app.core.meta import MetaBase from app.core.metainfo import MetaInfo +from app.helper.redis import RedisHelper from app.log import logger -from app.utils.singleton import WeakSingleton from app.schemas.types import MediaType +from app.utils.singleton import WeakSingleton lock = RLock() @@ -30,18 +31,35 @@ class DoubanCache(metaclass=WeakSingleton): } """ # TMDB缓存过期 - _tmdb_cache_expire: bool = True + _douban_cache_expire: bool = True def __init__(self): + # 初始化Redis缓存助手 + self._redis_helper = None + if settings.CACHE_BACKEND_TYPE == "redis": + try: + self._redis_helper = RedisHelper(redis_url=settings.CACHE_BACKEND_URL) + except RuntimeError as e: + logger.warning(f"豆瓣缓存Redis初始化失败，将使用本地缓存: {e}") + self._redis_helper = None + # 加载本地缓存数据 self._meta_path = settings.TEMP_PATH / "__douban_cache__" - self._meta_data = self.__load(self._meta_path) + if not self._redis_helper: + self._meta_data = self.__load(self._meta_path) def clear(self): """ - 清空所有TMDB缓存 + 清空所有豆瓣缓存 """ with lock: self._meta_data = {} + # 如果Redis可用，同时清理Redis缓存 + if self._redis_helper: + try: + self._redis_helper.clear(region="douban_cache") + logger.debug("已清理豆瓣Redis缓存") + except Exception as e: + logger.warning(f"清理豆瓣Redis缓存失败: {e}") @staticmethod def __get_key(meta: MetaBase) -> str: @@ -56,16 +74,28 @@ class DoubanCache(metaclass=WeakSingleton): 根据KEY值获取缓存值 """ key = self.__get_key(meta) - with lock: - info: dict = self._meta_data.get(key) - if info: - expire = info.get(CACHE_EXPIRE_TIMESTAMP_STR) - if not expire or int(time.time()) < expire: - info[CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP - self._meta_data[key] = info - elif expire and self._tmdb_cache_expire: - self.delete(key) - return info or {} + + if self._redis_helper: + # 如果Redis可用，从Redis读取 + try: + redis_data = self._redis_helper.get(key, region="douban_cache") + return redis_data or {} + except Exception as e: + logger.warning(f"从Redis获取豆瓣缓存失败: {e}") + else: + # Redis不可用时，从内存缓存读取 + with lock: + info: dict = self._meta_data.get(key) + if info: + # 检查过期时间 + expire = info.get(CACHE_EXPIRE_TIMESTAMP_STR) + if not expire or int(time.time()) < expire: + info[CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP + self._meta_data[key] = info + elif expire and self._douban_cache_expire: + self.delete(key) + return info or {} + return {} def delete(self, key: str) -> dict: """ @@ -73,39 +103,44 @@ class DoubanCache(metaclass=WeakSingleton): @param key: 缓存key @return: 被删除的缓存内容 """ - with lock: - return self._meta_data.pop(key, {}) - - def delete_by_doubanid(self, doubanid: str) -> None: - """ - 清空对应豆瓣ID的所有缓存记录，以强制更新TMDB中最新的数据 - """ - for key in list(self._meta_data): - if self._meta_data.get(key, {}).get("id") == doubanid: - with lock: - self._meta_data.pop(key) - - def delete_unknown(self) -> None: - """ - 清除未识别的缓存记录，以便重新搜索TMDB - """ - for key in list(self._meta_data): - if self._meta_data.get(key, {}).get("id") == "0": - with lock: - self._meta_data.pop(key) + if self._redis_helper: + # 如果Redis可用，删除Redis缓存 + try: + self._redis_helper.delete(key, region="douban_cache") + return {} + except Exception as e: + logger.warning(f"删除豆瓣Redis缓存失败: {e}") + return {} + else: + # Redis不可用时，删除内存缓存 + with lock: + return self._meta_data.pop(key, {}) def modify(self, key: str, title: str) -> dict: """ - 删除缓存信息 + 修改缓存信息 @param key: 缓存key @param title: 标题 @return: 被修改后缓存内容 """ - with lock: - if self._meta_data.get(key): - self._meta_data[key]['title'] = title - self._meta_data[key][CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP - return self._meta_data.get(key) + if self._redis_helper: + # 如果Redis可用，修改Redis缓存 + try: + redis_data = self._redis_helper.get(key, region="douban_cache") + if redis_data: + redis_data['title'] = title + self._redis_helper.set(key, redis_data, ttl=EXPIRE_TIMESTAMP, region="douban_cache") + return redis_data + except Exception as e: + logger.warning(f"修改豆瓣Redis缓存失败: {e}") + return {} + else: + # Redis不可用时，修改内存缓存 + with lock: + if self._meta_data.get(key): + self._meta_data[key]['title'] = title + self._meta_data[key][CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP + return self._meta_data.get(key) @staticmethod def __load(path: Path) -> dict: @@ -126,31 +161,47 @@ class DoubanCache(metaclass=WeakSingleton): """ 新增或更新缓存条目 """ - with lock: - if info: - # 缓存标题 - cache_title = info.get("title") - # 缓存年份 - cache_year = info.get('year') - # 类型 - if isinstance(info.get('media_type'), MediaType): - mtype = info.get('media_type') - elif info.get("type"): - mtype = MediaType.MOVIE if info.get("type") == "movie" else MediaType.TV + if info: + # 缓存标题 + cache_title = info.get("title") + # 缓存年份 + cache_year = info.get('year') + # 类型 + if isinstance(info.get('media_type'), MediaType): + mtype = info.get('media_type') + elif info.get("type"): + mtype = MediaType.MOVIE if info.get("type") == "movie" else MediaType.TV + else: + meta = MetaInfo(cache_title) + if meta.begin_season: + mtype = MediaType.TV else: - meta = MetaInfo(cache_title) - if meta.begin_season: - mtype = MediaType.TV - else: - mtype = MediaType.MOVIE - # 海报 - poster_path = info.get("pic", {}).get("large") - if not poster_path and info.get("cover_url"): - poster_path = info.get("cover_url") - if not poster_path and info.get("cover"): - poster_path = info.get("cover").get("url") + mtype = MediaType.MOVIE + # 海报 + poster_path = info.get("pic", {}).get("large") + if not poster_path and info.get("cover_url"): + poster_path = info.get("cover_url") + if not poster_path and info.get("cover"): + poster_path = info.get("cover").get("url") - self._meta_data[self.__get_key(meta)] = { + if self._redis_helper: + # 如果Redis可用，保存到Redis + cache_data = { + "id": info.get("id"), + "type": mtype, + "year": cache_year, + "title": cache_title, + "poster_path": poster_path + } + try: + self._redis_helper.set(self.__get_key(meta), cache_data, ttl=EXPIRE_TIMESTAMP, + region="douban_cache") + except Exception as e: + logger.warning(f"保存豆瓣缓存到Redis失败: {e}") + else: + # Redis不可用时，保存到内存缓存 + with lock: + cache_data = { "id": info.get("id"), "type": mtype, "year": cache_year, @@ -158,15 +209,29 @@ class DoubanCache(metaclass=WeakSingleton): "poster_path": poster_path, CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP } - elif info is not None: - # None时不缓存，此时代表网络错误，允许重复请求 - self._meta_data[self.__get_key(meta)] = {'id': "0"} + self._meta_data[self.__get_key(meta)] = cache_data + + elif info is not None: + # None时不缓存，此时代表网络错误，允许重复请求 + if self._redis_helper: + try: + self._redis_helper.set(self.__get_key(meta), {'id': "0"}, ttl=EXPIRE_TIMESTAMP, + region="douban_cache") + except Exception as e: + logger.warning(f"保存豆瓣缓存到Redis失败: {e}") + else: + with lock: + self._meta_data[self.__get_key(meta)] = {'id': "0"} def save(self, force: Optional[bool] = False) -> None: """ 保存缓存数据到文件 """ + # 如果Redis可用，不需要保存到本地文件 + if self._redis_helper: + return + # Redis不可用时，保存到本地文件 meta_data = self.__load(self._meta_path) new_meta_data = {k: v for k, v in self._meta_data.items() if v.get("id")} @@ -176,7 +241,7 @@ class DoubanCache(metaclass=WeakSingleton): return with open(self._meta_path, 'wb') as f: - pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) # noqa + pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) # noqa def _random_sample(self, new_meta_data: dict) -> bool: """ @@ -193,7 +258,7 @@ class DoubanCache(metaclass=WeakSingleton): info[CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP elif int(time.time()) >= expire: ret = True - if self._tmdb_cache_expire: + if self._douban_cache_expire: new_meta_data.pop(k) else: count = 0 @@ -206,30 +271,12 @@ class DoubanCache(metaclass=WeakSingleton): info[CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP elif int(time.time()) >= expire: ret = True - if self._tmdb_cache_expire: + if self._douban_cache_expire: new_meta_data.pop(k) count += 1 if count >= 5: ret |= self._random_sample(new_meta_data) return ret - def get_title(self, key: str) -> Optional[str]: - """ - 获取缓存的标题 - """ - cache_media_info = self._meta_data.get(key) - if not cache_media_info or not cache_media_info.get("id"): - return None - return cache_media_info.get("title") - - def set_title(self, key: str, cn_title: str) -> None: - """ - 重新设置缓存标题 - """ - cache_media_info = self._meta_data.get(key) - if not cache_media_info: - return - self._meta_data[key]['title'] = cn_title - def __del__(self): self.save() diff --git a/app/modules/themoviedb/tmdb_cache.py b/app/modules/themoviedb/tmdb_cache.py index 61b7c4e4..13482617 100644 --- a/app/modules/themoviedb/tmdb_cache.py +++ b/app/modules/themoviedb/tmdb_cache.py @@ -4,13 +4,13 @@ import time import traceback from pathlib import Path from threading import RLock -from typing import Optional from app.core.config import settings from app.core.meta import MetaBase +from app.helper.redis import RedisHelper from app.log import logger -from app.utils.singleton import WeakSingleton from app.schemas.types import MediaType +from app.utils.singleton import WeakSingleton lock = RLock() @@ -32,8 +32,19 @@ class TmdbCache(metaclass=WeakSingleton): _tmdb_cache_expire: bool = True def __init__(self): + # 初始化Redis缓存助手 + self._redis_helper = None + if settings.CACHE_BACKEND_TYPE == "redis": + try: + self._redis_helper = RedisHelper(redis_url=settings.CACHE_BACKEND_URL) + except RuntimeError as e: + logger.warning(f"TMDB缓存Redis初始化失败，将使用本地缓存: {e}") + self._redis_helper = None + + # 加载缓存数据 self._meta_path = settings.TEMP_PATH / "__tmdb_cache__" - self._meta_data = self.__load(self._meta_path) + if not self._redis_helper: + self._meta_data = self.__load(self._meta_path) def clear(self): """ @@ -41,6 +52,13 @@ class TmdbCache(metaclass=WeakSingleton): """ with lock: self._meta_data = {} + # 如果Redis可用，同时清理Redis缓存 + if self._redis_helper: + try: + self._redis_helper.clear(region="tmdb_cache") + logger.debug("已清理TMDB Redis缓存") + except Exception as e: + logger.warning(f"清理TMDB Redis缓存失败: {e}") @staticmethod def __get_key(meta: MetaBase) -> str: @@ -54,16 +72,28 @@ class TmdbCache(metaclass=WeakSingleton): 根据KEY值获取缓存值 """ key = self.__get_key(meta) - with lock: - info: dict = self._meta_data.get(key) - if info: - expire = info.get(CACHE_EXPIRE_TIMESTAMP_STR) - if not expire or int(time.time()) < expire: - info[CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP - self._meta_data[key] = info - elif expire and self._tmdb_cache_expire: - self.delete(key) - return info or {} + + if self._redis_helper: + # 如果Redis可用，从Redis读取 + try: + redis_data = self._redis_helper.get(key, region="tmdb_cache") + return redis_data or {} + except Exception as e: + logger.warning(f"从Redis获取TMDB缓存失败: {e}") + else: + # Redis不可用时，从内存缓存读取 + with lock: + info: dict = self._meta_data.get(key) + if info: + # 检查过期时间 + expire = info.get(CACHE_EXPIRE_TIMESTAMP_STR) + if not expire or int(time.time()) < expire: + info[CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP + self._meta_data[key] = info + elif expire and self._tmdb_cache_expire: + self.delete(key) + return info or {} + return {} def delete(self, key: str) -> dict: """ @@ -71,39 +101,44 @@ class TmdbCache(metaclass=WeakSingleton): @param key: 缓存key @return: 被删除的缓存内容 """ - with lock: - return self._meta_data.pop(key, {}) - - def delete_by_tmdbid(self, tmdbid: int) -> None: - """ - 清空对应TMDBID的所有缓存记录，以强制更新TMDB中最新的数据 - """ - for key in list(self._meta_data): - if self._meta_data.get(key, {}).get("id") == tmdbid: - with lock: - self._meta_data.pop(key) - - def delete_unknown(self) -> None: - """ - 清除未识别的缓存记录，以便重新搜索TMDB - """ - for key in list(self._meta_data): - if self._meta_data.get(key, {}).get("id") == 0: - with lock: - self._meta_data.pop(key) + if self._redis_helper: + # 如果Redis可用，删除Redis缓存 + try: + self._redis_helper.delete(key, region="tmdb_cache") + return {} + except Exception as e: + logger.warning(f"删除TMDB Redis缓存失败: {e}") + return {} + else: + # Redis不可用时，删除内存缓存 + with lock: + return self._meta_data.pop(key, {}) def modify(self, key: str, title: str) -> dict: """ - 删除缓存信息 + 修改缓存信息 @param key: 缓存key @param title: 标题 @return: 被修改后缓存内容 """ - with lock: - if self._meta_data.get(key): - self._meta_data[key]['title'] = title - self._meta_data[key][CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP - return self._meta_data.get(key) + if self._redis_helper: + # 如果Redis可用，修改Redis缓存 + try: + redis_data = self._redis_helper.get(key, region="tmdb_cache") + if redis_data: + redis_data['title'] = title + self._redis_helper.set(key, redis_data, ttl=EXPIRE_TIMESTAMP, region="tmdb_cache") + return redis_data + except Exception as e: + logger.warning(f"修改TMDB Redis缓存失败: {e}") + return {} + else: + # Redis不可用时，修改内存缓存 + with lock: + if self._meta_data.get(key): + self._meta_data[key]['title'] = title + self._meta_data[key][CACHE_EXPIRE_TIMESTAMP_STR] = int(time.time()) + EXPIRE_TIMESTAMP + return self._meta_data.get(key) @staticmethod def __load(path: Path) -> dict: @@ -115,43 +150,72 @@ class TmdbCache(metaclass=WeakSingleton): with open(path, 'rb') as f: data = pickle.load(f) return data - return {} except Exception as e: logger.error(f'加载缓存失败：{str(e)} - {traceback.format_exc()}') - return {} + return {} def update(self, meta: MetaBase, info: dict) -> None: """ 新增或更新缓存条目 """ - with lock: - if info: - # 缓存标题 - cache_title = info.get("title") \ - if info.get("media_type") == MediaType.MOVIE else info.get("name") - # 缓存年份 - cache_year = info.get('release_date') \ - if info.get("media_type") == MediaType.MOVIE else info.get('first_air_date') - if cache_year: - cache_year = cache_year[:4] - self._meta_data[self.__get_key(meta)] = { + if info: + # 缓存标题 + cache_title = info.get("title") \ + if info.get("media_type") == MediaType.MOVIE else info.get("name") + # 缓存年份 + cache_year = info.get('release_date') \ + if info.get("media_type") == MediaType.MOVIE else info.get('first_air_date') + if cache_year: + cache_year = cache_year[:4] + + if self._redis_helper: + # 如果Redis可用，保存到Redis + cache_data = { "id": info.get("id"), "type": info.get("media_type"), "year": cache_year, "title": cache_title, "poster_path": info.get("poster_path"), - "backdrop_path": info.get("backdrop_path"), - CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP + "backdrop_path": info.get("backdrop_path") } - elif info is not None: - # None时不缓存，此时代表网络错误，允许重复请求 - self._meta_data[self.__get_key(meta)] = {'id': 0} + try: + self._redis_helper.set(self.__get_key(meta), cache_data, ttl=EXPIRE_TIMESTAMP, region="tmdb_cache") + except Exception as e: + logger.warning(f"保存TMDB缓存到Redis失败: {e}") + else: + # Redis不可用时，保存到内存缓存 + with lock: + cache_data = { + "id": info.get("id"), + "type": info.get("media_type"), + "year": cache_year, + "title": cache_title, + "poster_path": info.get("poster_path"), + "backdrop_path": info.get("backdrop_path"), + CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP + } + self._meta_data[self.__get_key(meta)] = cache_data + + elif info is not None: + # None时不缓存，此时代表网络错误，允许重复请求 + if self._redis_helper: + try: + self._redis_helper.set(self.__get_key(meta), {'id': 0}, ttl=EXPIRE_TIMESTAMP, region="tmdb_cache") + except Exception as e: + logger.warning(f"保存TMDB缓存到Redis失败: {e}") + else: + with lock: + self._meta_data[self.__get_key(meta)] = {'id': 0} def save(self, force: bool = False) -> None: """ 保存缓存数据到文件 """ + # 如果Redis可用，不需要保存到本地文件 + if self._redis_helper: + return + # Redis不可用时，保存到本地文件 meta_data = self.__load(self._meta_path) new_meta_data = {k: v for k, v in self._meta_data.items() if v.get("id")} @@ -198,23 +262,5 @@ class TmdbCache(metaclass=WeakSingleton): ret |= self._random_sample(new_meta_data) return ret - def get_title(self, key: str) -> Optional[str]: - """ - 获取缓存的标题 - """ - cache_media_info = self._meta_data.get(key) - if not cache_media_info or not cache_media_info.get("id"): - return None - return cache_media_info.get("title") - - def set_title(self, key: str, cn_title: str) -> None: - """ - 重新设置缓存标题 - """ - cache_media_info = self._meta_data.get(key) - if not cache_media_info: - return - self._meta_data[key]['title'] = cn_title - def __del__(self): self.save()