refactor: 优化缓存策略

2026-02-13 15:37:33 +08:00 · 2024-11-21 15:52:08 +08:00
parent acd9e38477
commit ebffdebdb2
9 changed files with 90 additions and 60 deletions
--- a/app/chain/torrents.py
+++ b/app/chain/torrents.py
@@ -175,7 +175,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton):
            # 按pubdate降序排列
            torrents.sort(key=lambda x: x.pubdate or '', reverse=True)
            # 取前N条
-            torrents = torrents[:settings.CACHE_CONF.get('refresh')]
+            torrents = torrents[:settings.CACHE_CONF["refresh"]]
            if torrents:
                # 过滤出没有处理过的种子
                torrents = [torrent for torrent in torrents
@@ -215,8 +215,8 @@ class TorrentsChain(ChainBase, metaclass=Singleton):
                    else:
                        torrents_cache[domain].append(context)
                    # 如果超过了限制条数则移除掉前面的
-                    if len(torrents_cache[domain]) > settings.CACHE_CONF.get('torrents'):
-                        torrents_cache[domain] = torrents_cache[domain][-settings.CACHE_CONF.get('torrents'):]
+                    if len(torrents_cache[domain]) > settings.CACHE_CONF["torrents"]:
+                        torrents_cache[domain] = torrents_cache[domain][-settings.CACHE_CONF["torrents"]:]
                # 回收资源
                del torrents
            else:
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -8,7 +8,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Type

 from dotenv import set_key
-from pydantic import BaseModel, BaseSettings, validator
+from pydantic import BaseModel, BaseSettings, validator, Field

 from app.log import logger
 from app.utils.system import SystemUtils
@@ -36,7 +36,7 @@ class ConfigModel(BaseModel):
    # RESOURCE密钥
    RESOURCE_SECRET_KEY: str = secrets.token_urlsafe(32)
    # 允许的域名
-    ALLOWED_HOSTS: list = ["*"]
+    ALLOWED_HOSTS: list = Field(default_factory=lambda: ["*"])
    # TOKEN过期时间
    ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
    # RESOURCE_TOKEN过期时间
@@ -114,29 +114,39 @@ class ConfigModel(BaseModel):
    # 是否启用DOH解析域名
    DOH_ENABLE: bool = True
    # 使用 DOH 解析的域名列表
-    DOH_DOMAINS: str = "api.themoviedb.org,api.tmdb.org,webservice.fanart.tv,api.github.com,github.com,raw.githubusercontent.com,api.telegram.org"
+    DOH_DOMAINS: str = ("api.themoviedb.org,"
+                        "api.tmdb.org,"
+                        "webservice.fanart.tv,"
+                        "api.github.com,"
+                        "github.com,"
+                        "raw.githubusercontent.com,"
+                        "api.telegram.org")
    # DOH 解析服务器列表
    DOH_RESOLVERS: str = "1.0.0.1,1.1.1.1,9.9.9.9,149.112.112.112"
    # 支持的后缀格式
-    RMT_MEDIAEXT: list = ['.mp4', '.mkv', '.ts', '.iso',
-                          '.rmvb', '.avi', '.mov', '.mpeg',
-                          '.mpg', '.wmv', '.3gp', '.asf',
-                          '.m4v', '.flv', '.m2ts', '.strm',
-                          '.tp', '.f4v']
+    RMT_MEDIAEXT: list = Field(
+        default_factory=lambda: ['.mp4', '.mkv', '.ts', '.iso',
+                                 '.rmvb', '.avi', '.mov', '.mpeg',
+                                 '.mpg', '.wmv', '.3gp', '.asf',
+                                 '.m4v', '.flv', '.m2ts', '.strm',
+                                 '.tp', '.f4v']
+    )
    # 支持的字幕文件后缀格式
-    RMT_SUBEXT: list = ['.srt', '.ass', '.ssa', '.sup']
+    RMT_SUBEXT: list = Field(default_factory=lambda: ['.srt', '.ass', '.ssa', '.sup'])
    # 支持的音轨文件后缀格式
-    RMT_AUDIO_TRACK_EXT: list = ['.mka']
+    RMT_AUDIO_TRACK_EXT: list = Field(default_factory=lambda: ['.mka'])
    # 音轨文件后缀格式
-    RMT_AUDIOEXT: list = ['.aac', '.ac3', '.amr', '.caf', '.cda', '.dsf',
-                          '.dff', '.kar', '.m4a', '.mp1', '.mp2', '.mp3',
-                          '.mid', '.mod', '.mka', '.mpc', '.nsf', '.ogg',
-                          '.pcm', '.rmi', '.s3m', '.snd', '.spx', '.tak',
-                          '.tta', '.vqf', '.wav', '.wma',
-                          '.aifc', '.aiff', '.alac', '.adif', '.adts',
-                          '.flac', '.midi', '.opus', '.sfalc']
+    RMT_AUDIOEXT: list = Field(
+        default_factory=lambda: ['.aac', '.ac3', '.amr', '.caf', '.cda', '.dsf',
+                                 '.dff', '.kar', '.m4a', '.mp1', '.mp2', '.mp3',
+                                 '.mid', '.mod', '.mka', '.mpc', '.nsf', '.ogg',
+                                 '.pcm', '.rmi', '.s3m', '.snd', '.spx', '.tak',
+                                 '.tta', '.vqf', '.wav', '.wma',
+                                 '.aifc', '.aiff', '.alac', '.adif', '.adts',
+                                 '.flac', '.midi', '.opus', '.sfalc']
+    )
    # 下载器临时文件后缀
-    DOWNLOAD_TMPEXT: list = ['.!qb', '.part']
+    DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: ['.!qb', '.part'])
    # 媒体服务器同步间隔（小时）
    MEDIASERVER_SYNC_INTERVAL: int = 6
    # 订阅模式
@@ -189,7 +199,10 @@ class ConfigModel(BaseModel):
    # 服务器地址，对应 https://github.com/jxxghp/MoviePilot-Server 项目
    MP_SERVER_HOST: str = "https://movie-pilot.org"
    # 插件市场仓库地址，多个地址使用,分隔，地址以/结尾
-    PLUGIN_MARKET: str = "https://github.com/jxxghp/MoviePilot-Plugins,https://github.com/thsrite/MoviePilot-Plugins,https://github.com/honue/MoviePilot-Plugins,https://github.com/InfinityPacer/MoviePilot-Plugins"
+    PLUGIN_MARKET: str = ("https://github.com/jxxghp/MoviePilot-Plugins,"
+                          "https://github.com/thsrite/MoviePilot-Plugins,"
+                          "https://github.com/honue/MoviePilot-Plugins,"
+                          "https://github.com/InfinityPacer/MoviePilot-Plugins")
    # 插件安装数据共享
    PLUGIN_STATISTIC_SHARE: bool = True
    # 是否开启插件热加载
@@ -207,10 +220,18 @@ class ConfigModel(BaseModel):
    # 全局图片缓存，将媒体图片缓存到本地
    GLOBAL_IMAGE_CACHE: bool = False
    # 允许的图片缓存域名
-    SECURITY_IMAGE_DOMAINS: List[str] = ["image.tmdb.org", "static-mdb.v.geilijiasu.com", "doubanio.com", "lain.bgm.tv",
-                                         "raw.githubusercontent.com", "github.com"]
+    SECURITY_IMAGE_DOMAINS: List[str] = Field(
+        default_factory=lambda: ["image.tmdb.org",
+                                 "static-mdb.v.geilijiasu.com",
+                                 "doubanio.com",
+                                 "lain.bgm.tv",
+                                 "raw.githubusercontent.com",
+                                 "github.com"]
+    )
    # 允许的图片文件后缀格式
-    SECURITY_IMAGE_SUFFIXES: List[str] = [".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg"]
+    SECURITY_IMAGE_SUFFIXES: List[str] = Field(
+        default_factory=lambda: [".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg"]
+    )


 class Settings(BaseSettings, ConfigModel):
@@ -437,22 +458,32 @@ class Settings(BaseSettings, ConfigModel):

    @property
    def CACHE_CONF(self):
+        """
+        {
+            "torrents": "缓存种子数量",
+            "refresh": "订阅刷新处理数量",
+            "tmdb": "TMDB请求缓存数量",
+            "douban": "豆瓣请求缓存数量",
+            "fanart": "Fanart请求缓存数量",
+            "meta": "元数据缓存过期时间（秒）"
+        }
+        """
        if self.BIG_MEMORY_MODE:
            return {
+                "torrents": 200,
+                "refresh": 100,
                "tmdb": 1024,
-                "refresh": 50,
-                "torrents": 100,
                "douban": 512,
                "fanart": 512,
-                "meta": (self.META_CACHE_EXPIRE or 168) * 3600
+                "meta": (self.META_CACHE_EXPIRE or 24) * 3600
            }
        return {
+            "torrents": 100,
+            "refresh": 50,
            "tmdb": 256,
-            "refresh": 30,
-            "torrents": 50,
            "douban": 256,
            "fanart": 128,
-            "meta": (self.META_CACHE_EXPIRE or 72) * 3600
+            "meta": (self.META_CACHE_EXPIRE or 2) * 3600
        }

    @property
--- a/app/modules/douban/apiv2.py
+++ b/app/modules/douban/apiv2.py
@@ -3,11 +3,11 @@ import base64
 import hashlib
 import hmac
 from datetime import datetime
-from functools import lru_cache
 from random import choice
 from urllib import parse

 import requests
+from cachetools import TTLCache, cached

 from app.core.config import settings
 from app.utils.http import RequestUtils
@@ -160,12 +160,12 @@ class DoubanApi(metaclass=Singleton):
        self._session = requests.Session()

    @classmethod
-    def __sign(cls, url: str, ts: int, method='GET') -> str:
+    def __sign(cls, url: str, ts: str, method='GET') -> str:
        """
        签名
        """
        url_path = parse.urlparse(url).path
-        raw_sign = '&'.join([method.upper(), parse.quote(url_path, safe=''), str(ts)])
+        raw_sign = '&'.join([method.upper(), parse.quote(url_path, safe=''), ts])
        return base64.b64encode(
            hmac.new(
                cls._api_secret_key.encode(),
@@ -174,7 +174,7 @@ class DoubanApi(metaclass=Singleton):
            ).digest()
        ).decode()

-    @lru_cache(maxsize=settings.CACHE_CONF.get('douban'))
+    @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]))
    def __invoke(self, url: str, **kwargs) -> dict:
        """
        GET请求
@@ -203,7 +203,7 @@ class DoubanApi(metaclass=Singleton):
            return resp.json()
        return resp.json() if resp else {}

-    @lru_cache(maxsize=settings.CACHE_CONF.get('douban'))
+    @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]))
    def __post(self, url: str, **kwargs) -> dict:
        """
        POST请求
--- a/app/modules/douban/douban_cache.py
+++ b/app/modules/douban/douban_cache.py
@@ -16,7 +16,7 @@ from app.schemas.types import MediaType
 lock = RLock()

 CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp"
-EXPIRE_TIMESTAMP = settings.CACHE_CONF.get('meta')
+EXPIRE_TIMESTAMP = settings.CACHE_CONF["meta"]


 class DoubanCache(metaclass=Singleton):
@@ -77,7 +77,7 @@ class DoubanCache(metaclass=Singleton):
        @return: 被删除的缓存内容
        """
        with lock:
-            return self._meta_data.pop(key, None)
+            return self._meta_data.pop(key, {})

    def delete_by_doubanid(self, doubanid: str) -> None:
        """
--- a/app/modules/fanart/init.py
+++ b/app/modules/fanart/init.py
@@ -1,12 +1,13 @@
 import re
-from functools import lru_cache
 from typing import Optional, Tuple, Union

+from cachetools import TTLCache, cached
+
 from app.core.context import MediaInfo, settings
 from app.log import logger
 from app.modules import _ModuleBase
-from app.utils.http import RequestUtils
 from app.schemas.types import MediaType, ModuleType
+from app.utils.http import RequestUtils


 class FanartModule(_ModuleBase):
@@ -404,7 +405,7 @@ class FanartModule(_ModuleBase):
        return result

    @classmethod
-    @lru_cache(maxsize=settings.CACHE_CONF.get('fanart'))
+    @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["fanart"], ttl=settings.CACHE_CONF["meta"]))
    def __request_fanart(cls, media_type: MediaType, queryid: Union[str, int]) -> Optional[dict]:
        if media_type == MediaType.MOVIE:
            image_url = cls._movie_url % queryid
--- a/app/modules/themoviedb/tmdb_cache.py
+++ b/app/modules/themoviedb/tmdb_cache.py
@@ -15,7 +15,7 @@ from app.schemas.types import MediaType
 lock = RLock()

 CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp"
-EXPIRE_TIMESTAMP = settings.CACHE_CONF.get('meta')
+EXPIRE_TIMESTAMP = settings.CACHE_CONF["meta"]


 class TmdbCache(metaclass=Singleton):
@@ -75,7 +75,7 @@ class TmdbCache(metaclass=Singleton):
        @return: 被删除的缓存内容
        """
        with lock:
-            return self._meta_data.pop(key, None)
+            return self._meta_data.pop(key, {})

    def delete_by_tmdbid(self, tmdbid: int) -> None:
        """
@@ -138,14 +138,14 @@ class TmdbCache(metaclass=Singleton):
                if cache_year:
                    cache_year = cache_year[:4]
                self._meta_data[self.__get_key(meta)] = {
-                        "id": info.get("id"),
-                        "type": info.get("media_type"),
-                        "year": cache_year,
-                        "title": cache_title,
-                        "poster_path": info.get("poster_path"),
-                        "backdrop_path": info.get("backdrop_path"),
-                        CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP
-                    }
+                    "id": info.get("id"),
+                    "type": info.get("media_type"),
+                    "year": cache_year,
+                    "title": cache_title,
+                    "poster_path": info.get("poster_path"),
+                    "backdrop_path": info.get("backdrop_path"),
+                    CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP
+                }
            elif info is not None:
                # None时不缓存，此时代表网络错误，允许重复请求
                self._meta_data[self.__get_key(meta)] = {'id': 0}
@@ -164,7 +164,7 @@ class TmdbCache(metaclass=Singleton):
            return

        with open(self._meta_path, 'wb') as f:
-            pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL)
+            pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL)  # type: ignore

    def _random_sample(self, new_meta_data: dict) -> bool:
        """
--- a/app/modules/themoviedb/tmdbapi.py
+++ b/app/modules/themoviedb/tmdbapi.py
@@ -1,9 +1,9 @@
 import traceback
-from functools import lru_cache
 from typing import Optional, List
 from urllib.parse import quote

 import zhconv
+from cachetools import TTLCache, cached
 from lxml import etree

 from app.core.config import settings
@@ -27,8 +27,6 @@ class TmdbApi:
        self.tmdb.domain = settings.TMDB_API_DOMAIN
        # 开启缓存
        self.tmdb.cache = True
-        # 缓存大小
-        self.tmdb.REQUEST_CACHE_MAXSIZE = settings.CACHE_CONF.get('tmdb')
        # APIKEY
        self.tmdb.api_key = settings.TMDB_API_KEY
        # 语种
@@ -466,7 +464,7 @@ class TmdbApi:

            return ret_info

-    @lru_cache(maxsize=settings.CACHE_CONF.get('tmdb'))
+    @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"]))
    def match_web(self, name: str, mtype: MediaType) -> Optional[dict]:
        """
        搜索TMDB网站，直接抓取结果，结果只有一条时才返回
--- a/app/modules/themoviedb/tmdbv3api/tmdb.py
+++ b/app/modules/themoviedb/tmdbv3api/tmdb.py
@@ -4,11 +4,12 @@ import logging
 import os
 import time
 from datetime import datetime
-from functools import lru_cache

 import requests
 import requests.exceptions
+from cachetools import TTLCache, cached

+from app.core.config import settings
 from app.utils.http import RequestUtils
 from .exceptions import TMDbException

@@ -24,7 +25,6 @@ class TMDb(object):
    TMDB_CACHE_ENABLED = "TMDB_CACHE_ENABLED"
    TMDB_PROXIES = "TMDB_PROXIES"
    TMDB_DOMAIN = "TMDB_DOMAIN"
-    REQUEST_CACHE_MAXSIZE = None

    _req = None
    _session = None
@@ -137,7 +137,7 @@ class TMDb(object):
    def cache(self, cache):
        os.environ[self.TMDB_CACHE_ENABLED] = str(cache)

-    @lru_cache(maxsize=REQUEST_CACHE_MAXSIZE)
+    @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"]))
    def cached_request(self, method, url, data, json,
                       _ts=datetime.strftime(datetime.now(), '%Y%m%d')):
        """
--- a/app/scheduler.py
+++ b/app/scheduler.py
@@ -330,7 +330,7 @@ class Scheduler(metaclass=Singleton):
            "interval",
            id="clear_cache",
            name="缓存清理",
-            hours=settings.CACHE_CONF.get("meta") / 3600,
+            hours=settings.CACHE_CONF["meta"] / 3600,
            kwargs={
                'job_id': 'clear_cache'
            }