diff --git a/app/chain/torrents.py b/app/chain/torrents.py index 9dd91837..a352cc21 100644 --- a/app/chain/torrents.py +++ b/app/chain/torrents.py @@ -175,7 +175,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton): # 按pubdate降序排列 torrents.sort(key=lambda x: x.pubdate or '', reverse=True) # 取前N条 - torrents = torrents[:settings.CACHE_CONF.get('refresh')] + torrents = torrents[:settings.CACHE_CONF["refresh"]] if torrents: # 过滤出没有处理过的种子 torrents = [torrent for torrent in torrents @@ -215,8 +215,8 @@ class TorrentsChain(ChainBase, metaclass=Singleton): else: torrents_cache[domain].append(context) # 如果超过了限制条数则移除掉前面的 - if len(torrents_cache[domain]) > settings.CACHE_CONF.get('torrents'): - torrents_cache[domain] = torrents_cache[domain][-settings.CACHE_CONF.get('torrents'):] + if len(torrents_cache[domain]) > settings.CACHE_CONF["torrents"]: + torrents_cache[domain] = torrents_cache[domain][-settings.CACHE_CONF["torrents"]:] # 回收资源 del torrents else: diff --git a/app/core/config.py b/app/core/config.py index 65f3ff3d..aa47543e 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Type from dotenv import set_key -from pydantic import BaseModel, BaseSettings, validator +from pydantic import BaseModel, BaseSettings, validator, Field from app.log import logger from app.utils.system import SystemUtils @@ -36,7 +36,7 @@ class ConfigModel(BaseModel): # RESOURCE密钥 RESOURCE_SECRET_KEY: str = secrets.token_urlsafe(32) # 允许的域名 - ALLOWED_HOSTS: list = ["*"] + ALLOWED_HOSTS: list = Field(default_factory=lambda: ["*"]) # TOKEN过期时间 ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8 # RESOURCE_TOKEN过期时间 @@ -114,29 +114,39 @@ class ConfigModel(BaseModel): # 是否启用DOH解析域名 DOH_ENABLE: bool = True # 使用 DOH 解析的域名列表 - DOH_DOMAINS: str = "api.themoviedb.org,api.tmdb.org,webservice.fanart.tv,api.github.com,github.com,raw.githubusercontent.com,api.telegram.org" + DOH_DOMAINS: str = ("api.themoviedb.org," + "api.tmdb.org," + "webservice.fanart.tv," + "api.github.com," + "github.com," + "raw.githubusercontent.com," + "api.telegram.org") # DOH 解析服务器列表 DOH_RESOLVERS: str = "1.0.0.1,1.1.1.1,9.9.9.9,149.112.112.112" # 支持的后缀格式 - RMT_MEDIAEXT: list = ['.mp4', '.mkv', '.ts', '.iso', - '.rmvb', '.avi', '.mov', '.mpeg', - '.mpg', '.wmv', '.3gp', '.asf', - '.m4v', '.flv', '.m2ts', '.strm', - '.tp', '.f4v'] + RMT_MEDIAEXT: list = Field( + default_factory=lambda: ['.mp4', '.mkv', '.ts', '.iso', + '.rmvb', '.avi', '.mov', '.mpeg', + '.mpg', '.wmv', '.3gp', '.asf', + '.m4v', '.flv', '.m2ts', '.strm', + '.tp', '.f4v'] + ) # 支持的字幕文件后缀格式 - RMT_SUBEXT: list = ['.srt', '.ass', '.ssa', '.sup'] + RMT_SUBEXT: list = Field(default_factory=lambda: ['.srt', '.ass', '.ssa', '.sup']) # 支持的音轨文件后缀格式 - RMT_AUDIO_TRACK_EXT: list = ['.mka'] + RMT_AUDIO_TRACK_EXT: list = Field(default_factory=lambda: ['.mka']) # 音轨文件后缀格式 - RMT_AUDIOEXT: list = ['.aac', '.ac3', '.amr', '.caf', '.cda', '.dsf', - '.dff', '.kar', '.m4a', '.mp1', '.mp2', '.mp3', - '.mid', '.mod', '.mka', '.mpc', '.nsf', '.ogg', - '.pcm', '.rmi', '.s3m', '.snd', '.spx', '.tak', - '.tta', '.vqf', '.wav', '.wma', - '.aifc', '.aiff', '.alac', '.adif', '.adts', - '.flac', '.midi', '.opus', '.sfalc'] + RMT_AUDIOEXT: list = Field( + default_factory=lambda: ['.aac', '.ac3', '.amr', '.caf', '.cda', '.dsf', + '.dff', '.kar', '.m4a', '.mp1', '.mp2', '.mp3', + '.mid', '.mod', '.mka', '.mpc', '.nsf', '.ogg', + '.pcm', '.rmi', '.s3m', '.snd', '.spx', '.tak', + '.tta', '.vqf', '.wav', '.wma', + '.aifc', '.aiff', '.alac', '.adif', '.adts', + '.flac', '.midi', '.opus', '.sfalc'] + ) # 下载器临时文件后缀 - DOWNLOAD_TMPEXT: list = ['.!qb', '.part'] + DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: ['.!qb', '.part']) # 媒体服务器同步间隔(小时) MEDIASERVER_SYNC_INTERVAL: int = 6 # 订阅模式 @@ -189,7 +199,10 @@ class ConfigModel(BaseModel): # 服务器地址,对应 https://github.com/jxxghp/MoviePilot-Server 项目 MP_SERVER_HOST: str = "https://movie-pilot.org" # 插件市场仓库地址,多个地址使用,分隔,地址以/结尾 - PLUGIN_MARKET: str = "https://github.com/jxxghp/MoviePilot-Plugins,https://github.com/thsrite/MoviePilot-Plugins,https://github.com/honue/MoviePilot-Plugins,https://github.com/InfinityPacer/MoviePilot-Plugins" + PLUGIN_MARKET: str = ("https://github.com/jxxghp/MoviePilot-Plugins," + "https://github.com/thsrite/MoviePilot-Plugins," + "https://github.com/honue/MoviePilot-Plugins," + "https://github.com/InfinityPacer/MoviePilot-Plugins") # 插件安装数据共享 PLUGIN_STATISTIC_SHARE: bool = True # 是否开启插件热加载 @@ -207,10 +220,18 @@ class ConfigModel(BaseModel): # 全局图片缓存,将媒体图片缓存到本地 GLOBAL_IMAGE_CACHE: bool = False # 允许的图片缓存域名 - SECURITY_IMAGE_DOMAINS: List[str] = ["image.tmdb.org", "static-mdb.v.geilijiasu.com", "doubanio.com", "lain.bgm.tv", - "raw.githubusercontent.com", "github.com"] + SECURITY_IMAGE_DOMAINS: List[str] = Field( + default_factory=lambda: ["image.tmdb.org", + "static-mdb.v.geilijiasu.com", + "doubanio.com", + "lain.bgm.tv", + "raw.githubusercontent.com", + "github.com"] + ) # 允许的图片文件后缀格式 - SECURITY_IMAGE_SUFFIXES: List[str] = [".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg"] + SECURITY_IMAGE_SUFFIXES: List[str] = Field( + default_factory=lambda: [".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg"] + ) class Settings(BaseSettings, ConfigModel): @@ -437,22 +458,32 @@ class Settings(BaseSettings, ConfigModel): @property def CACHE_CONF(self): + """ + { + "torrents": "缓存种子数量", + "refresh": "订阅刷新处理数量", + "tmdb": "TMDB请求缓存数量", + "douban": "豆瓣请求缓存数量", + "fanart": "Fanart请求缓存数量", + "meta": "元数据缓存过期时间(秒)" + } + """ if self.BIG_MEMORY_MODE: return { + "torrents": 200, + "refresh": 100, "tmdb": 1024, - "refresh": 50, - "torrents": 100, "douban": 512, "fanart": 512, - "meta": (self.META_CACHE_EXPIRE or 168) * 3600 + "meta": (self.META_CACHE_EXPIRE or 24) * 3600 } return { + "torrents": 100, + "refresh": 50, "tmdb": 256, - "refresh": 30, - "torrents": 50, "douban": 256, "fanart": 128, - "meta": (self.META_CACHE_EXPIRE or 72) * 3600 + "meta": (self.META_CACHE_EXPIRE or 2) * 3600 } @property diff --git a/app/modules/douban/apiv2.py b/app/modules/douban/apiv2.py index 4ffe8595..ec884b2d 100644 --- a/app/modules/douban/apiv2.py +++ b/app/modules/douban/apiv2.py @@ -3,11 +3,11 @@ import base64 import hashlib import hmac from datetime import datetime -from functools import lru_cache from random import choice from urllib import parse import requests +from cachetools import TTLCache, cached from app.core.config import settings from app.utils.http import RequestUtils @@ -160,12 +160,12 @@ class DoubanApi(metaclass=Singleton): self._session = requests.Session() @classmethod - def __sign(cls, url: str, ts: int, method='GET') -> str: + def __sign(cls, url: str, ts: str, method='GET') -> str: """ 签名 """ url_path = parse.urlparse(url).path - raw_sign = '&'.join([method.upper(), parse.quote(url_path, safe=''), str(ts)]) + raw_sign = '&'.join([method.upper(), parse.quote(url_path, safe=''), ts]) return base64.b64encode( hmac.new( cls._api_secret_key.encode(), @@ -174,7 +174,7 @@ class DoubanApi(metaclass=Singleton): ).digest() ).decode() - @lru_cache(maxsize=settings.CACHE_CONF.get('douban')) + @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"])) def __invoke(self, url: str, **kwargs) -> dict: """ GET请求 @@ -203,7 +203,7 @@ class DoubanApi(metaclass=Singleton): return resp.json() return resp.json() if resp else {} - @lru_cache(maxsize=settings.CACHE_CONF.get('douban')) + @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"])) def __post(self, url: str, **kwargs) -> dict: """ POST请求 diff --git a/app/modules/douban/douban_cache.py b/app/modules/douban/douban_cache.py index a5fafbd5..8d3efa9b 100644 --- a/app/modules/douban/douban_cache.py +++ b/app/modules/douban/douban_cache.py @@ -16,7 +16,7 @@ from app.schemas.types import MediaType lock = RLock() CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp" -EXPIRE_TIMESTAMP = settings.CACHE_CONF.get('meta') +EXPIRE_TIMESTAMP = settings.CACHE_CONF["meta"] class DoubanCache(metaclass=Singleton): @@ -77,7 +77,7 @@ class DoubanCache(metaclass=Singleton): @return: 被删除的缓存内容 """ with lock: - return self._meta_data.pop(key, None) + return self._meta_data.pop(key, {}) def delete_by_doubanid(self, doubanid: str) -> None: """ diff --git a/app/modules/fanart/__init__.py b/app/modules/fanart/__init__.py index 57db0e43..4bc7a74d 100644 --- a/app/modules/fanart/__init__.py +++ b/app/modules/fanart/__init__.py @@ -1,12 +1,13 @@ import re -from functools import lru_cache from typing import Optional, Tuple, Union +from cachetools import TTLCache, cached + from app.core.context import MediaInfo, settings from app.log import logger from app.modules import _ModuleBase -from app.utils.http import RequestUtils from app.schemas.types import MediaType, ModuleType +from app.utils.http import RequestUtils class FanartModule(_ModuleBase): @@ -404,7 +405,7 @@ class FanartModule(_ModuleBase): return result @classmethod - @lru_cache(maxsize=settings.CACHE_CONF.get('fanart')) + @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["fanart"], ttl=settings.CACHE_CONF["meta"])) def __request_fanart(cls, media_type: MediaType, queryid: Union[str, int]) -> Optional[dict]: if media_type == MediaType.MOVIE: image_url = cls._movie_url % queryid diff --git a/app/modules/themoviedb/tmdb_cache.py b/app/modules/themoviedb/tmdb_cache.py index ba414e75..7f3908f1 100644 --- a/app/modules/themoviedb/tmdb_cache.py +++ b/app/modules/themoviedb/tmdb_cache.py @@ -15,7 +15,7 @@ from app.schemas.types import MediaType lock = RLock() CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp" -EXPIRE_TIMESTAMP = settings.CACHE_CONF.get('meta') +EXPIRE_TIMESTAMP = settings.CACHE_CONF["meta"] class TmdbCache(metaclass=Singleton): @@ -75,7 +75,7 @@ class TmdbCache(metaclass=Singleton): @return: 被删除的缓存内容 """ with lock: - return self._meta_data.pop(key, None) + return self._meta_data.pop(key, {}) def delete_by_tmdbid(self, tmdbid: int) -> None: """ @@ -138,14 +138,14 @@ class TmdbCache(metaclass=Singleton): if cache_year: cache_year = cache_year[:4] self._meta_data[self.__get_key(meta)] = { - "id": info.get("id"), - "type": info.get("media_type"), - "year": cache_year, - "title": cache_title, - "poster_path": info.get("poster_path"), - "backdrop_path": info.get("backdrop_path"), - CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP - } + "id": info.get("id"), + "type": info.get("media_type"), + "year": cache_year, + "title": cache_title, + "poster_path": info.get("poster_path"), + "backdrop_path": info.get("backdrop_path"), + CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP + } elif info is not None: # None时不缓存,此时代表网络错误,允许重复请求 self._meta_data[self.__get_key(meta)] = {'id': 0} @@ -164,7 +164,7 @@ class TmdbCache(metaclass=Singleton): return with open(self._meta_path, 'wb') as f: - pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) + pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) # type: ignore def _random_sample(self, new_meta_data: dict) -> bool: """ diff --git a/app/modules/themoviedb/tmdbapi.py b/app/modules/themoviedb/tmdbapi.py index 02cf1651..8201d1a1 100644 --- a/app/modules/themoviedb/tmdbapi.py +++ b/app/modules/themoviedb/tmdbapi.py @@ -1,9 +1,9 @@ import traceback -from functools import lru_cache from typing import Optional, List from urllib.parse import quote import zhconv +from cachetools import TTLCache, cached from lxml import etree from app.core.config import settings @@ -27,8 +27,6 @@ class TmdbApi: self.tmdb.domain = settings.TMDB_API_DOMAIN # 开启缓存 self.tmdb.cache = True - # 缓存大小 - self.tmdb.REQUEST_CACHE_MAXSIZE = settings.CACHE_CONF.get('tmdb') # APIKEY self.tmdb.api_key = settings.TMDB_API_KEY # 语种 @@ -466,7 +464,7 @@ class TmdbApi: return ret_info - @lru_cache(maxsize=settings.CACHE_CONF.get('tmdb')) + @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"])) def match_web(self, name: str, mtype: MediaType) -> Optional[dict]: """ 搜索TMDB网站,直接抓取结果,结果只有一条时才返回 diff --git a/app/modules/themoviedb/tmdbv3api/tmdb.py b/app/modules/themoviedb/tmdbv3api/tmdb.py index 2117c1be..e6fb1bb0 100644 --- a/app/modules/themoviedb/tmdbv3api/tmdb.py +++ b/app/modules/themoviedb/tmdbv3api/tmdb.py @@ -4,11 +4,12 @@ import logging import os import time from datetime import datetime -from functools import lru_cache import requests import requests.exceptions +from cachetools import TTLCache, cached +from app.core.config import settings from app.utils.http import RequestUtils from .exceptions import TMDbException @@ -24,7 +25,6 @@ class TMDb(object): TMDB_CACHE_ENABLED = "TMDB_CACHE_ENABLED" TMDB_PROXIES = "TMDB_PROXIES" TMDB_DOMAIN = "TMDB_DOMAIN" - REQUEST_CACHE_MAXSIZE = None _req = None _session = None @@ -137,7 +137,7 @@ class TMDb(object): def cache(self, cache): os.environ[self.TMDB_CACHE_ENABLED] = str(cache) - @lru_cache(maxsize=REQUEST_CACHE_MAXSIZE) + @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"])) def cached_request(self, method, url, data, json, _ts=datetime.strftime(datetime.now(), '%Y%m%d')): """ diff --git a/app/scheduler.py b/app/scheduler.py index c7fbe559..cb6acdde 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -330,7 +330,7 @@ class Scheduler(metaclass=Singleton): "interval", id="clear_cache", name="缓存清理", - hours=settings.CACHE_CONF.get("meta") / 3600, + hours=settings.CACHE_CONF["meta"] / 3600, kwargs={ 'job_id': 'clear_cache' }