diff --git a/app/api/endpoints/subscribe.py b/app/api/endpoints/subscribe.py index beaf3b94..fc9b59b0 100644 --- a/app/api/endpoints/subscribe.py +++ b/app/api/endpoints/subscribe.py @@ -113,8 +113,6 @@ def update_subscribe( subscribe = Subscribe.get(db, subscribe_in.id) if not subscribe: return schemas.Response(success=False, message="订阅不存在") - if subscribe_in.sites is not None: - subscribe_in.sites = json.dumps(subscribe_in.sites) # 避免更新缺失集数 subscribe_dict = subscribe_in.dict() if not subscribe_in.lack_episode: diff --git a/app/chain/site.py b/app/chain/site.py index 353e64ba..0a0a6114 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -59,13 +59,26 @@ class SiteChain(ChainBase): "yemapt.org": self.__yema_test, } - def site_userdata(self, site: CommentedMap) -> Optional[SiteUserData]: + def refresh_userdata(self, site: CommentedMap = None) -> Optional[SiteUserData]: """ - 获取站点的所有用户数据 + 刷新站点的用户数据 :param site: 站点 :return: 用户数据 """ - return self.run_module("site_userdata", site=site) + userdata = self.run_module("refresh_userdata", site=site) + if userdata: + self.siteoper.update_userdata(domain=StringUtils.get_url_domain(site.get("domain")), + payload=userdata) + return userdata + + def refresh_userdatas(self) -> None: + """ + 刷新所有站点的用户数据 + """ + sites = self.siteshelper.get_indexers() + for site in sites: + if site.get("is_active"): + self.refresh_userdata(site) def is_special_site(self, domain: str) -> bool: """ diff --git a/app/chain/subscribe.py b/app/chain/subscribe.py index 4f151a15..c5bb5955 100644 --- a/app/chain/subscribe.py +++ b/app/chain/subscribe.py @@ -487,7 +487,7 @@ class SubscribeChain(ChainBase): # 如果交集与原始订阅不一致,更新数据库 if set(intersection_sites) != set(user_sites): self.subscribeoper.update(subscribe.id, { - "sites": json.dumps(intersection_sites) + "sites": intersection_sites }) # 如果交集为空,返回默认站点 return intersection_sites if intersection_sites else default_sites @@ -857,7 +857,7 @@ class SubscribeChain(ChainBase): note = list(set(note).union(set(episodes))) # 更新订阅 self.subscribeoper.update(subscribe.id, { - "note": json.dumps(note) + "note": note }) @staticmethod @@ -1140,7 +1140,7 @@ class SubscribeChain(ChainBase): continue sites.remove(site_id) self.subscribeoper.update(subscribe.id, { - "sites": json.dumps(sites) + "sites": sites }) @staticmethod diff --git a/app/core/config.py b/app/core/config.py index d1cd966e..c8eb786f 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -101,6 +101,8 @@ class Settings(BaseSettings): SUBSCRIBE_SEARCH: bool = False # 搜索多个名称 SEARCH_MULTIPLE_NAME: bool = False + # 站点数据刷新间隔(小时) + SITEDATA_REFRESH_INTERVAL: int = 6 # 种子标签 TORRENT_TAG: str = "MOVIEPILOT" # 下载站点字幕 diff --git a/app/db/__init__.py b/app/db/__init__.py index 909b8d2b..3da3bd9b 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -1,3 +1,4 @@ +import json from typing import Any, Self, List from typing import Tuple, Optional, Generator @@ -7,6 +8,7 @@ from sqlalchemy.orm import declared_attr from sqlalchemy.orm import sessionmaker, Session, scoped_session, as_declarative from app.core.config import settings +from app.utils.object import ObjectUtils # 数据库引擎 Engine = create_engine(f"sqlite:///{settings.CONFIG_PATH}/user.db", @@ -156,6 +158,8 @@ class Base: def update(self, db: Session, payload: dict): payload = {k: v for k, v in payload.items() if v is not None} for key, value in payload.items(): + if ObjectUtils.is_obj(value): + value = json.dumps(value) setattr(self, key, value) if inspect(self).detached: db.add(self) diff --git a/app/db/message_oper.py b/app/db/message_oper.py index a1a02058..0be459f4 100644 --- a/app/db/message_oper.py +++ b/app/db/message_oper.py @@ -1,4 +1,3 @@ -import json import time from typing import Optional, Union @@ -53,7 +52,7 @@ class MessageOper(DbOper): "userid": userid, "action": action, "reg_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), - "note": json.dumps(note) if note else '' + "note": note }) Message(**kwargs).create(self._db) diff --git a/app/db/models/siteuserdata.py b/app/db/models/siteuserdata.py new file mode 100644 index 00000000..3873b228 --- /dev/null +++ b/app/db/models/siteuserdata.py @@ -0,0 +1,61 @@ +from datetime import datetime + +from sqlalchemy import Column, Integer, String, Sequence +from sqlalchemy.orm import Session + +from app.db import db_query, Base + + +class SiteUserData(Base): + """ + 站点数据表 + """ + id = Column(Integer, Sequence('id'), primary_key=True, index=True) + # 站点域名 + domain = Column(String, index=True) + # 用户名 + username = Column(String) + # 用户ID + userid = Column(Integer) + # 用户等级 + user_level = Column(String) + # 加入时间 + join_at = Column(String) + # 积分 + bonus = Column(Integer, default=0) + # 上传量 + upload = Column(Integer, default=0) + # 下载量 + download = Column(Integer, default=0) + # 分享率 + ratio = Column(Integer, default=0) + # 做种数 + seeding = Column(Integer, default=0) + # 下载数 + leeching = Column(Integer, default=0) + # 做种体积 + seeding_size = Column(Integer, default=0) + # 下载体积 + leeching_size = Column(Integer, default=0) + # 做种人数, 种子大小 JSON + seeding_info = Column(String) + # 未读消息 + message_unread = Column(Integer, default=0) + # 未读消息内容 JSON + message_unread_contents = Column(String) + # 错误信息 + err_msg = Column(String) + # 更新日期 + updated_day = Column(String, index=True, default=datetime.now().strftime('%Y-%m-%d')) + # 更新时间 + updated_time = Column(String, default=datetime.now().strftime('%H:%M:%S')) + + @staticmethod + @db_query + def get_by_domain(db: Session, domain: str): + return db.query(SiteUserData).filter(SiteUserData.domain == domain).all() + + @staticmethod + @db_query + def get_by_date(db: Session, date: str): + return db.query(SiteUserData).filter(SiteUserData.updated_day == date).all() diff --git a/app/db/plugindata_oper.py b/app/db/plugindata_oper.py index 4469cfa0..579d3550 100644 --- a/app/db/plugindata_oper.py +++ b/app/db/plugindata_oper.py @@ -18,8 +18,6 @@ class PluginDataOper(DbOper): :param key: 数据key :param value: 数据值 """ - if ObjectUtils.is_obj(value): - value = json.dumps(value) plugin = PluginData.get_plugin_data_by_key(self._db, plugin_id, key) if plugin: plugin.update(self._db, { diff --git a/app/db/site_oper.py b/app/db/site_oper.py index ce153277..7ec11edc 100644 --- a/app/db/site_oper.py +++ b/app/db/site_oper.py @@ -1,7 +1,9 @@ +from datetime import datetime from typing import Tuple, List from app.db import DbOper from app.db.models.site import Site +from app.db.models.siteuserdata import SiteUserData class SiteOper(DbOper): @@ -98,3 +100,30 @@ class SiteOper(DbOper): "rss": rss }) return True, "更新站点RSS地址成功" + + def update_userdata(self, domain: str, payload: dict) -> Tuple[bool, str]: + """ + 更新站点用户数据 + """ + site = Site.get_by_domain(self._db, domain) + if not site: + return False, "站点不存在" + payload.update({ + "domain": domain, + "updated_day": datetime.now().strftime('%Y-%m-%d'), + "updated_time": datetime.now().strftime('%H:%M:%S') + }) + SiteUserData.update(self._db, payload) + return True, "更新站点用户数据成功" + + def get_userdata_by_domain(self, domain: str) -> List[SiteUserData]: + """ + 获取站点用户数据 + """ + return SiteUserData.get_by_domain(self._db, domain) + + def get_userdata_by_date(self, date: str) -> List[SiteUserData]: + """ + 获取站点用户数据 + """ + return SiteUserData.get_by_date(self._db, date) diff --git a/app/db/sitestatistic_oper.py b/app/db/sitestatistic_oper.py index 348fbd1f..18fa6d7d 100644 --- a/app/db/sitestatistic_oper.py +++ b/app/db/sitestatistic_oper.py @@ -30,7 +30,7 @@ class SiteStatisticOper(DbOper): "seconds": avg_seconds or sta.seconds, "lst_state": 0, "lst_mod_date": lst_date, - "note": json.dumps(note) if note else sta.note + "note": note or sta.note }) else: note = {} diff --git a/app/db/subscribe_oper.py b/app/db/subscribe_oper.py index e7895982..e05827c0 100644 --- a/app/db/subscribe_oper.py +++ b/app/db/subscribe_oper.py @@ -1,4 +1,3 @@ -import json import time from typing import Tuple, List @@ -21,9 +20,6 @@ class SubscribeOper(DbOper): doubanid=mediainfo.douban_id, season=kwargs.get('season')) if not subscribe: - if kwargs.get("sites") and not isinstance(kwargs.get("sites"), str): - kwargs["sites"] = json.dumps(kwargs.get("sites")) - subscribe = Subscribe(name=mediainfo.title, year=mediainfo.year, type=mediainfo.type.value, diff --git a/app/modules/indexer/__init__.py b/app/modules/indexer/__init__.py index c102fad9..ee2b470c 100644 --- a/app/modules/indexer/__init__.py +++ b/app/modules/indexer/__init__.py @@ -207,9 +207,9 @@ class IndexerModule(_ModuleBase): """ return self.search_torrents(site=site) - def site_userdata(self, site: CommentedMap) -> Optional[SiteUserData]: + def refresh_userdata(self, site: CommentedMap) -> Optional[SiteUserData]: """ - 获取站点的所有用户数据 + 刷新站点的用户数据 :param site: 站点 :return: 用户数据 """ @@ -240,6 +240,7 @@ class IndexerModule(_ModuleBase): site_obj.parse() logger.debug(f"站点 {site.get('name')} 解析完成") return SiteUserData( + domain=StringUtils.get_url_domain(site.get("url")), userid=site_obj.userid, username=site_obj.username, user_level=site_obj.user_level, diff --git a/app/scheduler.py b/app/scheduler.py index 81679d1d..21b39dca 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -166,6 +166,11 @@ class Scheduler(metaclass=Singleton): "name": "壁纸缓存", "func": TmdbChain().get_trending_wallpapers, "running": False, + }, + "sitedata_refresh": { + "name": "站点数据刷新", + "func": SiteChain().refresh_userdatas, + "running": False, } } @@ -343,6 +348,18 @@ class Scheduler(metaclass=Singleton): } ) + # 站点数据刷新,每隔30分钟 + self._scheduler.add_job( + self.start, + "interval", + id="sitedata_refresh", + name="站点数据刷新", + minutes=settings.SITEDATA_REFRESH_INTERVAL * 60, + kwargs={ + 'job_id': 'sitedata_refresh' + } + ) + # 注册插件公共服务 for pid in PluginManager().get_running_plugin_ids(): self.update_plugin_job(pid) diff --git a/app/schemas/site.py b/app/schemas/site.py index 2828651e..1fe73f38 100644 --- a/app/schemas/site.py +++ b/app/schemas/site.py @@ -70,6 +70,8 @@ class SiteStatistic(BaseModel): class SiteUserData(BaseModel): + # 站点域名 + domain: Optional[str] # 用户名 username: Optional[str] # 用户ID @@ -80,32 +82,20 @@ class SiteUserData(BaseModel): join_at: Optional[str] # 积分 bonus: Optional[float] = 0.0 - # 上传 + # 上传量 upload: Optional[int] = 0 - # 下载 + # 下载量 download: Optional[int] = 0 # 分享率 ratio: Optional[float] = 0 - # 做种 + # 做种数 seeding: Optional[int] = 0 - # 下载 + # 下载数 leeching: Optional[int] = 0 - # 做种大小 + # 做种体积 seeding_size: Optional[int] = 0 - # 下载大小 + # 下载体积 leeching_size: Optional[int] = 0 - # 上传量 - uploaded: Optional[int] = 0 - # 完成量 - completed: Optional[int] = 0 - # 未完成量 - incomplete: Optional[int] = 0 - # 上传量 - uploaded_size: Optional[int] = 0 - # 完成量 - completed_size: Optional[int] = 0 - # 未完成量 - incomplete_size: Optional[int] = 0 # 做种人数, 种子大小 seeding_info: Optional[list] = [] # 未读消息