add siteuserdata

app/api/endpoints/subscribe.py

@@ -113,8 +113,6 @@ def update_subscribe(
     subscribe = Subscribe.get(db, subscribe_in.id)
     if not subscribe:
         return schemas.Response(success=False, message="订阅不存在")
-    if subscribe_in.sites is not None:
-        subscribe_in.sites = json.dumps(subscribe_in.sites)
     # 避免更新缺失集数
     subscribe_dict = subscribe_in.dict()
     if not subscribe_in.lack_episode:

app/chain/site.py

@@ -59,13 +59,26 @@ class SiteChain(ChainBase):
             "yemapt.org": self.__yema_test,
         }
 
-    def site_userdata(self, site: CommentedMap) -> Optional[SiteUserData]:
+    def refresh_userdata(self, site: CommentedMap = None) -> Optional[SiteUserData]:
         """
-        获取站点的所有用户数据
+        刷新站点的用户数据
         :param site:  站点
         :return: 用户数据
         """
-        return self.run_module("site_userdata", site=site)
+        userdata = self.run_module("refresh_userdata", site=site)
+        if userdata:
+            self.siteoper.update_userdata(domain=StringUtils.get_url_domain(site.get("domain")),
+                                          payload=userdata)
+        return userdata
+
+    def refresh_userdatas(self) -> None:
+        """
+        刷新所有站点的用户数据
+        """
+        sites = self.siteshelper.get_indexers()
+        for site in sites:
+            if site.get("is_active"):
+                self.refresh_userdata(site)
 
     def is_special_site(self, domain: str) -> bool:
         """

app/chain/subscribe.py

@@ -487,7 +487,7 @@ class SubscribeChain(ChainBase):
             # 如果交集与原始订阅不一致，更新数据库
             if set(intersection_sites) != set(user_sites):
                 self.subscribeoper.update(subscribe.id, {
-                    "sites": json.dumps(intersection_sites)
+                    "sites": intersection_sites
                 })
             # 如果交集为空，返回默认站点
             return intersection_sites if intersection_sites else default_sites
@@ -857,7 +857,7 @@ class SubscribeChain(ChainBase):
             note = list(set(note).union(set(episodes)))
             # 更新订阅
             self.subscribeoper.update(subscribe.id, {
-                "note": json.dumps(note)
+                "note": note
             })
 
     @staticmethod
@@ -1140,7 +1140,7 @@ class SubscribeChain(ChainBase):
                 continue
             sites.remove(site_id)
             self.subscribeoper.update(subscribe.id, {
-                "sites": json.dumps(sites)
+                "sites": sites
             })
 
     @staticmethod

app/core/config.py

@@ -101,6 +101,8 @@ class Settings(BaseSettings):
     SUBSCRIBE_SEARCH: bool = False
     # 搜索多个名称
     SEARCH_MULTIPLE_NAME: bool = False
+    # 站点数据刷新间隔（小时）
+    SITEDATA_REFRESH_INTERVAL: int = 6
     # 种子标签
     TORRENT_TAG: str = "MOVIEPILOT"
     # 下载站点字幕

app/db/__init__.py

@@ -1,3 +1,4 @@
+import json
 from typing import Any, Self, List
 from typing import Tuple, Optional, Generator
 
@@ -7,6 +8,7 @@ from sqlalchemy.orm import declared_attr
 from sqlalchemy.orm import sessionmaker, Session, scoped_session, as_declarative
 
 from app.core.config import settings
+from app.utils.object import ObjectUtils
 
 # 数据库引擎
 Engine = create_engine(f"sqlite:///{settings.CONFIG_PATH}/user.db",
@@ -156,6 +158,8 @@ class Base:
     def update(self, db: Session, payload: dict):
         payload = {k: v for k, v in payload.items() if v is not None}
         for key, value in payload.items():
+            if ObjectUtils.is_obj(value):
+                value = json.dumps(value)
             setattr(self, key, value)
         if inspect(self).detached:
             db.add(self)

app/db/message_oper.py

@@ -1,4 +1,3 @@
-import json
 import time
 from typing import Optional, Union
 
@@ -53,7 +52,7 @@ class MessageOper(DbOper):
             "userid": userid,
             "action": action,
             "reg_time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
-            "note": json.dumps(note) if note else ''
+            "note": note
         })
         Message(**kwargs).create(self._db)
 

app/db/models/siteuserdata.py

@@ -0,0 +1,61 @@
+from datetime import datetime
+
+from sqlalchemy import Column, Integer, String, Sequence
+from sqlalchemy.orm import Session
+
+from app.db import db_query, Base
+
+
+class SiteUserData(Base):
+    """
+    站点数据表
+    """
+    id = Column(Integer, Sequence('id'), primary_key=True, index=True)
+    # 站点域名
+    domain = Column(String, index=True)
+    # 用户名
+    username = Column(String)
+    # 用户ID
+    userid = Column(Integer)
+    # 用户等级
+    user_level = Column(String)
+    # 加入时间
+    join_at = Column(String)
+    # 积分
+    bonus = Column(Integer, default=0)
+    # 上传量
+    upload = Column(Integer, default=0)
+    # 下载量
+    download = Column(Integer, default=0)
+    # 分享率
+    ratio = Column(Integer, default=0)
+    # 做种数
+    seeding = Column(Integer, default=0)
+    # 下载数
+    leeching = Column(Integer, default=0)
+    # 做种体积
+    seeding_size = Column(Integer, default=0)
+    # 下载体积
+    leeching_size = Column(Integer, default=0)
+    # 做种人数, 种子大小 JSON
+    seeding_info = Column(String)
+    # 未读消息
+    message_unread = Column(Integer, default=0)
+    # 未读消息内容 JSON
+    message_unread_contents = Column(String)
+    # 错误信息
+    err_msg = Column(String)
+    # 更新日期
+    updated_day = Column(String, index=True, default=datetime.now().strftime('%Y-%m-%d'))
+    # 更新时间
+    updated_time = Column(String, default=datetime.now().strftime('%H:%M:%S'))
+
+    @staticmethod
+    @db_query
+    def get_by_domain(db: Session, domain: str):
+        return db.query(SiteUserData).filter(SiteUserData.domain == domain).all()
+
+    @staticmethod
+    @db_query
+    def get_by_date(db: Session, date: str):
+        return db.query(SiteUserData).filter(SiteUserData.updated_day == date).all()

app/db/plugindata_oper.py

@@ -18,8 +18,6 @@ class PluginDataOper(DbOper):
         :param key: 数据key
         :param value: 数据值
         """
-        if ObjectUtils.is_obj(value):
-            value = json.dumps(value)
         plugin = PluginData.get_plugin_data_by_key(self._db, plugin_id, key)
         if plugin:
             plugin.update(self._db, {

app/db/site_oper.py

@@ -1,7 +1,9 @@
+from datetime import datetime
 from typing import Tuple, List
 
 from app.db import DbOper
 from app.db.models.site import Site
+from app.db.models.siteuserdata import SiteUserData
 
 
 class SiteOper(DbOper):
@@ -98,3 +100,30 @@ class SiteOper(DbOper):
             "rss": rss
         })
         return True, "更新站点RSS地址成功"
+
+    def update_userdata(self, domain: str, payload: dict) -> Tuple[bool, str]:
+        """
+        更新站点用户数据
+        """
+        site = Site.get_by_domain(self._db, domain)
+        if not site:
+            return False, "站点不存在"
+        payload.update({
+            "domain": domain,
+            "updated_day": datetime.now().strftime('%Y-%m-%d'),
+            "updated_time": datetime.now().strftime('%H:%M:%S')
+        })
+        SiteUserData.update(self._db, payload)
+        return True, "更新站点用户数据成功"
+
+    def get_userdata_by_domain(self, domain: str) -> List[SiteUserData]:
+        """
+        获取站点用户数据
+        """
+        return SiteUserData.get_by_domain(self._db, domain)
+
+    def get_userdata_by_date(self, date: str) -> List[SiteUserData]:
+        """
+        获取站点用户数据
+        """
+        return SiteUserData.get_by_date(self._db, date)

app/db/sitestatistic_oper.py

@@ -30,7 +30,7 @@ class SiteStatisticOper(DbOper):
                 "seconds": avg_seconds or sta.seconds,
                 "lst_state": 0,
                 "lst_mod_date": lst_date,
-                "note": json.dumps(note) if note else sta.note
+                "note": note or sta.note
             })
         else:
             note = {}

app/db/subscribe_oper.py

@@ -1,4 +1,3 @@
-import json
 import time
 from typing import Tuple, List
 
@@ -21,9 +20,6 @@ class SubscribeOper(DbOper):
                                      doubanid=mediainfo.douban_id,
                                      season=kwargs.get('season'))
         if not subscribe:
-            if kwargs.get("sites") and not isinstance(kwargs.get("sites"), str):
-                kwargs["sites"] = json.dumps(kwargs.get("sites"))
-
             subscribe = Subscribe(name=mediainfo.title,
                                   year=mediainfo.year,
                                   type=mediainfo.type.value,

app/modules/indexer/__init__.py

@@ -207,9 +207,9 @@ class IndexerModule(_ModuleBase):
         """
         return self.search_torrents(site=site)
 
-    def site_userdata(self, site: CommentedMap) -> Optional[SiteUserData]:
+    def refresh_userdata(self, site: CommentedMap) -> Optional[SiteUserData]:
         """
-        获取站点的所有用户数据
+        刷新站点的用户数据
         :param site:  站点
         :return: 用户数据
         """
@@ -240,6 +240,7 @@ class IndexerModule(_ModuleBase):
         site_obj.parse()
         logger.debug(f"站点 {site.get('name')} 解析完成")
         return SiteUserData(
+            domain=StringUtils.get_url_domain(site.get("url")),
             userid=site_obj.userid,
             username=site_obj.username,
             user_level=site_obj.user_level,

app/scheduler.py

@@ -166,6 +166,11 @@ class Scheduler(metaclass=Singleton):
                 "name": "壁纸缓存",
                 "func": TmdbChain().get_trending_wallpapers,
                 "running": False,
+            },
+            "sitedata_refresh": {
+                "name": "站点数据刷新",
+                "func": SiteChain().refresh_userdatas,
+                "running": False,
             }
         }
 
@@ -343,6 +348,18 @@ class Scheduler(metaclass=Singleton):
             }
         )
 
+        # 站点数据刷新，每隔30分钟
+        self._scheduler.add_job(
+            self.start,
+            "interval",
+            id="sitedata_refresh",
+            name="站点数据刷新",
+            minutes=settings.SITEDATA_REFRESH_INTERVAL * 60,
+            kwargs={
+                'job_id': 'sitedata_refresh'
+            }
+        )
+
         # 注册插件公共服务
         for pid in PluginManager().get_running_plugin_ids():
             self.update_plugin_job(pid)

app/schemas/site.py

@@ -70,6 +70,8 @@ class SiteStatistic(BaseModel):
 
 
 class SiteUserData(BaseModel):
+    # 站点域名
+    domain: Optional[str]
     # 用户名
     username: Optional[str]
     # 用户ID
@@ -80,32 +82,20 @@ class SiteUserData(BaseModel):
     join_at: Optional[str]
     # 积分
     bonus: Optional[float] = 0.0
-    # 上传
+    # 上传量
     upload: Optional[int] = 0
-    # 下载
+    # 下载量
     download: Optional[int] = 0
     # 分享率
     ratio: Optional[float] = 0
-    # 做种
+    # 做种数
     seeding: Optional[int] = 0
-    # 下载
+    # 下载数
     leeching: Optional[int] = 0
-    # 做种大小
+    # 做种体积
     seeding_size: Optional[int] = 0
-    # 下载大小
+    # 下载体积
     leeching_size: Optional[int] = 0
-    # 上传量
-    uploaded: Optional[int] = 0
-    # 完成量
-    completed: Optional[int] = 0
-    # 未完成量
-    incomplete: Optional[int] = 0
-    # 上传量
-    uploaded_size: Optional[int] = 0
-    # 完成量
-    completed_size: Optional[int] = 0
-    # 未完成量
-    incomplete_size: Optional[int] = 0
     # 做种人数, 种子大小
     seeding_info: Optional[list] = []
     # 未读消息