From 46fb52fff92d1de24233c9c984057092f24b6582 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 26 Sep 2024 14:13:29 +0800 Subject: [PATCH] merge db oper --- app/chain/site.py | 18 +++----- app/chain/subscribe.py | 4 +- app/db/site_oper.py | 82 +++++++++++++++++++++++++++++++++ app/db/siteicon_oper.py | 37 --------------- app/db/sitestatistic_oper.py | 70 ---------------------------- app/db/subscribe_oper.py | 16 +++++++ app/db/subscribehistory_oper.py | 30 ------------ app/modules/indexer/__init__.py | 6 +-- 8 files changed, 109 insertions(+), 154 deletions(-) delete mode 100644 app/db/siteicon_oper.py delete mode 100644 app/db/sitestatistic_oper.py delete mode 100644 app/db/subscribehistory_oper.py diff --git a/app/chain/site.py b/app/chain/site.py index f4be1f12..748b4330 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -13,9 +13,7 @@ from app.core.config import settings from app.core.event import eventmanager, Event, EventManager from app.db.models.site import Site from app.db.site_oper import SiteOper -from app.db.siteicon_oper import SiteIconOper from app.db.systemconfig_oper import SystemConfigOper -from app.db.sitestatistic_oper import SiteStatisticOper from app.helper.browser import PlaywrightHelper from app.helper.cloudflare import under_challenge from app.helper.cookie import CookieHelper @@ -39,14 +37,12 @@ class SiteChain(ChainBase): def __init__(self): super().__init__() self.siteoper = SiteOper() - self.siteiconoper = SiteIconOper() self.siteshelper = SitesHelper() self.rsshelper = RssHelper() self.cookiehelper = CookieHelper() self.message = MessageHelper() self.cookiecloud = CookieCloudHelper() self.systemconfig = SystemConfigOper() - self.sitestatistic = SiteStatisticOper() # 特殊站点登录验证 self.special_site_test = { @@ -362,17 +358,17 @@ class SiteChain(ChainBase): logger.warn(f"站点 {domain} 索引器不存在！") return # 查询站点图标 - site_icon = self.siteiconoper.get_by_domain(domain) + site_icon = self.siteoper.get_icon_by_domain(domain) if not site_icon or not site_icon.base64: logger.info(f"开始缓存站点 {indexer.get('name')} 图标 ...") icon_url, icon_base64 = self.__parse_favicon(url=indexer.get("domain"), cookie=cookie, ua=settings.USER_AGENT) if icon_url: - self.siteiconoper.update_icon(name=indexer.get("name"), - domain=domain, - icon_url=icon_url, - icon_base64=icon_base64) + self.siteoper.update_icon(name=indexer.get("name"), + domain=domain, + icon_url=icon_url, + icon_base64=icon_base64) logger.info(f"缓存站点 {indexer.get('name')} 图标成功") else: logger.warn(f"缓存站点 {indexer.get('name')} 图标失败") @@ -443,9 +439,9 @@ class SiteChain(ChainBase): # 统计 seconds = (datetime.now() - start_time).seconds if state: - self.sitestatistic.success(domain=domain, seconds=seconds) + self.siteoper.success(domain=domain, seconds=seconds) else: - self.sitestatistic.fail(domain) + self.siteoper.fail(domain) return state, message except Exception as e: return False, f"{str(e)}！" diff --git a/app/chain/subscribe.py b/app/chain/subscribe.py index dfd9a335..f2578bd6 100644 --- a/app/chain/subscribe.py +++ b/app/chain/subscribe.py @@ -20,7 +20,6 @@ from app.db.downloadhistory_oper import DownloadHistoryOper from app.db.models.subscribe import Subscribe from app.db.site_oper import SiteOper from app.db.subscribe_oper import SubscribeOper -from app.db.subscribehistory_oper import SubscribeHistoryOper from app.db.systemconfig_oper import SystemConfigOper from app.helper.message import MessageHelper from app.helper.subscribe import SubscribeHelper @@ -41,7 +40,6 @@ class SubscribeChain(ChainBase): self.downloadhis = DownloadHistoryOper() self.searchchain = SearchChain() self.subscribeoper = SubscribeOper() - self.subscribehistoryoper = SubscribeHistoryOper() self.subscribehelper = SubscribeHelper() self.torrentschain = TorrentsChain() self.mediachain = MediaChain() @@ -898,7 +896,7 @@ class SubscribeChain(ChainBase): msgstr = "洗版" logger.info(f'{mediainfo.title_year} 完成{msgstr}') # 新增订阅历史 - self.subscribehistoryoper.add(**subscribe.to_dict()) + self.subscribeoper.add_history(**subscribe.to_dict()) # 删除订阅 self.subscribeoper.delete(subscribe.id) # 发送通知 diff --git a/app/db/site_oper.py b/app/db/site_oper.py index beb1984f..7956e65d 100644 --- a/app/db/site_oper.py +++ b/app/db/site_oper.py @@ -3,7 +3,9 @@ from datetime import datetime from typing import Tuple, List from app.db import DbOper +from app.db.models import SiteIcon from app.db.models.site import Site +from app.db.models.sitestatistic import SiteStatistic from app.db.models.siteuserdata import SiteUserData from app.utils.object import ObjectUtils @@ -139,3 +141,83 @@ class SiteOper(DbOper): 获取站点用户数据 """ return SiteUserData.get_by_date(self._db, date) + + def get_icon_by_domain(self, domain: str) -> SiteIcon: + """ + 按域名获取站点图标 + """ + return SiteIcon.get_by_domain(self._db, domain) + + def update_icon(self, name: str, domain: str, icon_url: str, icon_base64: str) -> bool: + """ + 更新站点图标 + """ + icon_base64 = f"data:image/ico;base64,{icon_base64}" if icon_base64 else "" + siteicon = self.get_by_domain(domain) + if not siteicon: + SiteIcon(name=name, domain=domain, url=icon_url, base64=icon_base64).create(self._db) + elif icon_base64: + siteicon.update(self._db, { + "url": icon_url, + "base64": icon_base64 + }) + return True + + def success(self, domain: str, seconds: int = None): + """ + 站点访问成功 + """ + lst_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + sta = SiteStatistic.get_by_domain(self._db, domain) + if sta: + avg_seconds, note = None, {} + if seconds is not None: + note: dict = json.loads(sta.note or "{}") + note[lst_date] = seconds or 1 + avg_times = len(note.keys()) + if avg_times > 10: + note = dict(sorted(note.items(), key=lambda x: x[0], reverse=True)[:10]) + avg_seconds = sum([v for v in note.values()]) // avg_times + sta.update(self._db, { + "success": sta.success + 1, + "seconds": avg_seconds or sta.seconds, + "lst_state": 0, + "lst_mod_date": lst_date, + "note": note or sta.note + }) + else: + note = {} + if seconds is not None: + note = { + lst_date: seconds or 1 + } + SiteStatistic( + domain=domain, + success=1, + fail=0, + seconds=seconds or 1, + lst_state=0, + lst_mod_date=lst_date, + note=json.dumps(note) + ).create(self._db) + + def fail(self, domain: str): + """ + 站点访问失败 + """ + lst_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + sta = SiteStatistic.get_by_domain(self._db, domain) + if sta: + sta.update(self._db, { + "fail": sta.fail + 1, + "lst_state": 1, + "lst_mod_date": lst_date + }) + else: + SiteStatistic( + domain=domain, + success=0, + fail=1, + lst_state=1, + lst_mod_date=lst_date + ).create(self._db) diff --git a/app/db/siteicon_oper.py b/app/db/siteicon_oper.py deleted file mode 100644 index a0dea295..00000000 --- a/app/db/siteicon_oper.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import List - -from app.db import DbOper -from app.db.models.siteicon import SiteIcon - - -class SiteIconOper(DbOper): - """ - 站点管理 - """ - - def list(self) -> List[SiteIcon]: - """ - 获取站点图标列表 - """ - return SiteIcon.list(self._db) - - def get_by_domain(self, domain: str) -> SiteIcon: - """ - 按域名获取站点图标 - """ - return SiteIcon.get_by_domain(self._db, domain) - - def update_icon(self, name: str, domain: str, icon_url: str, icon_base64: str) -> bool: - """ - 更新站点图标 - """ - icon_base64 = f"data:image/ico;base64,{icon_base64}" if icon_base64 else "" - siteicon = self.get_by_domain(domain) - if not siteicon: - SiteIcon(name=name, domain=domain, url=icon_url, base64=icon_base64).create(self._db) - elif icon_base64: - siteicon.update(self._db, { - "url": icon_url, - "base64": icon_base64 - }) - return True diff --git a/app/db/sitestatistic_oper.py b/app/db/sitestatistic_oper.py deleted file mode 100644 index 18fa6d7d..00000000 --- a/app/db/sitestatistic_oper.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -from datetime import datetime - -from app.db import DbOper -from app.db.models.sitestatistic import SiteStatistic - - -class SiteStatisticOper(DbOper): - """ - 站点统计管理 - """ - - def success(self, domain: str, seconds: int = None): - """ - 站点访问成功 - """ - lst_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - sta = SiteStatistic.get_by_domain(self._db, domain) - if sta: - avg_seconds, note = None, {} - if seconds is not None: - note: dict = json.loads(sta.note or "{}") - note[lst_date] = seconds or 1 - avg_times = len(note.keys()) - if avg_times > 10: - note = dict(sorted(note.items(), key=lambda x: x[0], reverse=True)[:10]) - avg_seconds = sum([v for v in note.values()]) // avg_times - sta.update(self._db, { - "success": sta.success + 1, - "seconds": avg_seconds or sta.seconds, - "lst_state": 0, - "lst_mod_date": lst_date, - "note": note or sta.note - }) - else: - note = {} - if seconds is not None: - note = { - lst_date: seconds or 1 - } - SiteStatistic( - domain=domain, - success=1, - fail=0, - seconds=seconds or 1, - lst_state=0, - lst_mod_date=lst_date, - note=json.dumps(note) - ).create(self._db) - - def fail(self, domain: str): - """ - 站点访问失败 - """ - lst_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - sta = SiteStatistic.get_by_domain(self._db, domain) - if sta: - sta.update(self._db, { - "fail": sta.fail + 1, - "lst_state": 1, - "lst_mod_date": lst_date - }) - else: - SiteStatistic( - domain=domain, - success=0, - fail=1, - lst_state=1, - lst_mod_date=lst_date - ).create(self._db) diff --git a/app/db/subscribe_oper.py b/app/db/subscribe_oper.py index e05827c0..82eec734 100644 --- a/app/db/subscribe_oper.py +++ b/app/db/subscribe_oper.py @@ -4,6 +4,7 @@ from typing import Tuple, List from app.core.context import MediaInfo from app.db import DbOper from app.db.models.subscribe import Subscribe +from app.db.models.subscribehistory import SubscribeHistory class SubscribeOper(DbOper): @@ -102,3 +103,18 @@ class SubscribeOper(DbOper): 获取指定类型的订阅 """ return Subscribe.list_by_type(self._db, mtype=mtype, days=days) + + def add_history(self, **kwargs): + """ + 新增订阅 + """ + # 去除kwargs中 SubscribeHistory 没有的字段 + kwargs = {k: v for k, v in kwargs.items() if hasattr(SubscribeHistory, k)} + # 更新完成订阅时间 + kwargs.update({"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}) + # 去掉主键 + if "id" in kwargs: + kwargs.pop("id") + subscribe = SubscribeHistory(**kwargs) + subscribe.create(self._db) + diff --git a/app/db/subscribehistory_oper.py b/app/db/subscribehistory_oper.py deleted file mode 100644 index 701396d2..00000000 --- a/app/db/subscribehistory_oper.py +++ /dev/null @@ -1,30 +0,0 @@ -import time - -from app.db import DbOper -from app.db.models.subscribehistory import SubscribeHistory - - -class SubscribeHistoryOper(DbOper): - """ - 订阅历史管理 - """ - - def add(self, **kwargs): - """ - 新增订阅 - """ - # 去除kwargs中 SubscribeHistory 没有的字段 - kwargs = {k: v for k, v in kwargs.items() if hasattr(SubscribeHistory, k)} - # 更新完成订阅时间 - kwargs.update({"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}) - # 去掉主键 - if "id" in kwargs: - kwargs.pop("id") - subscribe = SubscribeHistory(**kwargs) - subscribe.create(self._db) - - def list_by_type(self, mtype: str, page: int = 1, count: int = 30) -> SubscribeHistory: - """ - 获取指定类型的订阅 - """ - return SubscribeHistory.list_by_type(self._db, mtype=mtype, page=page, count=count) diff --git a/app/modules/indexer/__init__.py b/app/modules/indexer/__init__.py index 47c693a1..ad548ca3 100644 --- a/app/modules/indexer/__init__.py +++ b/app/modules/indexer/__init__.py @@ -5,7 +5,7 @@ from ruamel.yaml import CommentedMap from app.core.config import settings from app.core.context import TorrentInfo -from app.db.sitestatistic_oper import SiteStatisticOper +from app.db.site_oper import SiteOper from app.helper.module import ModuleHelper from app.helper.sites import SitesHelper from app.log import logger @@ -160,9 +160,9 @@ class IndexerModule(_ModuleBase): # 统计索引情况 domain = StringUtils.get_url_domain(site.get("domain")) if error_flag: - SiteStatisticOper().fail(domain) + SiteOper().fail(domain) else: - SiteStatisticOper().success(domain=domain, seconds=seconds) + SiteOper().success(domain=domain, seconds=seconds) # 返回结果 if not result_array or len(result_array) == 0: