diff --git a/app/api/endpoints/aliyun.py b/app/api/endpoints/aliyun.py index 3354a31d..bd0f97c2 100644 --- a/app/api/endpoints/aliyun.py +++ b/app/api/endpoints/aliyun.py @@ -9,7 +9,7 @@ from app.chain.transfer import TransferChain from app.core.config import settings from app.core.metainfo import MetaInfoPath from app.core.security import verify_token, verify_uri_token -from app.helper.aliyun import AliyunHelper +from app.modules.filetransfer.storage.alipan import AliyunHelper from app.helper.progress import ProgressHelper from app.schemas.types import ProgressKey diff --git a/app/api/endpoints/u115.py b/app/api/endpoints/u115.py index b19067ea..2c68b8cb 100644 --- a/app/api/endpoints/u115.py +++ b/app/api/endpoints/u115.py @@ -10,7 +10,7 @@ from app.core.config import settings from app.core.metainfo import MetaInfoPath from app.core.security import verify_token, verify_uri_token from app.helper.progress import ProgressHelper -from app.helper.u115 import U115Helper +from app.modules.filetransfer.storage.u115 import U115Helper from app.schemas.types import ProgressKey from app.utils.http import RequestUtils diff --git a/app/chain/media.py b/app/chain/media.py index b2771884..d2e18fdd 100644 --- a/app/chain/media.py +++ b/app/chain/media.py @@ -11,8 +11,8 @@ from app.core.context import Context, MediaInfo from app.core.event import eventmanager, Event from app.core.meta import MetaBase from app.core.metainfo import MetaInfo, MetaInfoPath -from app.helper.aliyun import AliyunHelper -from app.helper.u115 import U115Helper +from app.modules.filetransfer.storage.alipan import AliyunHelper +from app.modules.filetransfer.storage.u115 import U115Helper from app.log import logger from app.schemas.types import EventType, MediaType from app.utils.http import RequestUtils diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 0f8a4ea6..6f901183 100644 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -17,11 +17,11 @@ from app.db.models.downloadhistory import DownloadHistory from app.db.models.transferhistory import TransferHistory from app.db.systemconfig_oper import SystemConfigOper from app.db.transferhistory_oper import TransferHistoryOper -from app.helper.aliyun import AliyunHelper +from app.modules.filetransfer.storage.alipan import AliyunHelper from app.helper.directory import DirectoryHelper from app.helper.format import FormatParser from app.helper.progress import ProgressHelper -from app.helper.u115 import U115Helper +from app.modules.filetransfer.storage.u115 import U115Helper from app.log import logger from app.schemas import TransferInfo, TransferTorrent, Notification, EpisodeFormat from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey, NotificationType, MessageChannel, \ diff --git a/app/modules/filetransfer/storage/__init__.py b/app/modules/filetransfer/storage/__init__.py new file mode 100644 index 00000000..bb8937b1 --- /dev/null +++ b/app/modules/filetransfer/storage/__init__.py @@ -0,0 +1,75 @@ +from abc import ABCMeta, abstractmethod +from pathlib import Path +from typing import Optional, List, Any + +from app import schemas + + +class StorageBase(metaclass=ABCMeta): + """ + 存储基类 + """ + + @abstractmethod + def check(self) -> bool: + """ + 检查存储是否可用 + """ + pass + + @abstractmethod + def list(self, fileitm: schemas.FileItem) -> Optional[List[schemas.FileItem]]: + """ + 浏览文件 + """ + pass + + @abstractmethod + def create_folder(self, fileitm: schemas.FileItem, name: str) -> Optional[schemas.FileItem]: + """ + 创建目录 + """ + pass + + @abstractmethod + def delete(self, fileitm: schemas.FileItem) -> bool: + """ + 删除文件 + """ + pass + + @abstractmethod + def rename(self, fileitm: schemas.FileItem, name: str) -> bool: + """ + 重命名文件 + """ + pass + + @abstractmethod + def download(self, fileitm: schemas.FileItem) -> Any: + """ + 下载链接 + """ + pass + + @abstractmethod + def move(self, fileitm: schemas.FileItem, target_dir: schemas.FileItem) -> bool: + """ + 移动文件 + """ + pass + + @abstractmethod + def upload(self, fileitm: schemas.FileItem, path: Path) -> Optional[schemas.FileItem]: + """ + 上传文件 + """ + pass + + @abstractmethod + def detail(self, fileitm: schemas.FileItem) -> Optional[schemas.FileItem]: + """ + 获取文件详情 + """ + pass + \ No newline at end of file diff --git a/app/helper/aliyun.py b/app/modules/filetransfer/storage/alipan.py similarity index 92% rename from app/helper/aliyun.py rename to app/modules/filetransfer/storage/alipan.py index 190e46d5..859f1dc2 100644 --- a/app/helper/aliyun.py +++ b/app/modules/filetransfer/storage/alipan.py @@ -11,13 +11,14 @@ from app import schemas from app.core.config import settings from app.db.systemconfig_oper import SystemConfigOper from app.log import logger +from app.modules.filetransfer.storage import StorageBase from app.schemas.types import SystemConfigKey from app.utils.http import RequestUtils from app.utils.string import StringUtils from app.utils.system import SystemUtils -class AliyunHelper: +class AliPan(StorageBase): """ 阿里云相关操作 """ @@ -297,6 +298,12 @@ class AliyunHelper: "x-signature": self._X_SIGNATURE } + def check(self) -> bool: + """ + 检查存储是否可用 + """ + pass + def user_info(self) -> dict: """ 获取用户信息（drive_id等） @@ -398,7 +405,7 @@ class AliyunHelper: drive_id=fileinfo.get("drive_id"), ) for fileinfo in ret_items] - def create_folder(self, drive_id: str, parent_file_id: str, name: str, path: str = "/") -> Optional[schemas.FileItem]: + def create_folder(self, fileitem: schemas.FileItem, name: str) -> Optional[schemas.FileItem]: """ 创建目录 """ @@ -407,8 +414,8 @@ class AliyunHelper: return None headers = self.__get_headers(params) res = RequestUtils(headers=headers, timeout=10).post_res(self.create_folder_file_url, json={ - "drive_id": drive_id, - "parent_file_id": parent_file_id, + "drive_id": fileitem.drive_id, + "parent_file_id": fileitem.parent_fileid, "name": name, "check_name_mode": "refuse", "type": "folder" @@ -432,13 +439,13 @@ class AliyunHelper: parent_fileid=result.get("parent_file_id"), type=result.get("type"), name=result.get("file_name"), - path=f"{path}{result.get('file_name')}", + path=f"{fileitem.path}{result.get('file_name')}", ) else: self.__handle_error(res, "创建目录") return None - def delete(self, drive_id: str, file_id: str) -> bool: + def delete(self, fileitem: schemas.FileItem) -> bool: """ 删除文件 """ @@ -447,8 +454,8 @@ class AliyunHelper: return False headers = self.__get_headers(params) res = RequestUtils(headers=headers, timeout=10).post_res(self.delete_file_url, json={ - "drive_id": drive_id, - "file_id": file_id + "drive_id": fileitem.drive_id, + "file_id": fileitem.fileid }) if res: return True @@ -456,7 +463,7 @@ class AliyunHelper: self.__handle_error(res, "删除文件") return False - def detail(self, drive_id: str, file_id: str, path: str = "/") -> Optional[schemas.FileItem]: + def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]: """ 获取文件详情 """ @@ -465,8 +472,8 @@ class AliyunHelper: return None headers = self.__get_headers(params) res = RequestUtils(headers=headers, timeout=10).post_res(self.file_detail_url, json={ - "drive_id": drive_id, - "file_id": file_id + "drive_id": fileitem.drive_id, + "file_id": fileitem.fileid }) if res: result = res.json() @@ -480,13 +487,13 @@ class AliyunHelper: extension=result.get("file_extension"), modify_time=StringUtils.str_to_timestamp(result.get("updated_at")), thumbnail=result.get("thumbnail"), - path=f"{path}{result.get('name')}" + path=f"{fileitem.path}{result.get('name')}" ) else: self.__handle_error(res, "获取文件详情") return None - def rename(self, drive_id: str, file_id: str, name: str) -> bool: + def rename(self, fileitem: schemas.FileItem, name: str) -> bool: """ 重命名文件 """ @@ -495,8 +502,8 @@ class AliyunHelper: return False headers = self.__get_headers(params) res = RequestUtils(headers=headers, timeout=10).post_res(self.rename_file_url, json={ - "drive_id": drive_id, - "file_id": file_id, + "drive_id": fileitem.drive_id, + "file_id": fileitem.fileid, "name": name, "check_name_mode": "refuse" }) @@ -506,7 +513,7 @@ class AliyunHelper: self.__handle_error(res, "重命名文件") return False - def download(self, drive_id: str, file_id: str) -> Optional[str]: + def download(self, fileitem: schemas.FileItem) -> Optional[str]: """ 获取下载链接 """ @@ -515,8 +522,8 @@ class AliyunHelper: return None headers = self.__get_headers(params) res = RequestUtils(headers=headers, timeout=10).post_res(self.download_url, json={ - "drive_id": drive_id, - "file_id": file_id + "drive_id": fileitem.drive_id, + "file_id": fileitem.fileid }) if res: return res.json().get("url") @@ -524,7 +531,7 @@ class AliyunHelper: self.__handle_error(res, "获取下载链接") return None - def move(self, drive_id: str, file_id: str, target_id: str) -> bool: + def move(self, fileitem: schemas.FileItem, target_dir: schemas.FileItem) -> bool: """ 移动文件 """ @@ -533,9 +540,9 @@ class AliyunHelper: return False headers = self.__get_headers(params) res = RequestUtils(headers=headers, timeout=10).post_res(self.move_file_url, json={ - "drive_id": drive_id, - "file_id": file_id, - "to_parent_file_id": target_id, + "drive_id": fileitem.drive_id, + "file_id": fileitem.fileid, + "to_parent_file_id": target_dir.fileid, "check_name_mode": "refuse" }) if res: @@ -544,7 +551,7 @@ class AliyunHelper: self.__handle_error(res, "移动文件") return False - def upload(self, drive_id: str, parent_file_id: str, file_path: Path) -> Optional[schemas.FileItem]: + def upload(self, fileitem: schemas.FileItem, path: Path) -> Optional[schemas.FileItem]: """ 上传文件，并标记完成 """ @@ -553,9 +560,9 @@ class AliyunHelper: return None headers = self.__get_headers(params) res = RequestUtils(headers=headers, timeout=10).post_res(self.create_folder_file_url, json={ - "drive_id": drive_id, - "parent_file_id": parent_file_id, - "name": file_path.name, + "drive_id": fileitem.drive_id, + "parent_file_id": fileitem.parent_fileid, + "name": path.name, "check_name_mode": "refuse", "create_scene": "file_upload", "type": "file", @@ -564,7 +571,7 @@ class AliyunHelper: "part_number": 1 } ], - "size": file_path.stat().st_size + "size": path.stat().st_size }) if not res: self.__handle_error(res, "创建文件") @@ -579,7 +586,7 @@ class AliyunHelper: parent_fileid=result.get("parent_file_id"), type="file", name=result.get("file_name"), - path=f"{file_path.parent}/{result.get('file_name')}" + path=f"{fileitem.path}{result.get('file_name')}" ) file_id = result.get("file_id") upload_id = result.get("upload_id") @@ -593,13 +600,13 @@ class AliyunHelper: "User-Agent": settings.USER_AGENT, "Referer": "https://www.alipan.com/", "Accept": "*/*", - }).put_res(upload_url, data=file_path.read_bytes()) + }).put_res(upload_url, data=path.read_bytes()) if not res: self.__handle_error(res, "上传文件") return None # 标记文件上传完毕 res = RequestUtils(headers=headers, timeout=10).post_res(self.upload_file_complete_url, json={ - "drive_id": drive_id, + "drive_id": fileitem.drive_id, "file_id": file_id, "upload_id": upload_id }) @@ -613,7 +620,7 @@ class AliyunHelper: parent_fileid=result.get("parent_file_id"), type="file", name=result.get("name"), - path=f"{file_path.parent}/{result.get('name')}", + path=f"{fileitem.path}{result.get('name')}", ) else: logger.warn("上传文件失败：无法获取上传地址！") diff --git a/app/modules/filetransfer/storage/local.py b/app/modules/filetransfer/storage/local.py new file mode 100644 index 00000000..aaae0992 --- /dev/null +++ b/app/modules/filetransfer/storage/local.py @@ -0,0 +1,200 @@ +import shutil +from pathlib import Path +from typing import Optional, List + +from starlette.responses import FileResponse, Response + +from app import schemas +from app.log import logger +from app.modules.filetransfer.storage import StorageBase +from app.utils.system import SystemUtils + + +class LocalStorage(StorageBase): + """ + 本地文件操作 + """ + + def check(self) -> bool: + """ + 检查存储是否可用 + """ + return True + + def list(self, fileitem: schemas.FileItem) -> Optional[List[schemas.FileItem]]: + """ + 浏览文件 + """ + # 返回结果 + ret_items = [] + path = fileitem.path + if not fileitem.path or fileitem.path == "/": + if SystemUtils.is_windows(): + partitions = SystemUtils.get_windows_drives() or ["C:/"] + for partition in partitions: + ret_items.append(schemas.FileItem( + type="dir", + path=partition + "/", + name=partition, + basename=partition + )) + return ret_items + else: + path = "/" + else: + if SystemUtils.is_windows(): + path = path.lstrip("/") + elif not path.startswith("/"): + path = "/" + path + + # 遍历目录 + path_obj = Path(path) + if not path_obj.exists(): + logger.warn(f"目录不存在：{path}") + return [] + + # 如果是文件 + if path_obj.is_file(): + ret_items.append(schemas.FileItem( + type="file", + path=str(path_obj).replace("\\", "/"), + name=path_obj.name, + basename=path_obj.stem, + extension=path_obj.suffix[1:], + size=path_obj.stat().st_size, + modify_time=path_obj.stat().st_mtime, + )) + return ret_items + + # 扁历所有目录 + for item in SystemUtils.list_sub_directory(path_obj): + ret_items.append(schemas.FileItem( + type="dir", + path=str(item).replace("\\", "/") + "/", + name=item.name, + basename=item.stem, + modify_time=item.stat().st_mtime, + )) + + # 遍历所有文件，不含子目录 + for item in SystemUtils.list_sub_all(path_obj): + ret_items.append(schemas.FileItem( + type="file", + path=str(item).replace("\\", "/"), + name=item.name, + basename=item.stem, + extension=item.suffix[1:], + size=item.stat().st_size, + modify_time=item.stat().st_mtime, + )) + return ret_items + + def create_folder(self, fileitem: schemas.FileItem, name: str) -> Optional[schemas.FileItem]: + """ + 创建目录 + """ + if not fileitem.path: + return None + path_obj = Path(fileitem.path) / name + if path_obj.exists(): + return None + path_obj.mkdir(parents=True, exist_ok=True) + return schemas.FileItem( + type="dir", + path=str(path_obj).replace("\\", "/") + "/", + name=name, + basename=name, + modify_time=path_obj.stat().st_mtime, + ) + + def detail(self, fileitm: schemas.FileItem) -> Optional[schemas.FileItem]: + """ + 获取文件详情 + """ + path_obj = Path(fileitm.path) + return schemas.FileItem( + type="file", + path=str(path_obj).replace("\\", "/"), + name=path_obj.name, + basename=path_obj.stem, + extension=path_obj.suffix[1:], + size=path_obj.stat().st_size, + modify_time=path_obj.stat().st_mtime, + ) + + def delete(self, fileitem: schemas.FileItem) -> bool: + """ + 删除文件 + """ + if not fileitem.path: + return False + path_obj = Path(fileitem.path) + if not path_obj.exists(): + return False + if path_obj.is_file(): + path_obj.unlink() + else: + shutil.rmtree(path_obj, ignore_errors=True) + return True + + def rename(self, fileitem: schemas.FileItem, name: str) -> bool: + """ + 重命名文件 + """ + path_obj = Path(fileitem.path) + if not path_obj.exists(): + return False + path_obj.rename(path_obj.parent / name) + + def download(self, fileitem: schemas.FileItem) -> Optional[Response]: + """ + 下载文件 + """ + if not fileitem.path: + return None + path_obj = Path(fileitem.path) + if not path_obj.exists(): + return None + if path_obj.is_file(): + # 做为文件流式下载 + return FileResponse(path_obj) + else: + # 做为压缩包下载 + shutil.make_archive(base_name=path_obj.stem, format="zip", root_dir=path_obj) + reponse = Response(content=path_obj.read_bytes(), media_type="application/zip") + # 删除压缩包 + Path(f"{path_obj.stem}.zip").unlink() + return reponse + + def move(self, fileitem: schemas.FileItem, target_dir: schemas.FileItem) -> bool: + """ + 移动文件 + """ + if not fileitem.path or not target_dir.path: + return False + path_obj = Path(fileitem.path) + target_obj = Path(target_dir.path) + if not path_obj.exists() or not target_obj.exists(): + return False + path_obj.rename(target_obj / path_obj.name) + return True + + def upload(self, fileitem: schemas.FileItem, path: Path) -> Optional[schemas.FileItem]: + """ + 上传文件 + """ + if not fileitem.path: + return None + path_obj = Path(fileitem.path) + if not path_obj.exists(): + return None + shutil.copy(path, path_obj / path.name) + return schemas.FileItem( + type="file", + path=str(path_obj / path.name).replace("\\", "/"), + name=path.name, + basename=path.stem, + extension=path.suffix[1:], + size=path.stat().st_size, + modify_time=path.stat().st_mtime, + ) diff --git a/app/helper/u115.py b/app/modules/filetransfer/storage/u115.py similarity index 83% rename from app/helper/u115.py rename to app/modules/filetransfer/storage/u115.py index d7752eaa..4e29ff23 100644 --- a/app/helper/u115.py +++ b/app/modules/filetransfer/storage/u115.py @@ -10,11 +10,12 @@ from py115.types import LoginTarget, QrcodeSession, QrcodeStatus, Credential, Do from app import schemas from app.db.systemconfig_oper import SystemConfigOper from app.log import logger +from app.modules.filetransfer.storage import StorageBase from app.schemas.types import SystemConfigKey from app.utils.singleton import Singleton -class U115Helper(metaclass=Singleton): +class U115Pan(StorageBase, metaclass=Singleton): """ 115相关操作 """ @@ -138,19 +139,25 @@ class U115Helper(metaclass=Singleton): logger.error(f"获取115存储空间失败：{str(e)}") return None - def list(self, parent_file_id: str = '0', path: str = "/") -> Optional[List[schemas.FileItem]]: + def check(self) -> bool: + """ + 检查存储是否可用 + """ + pass + + def list(self, fileitem: schemas.FileItem) -> Optional[List[schemas.FileItem]]: """ 浏览文件 """ if not self.__init_cloud(): return None try: - items = self.cloud.storage().list(dir_id=parent_file_id) + items = self.cloud.storage().list(dir_id=fileitem.parent_fileid) return [schemas.FileItem( fileid=item.file_id, parent_fileid=item.parent_id, type="dir" if item.is_dir else "file", - path=f"{path}{item.name}" + ("/" if item.is_dir else ""), + path=f"{fileitem.path}{item.name}" + ("/" if item.is_dir else ""), name=item.name, size=item.size, extension=Path(item.name).suffix[1:], @@ -161,19 +168,19 @@ class U115Helper(metaclass=Singleton): logger.error(f"浏览115文件失败：{str(e)}") return None - def create_folder(self, parent_file_id: str, name: str, path: str = "/") -> Optional[schemas.FileItem]: + def create_folder(self, fileitem: schemas.FileItem, name: str) -> Optional[schemas.FileItem]: """ 创建目录 """ if not self.__init_cloud(): return None try: - result = self.cloud.storage().make_dir(parent_file_id, name) + result = self.cloud.storage().make_dir(fileitem.parent_fileid, name) return schemas.FileItem( fileid=result.file_id, parent_fileid=result.parent_id, type="dir", - path=f"{path}{name}/", + path=f"{fileitem.path}{name}/", name=name, modify_time=result.modified_time.timestamp() if result.modified_time else 0, pickcode=result.pickcode @@ -182,65 +189,71 @@ class U115Helper(metaclass=Singleton): logger.error(f"创建115目录失败：{str(e)}") return None - def delete(self, file_id: str) -> bool: + def detail(self, fileitm: schemas.FileItem) -> Optional[schemas.FileItem]: + """ + 获取文件详情 + """ + pass + + def delete(self, fileitem: schemas.FileItem) -> bool: """ 删除文件 """ if not self.__init_cloud(): return False try: - self.cloud.storage().delete(file_id) + self.cloud.storage().delete(fileitem.fileid) return True except Exception as e: logger.error(f"删除115文件失败：{str(e)}") return False - def rename(self, file_id: str, name: str) -> bool: + def rename(self, fileitem: schemas.FileItem, name: str) -> bool: """ 重命名文件 """ if not self.__init_cloud(): return False try: - self.cloud.storage().rename(file_id, name) + self.cloud.storage().rename(fileitem.fileid, name) return True except Exception as e: logger.error(f"重命名115文件失败：{str(e)}") return False - def download(self, pickcode: str) -> Optional[DownloadTicket]: + def download(self, fileitem: schemas.FileItem) -> Optional[DownloadTicket]: """ 获取下载链接 """ if not self.__init_cloud(): return None try: - return self.cloud.storage().request_download(pickcode) + return self.cloud.storage().request_download(fileitem.pickcode) except Exception as e: logger.error(f"115下载失败：{str(e)}") return None - def move(self, file_id: str, target_id: str) -> bool: + def move(self, fileitem: schemas.FileItem, target_dir: schemas.FileItem) -> bool: """ 移动文件 """ if not self.__init_cloud(): return False try: - self.cloud.storage().move(file_id, target_id) + self.cloud.storage().move(fileitem.fileid, target_dir.fileid) return True except Exception as e: logger.error(f"移动115文件失败：{str(e)}") return False - def upload(self, parent_file_id: str, file_path: Path) -> Optional[schemas.FileItem]: + def upload(self, fileitem: schemas.FileItem, path: Path) -> Optional[schemas.FileItem]: """ 上传文件 """ if not self.__init_cloud(): return None try: - ticket = self.cloud.storage().request_upload(dir_id=parent_file_id, file_path=str(file_path)) + ticket = self.cloud.storage().request_upload(dir_id=fileitem.fileid, file_path=str(path)) if ticket is None: logger.warn(f"115请求上传出错") return None @@ -256,7 +269,7 @@ class U115Helper(metaclass=Singleton): ) por = bucket.put_object_from_file( key=ticket.object_key, - filename=str(file_path), + filename=str(path), headers=ticket.headers, ) result = por.resp.response.json() @@ -265,10 +278,10 @@ class U115Helper(metaclass=Singleton): logger.info(f"115上传文件成功：{fileitem}") return schemas.FileItem( fileid=fileitem.get('file_id'), - parent_fileid=parent_file_id, + parent_fileid=fileitem.fileid, type="file", name=fileitem.get('file_name'), - path=f"{file_path / fileitem.get('file_name')}", + path=f"{fileitem.path}{fileitem.get('file_name')}", size=fileitem.get('file_size'), extension=Path(fileitem.get('file_name')).suffix[1:], pickcode=fileitem.get('pickcode') diff --git a/app/modules/indexer/parser/__init__.py b/app/modules/indexer/parser/__init__.py index d7b5ea81..97ae48d1 100644 --- a/app/modules/indexer/parser/__init__.py +++ b/app/modules/indexer/parser/__init__.py @@ -14,8 +14,6 @@ from app.log import logger from app.utils.http import RequestUtils from app.utils.site import SiteUtils -SITE_BASE_ORDER = 1000 - # 站点框架 class SiteSchema(Enum): @@ -39,8 +37,6 @@ class SiteSchema(Enum): class SiteParserBase(metaclass=ABCMeta): # 站点模版 schema = SiteSchema.NexusPhp - # 站点解析时判断顺序，值越小越先解析 - order = SITE_BASE_ORDER # 请求模式 cookie/apikey request_mode = "cookie" @@ -155,15 +151,6 @@ class SiteParserBase(metaclass=ABCMeta): """ return self.schema - @classmethod - def match(cls, html_text: str) -> bool: - """ - 是否匹配当前解析模型 - :param html_text: 站点首页html - :return: 是否匹配 - """ - pass - def parse(self): """ 解析站点信息 diff --git a/app/modules/indexer/parser/discuz.py b/app/modules/indexer/parser/discuz.py index 70389ce6..67098daa 100644 --- a/app/modules/indexer/parser/discuz.py +++ b/app/modules/indexer/parser/discuz.py @@ -4,21 +4,12 @@ from typing import Optional from lxml import etree -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class DiscuzUserInfo(SiteParserBase): schema = SiteSchema.DiscuzX - order = SITE_BASE_ORDER + 10 - - @classmethod - def match(cls, html_text: str) -> bool: - html = etree.HTML(html_text) - if not html: - return False - printable_text = html.xpath("string(.)") if html else "" - return 'Powered by Discuz!' in printable_text def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/file_list.py b/app/modules/indexer/parser/file_list.py index dd8a484f..4991a8b9 100644 --- a/app/modules/indexer/parser/file_list.py +++ b/app/modules/indexer/parser/file_list.py @@ -4,22 +4,12 @@ from typing import Optional from lxml import etree -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class FileListSiteUserInfo(SiteParserBase): schema = SiteSchema.FileList - order = SITE_BASE_ORDER + 50 - - @classmethod - def match(cls, html_text: str) -> bool: - html = etree.HTML(html_text) - if not html: - return False - - printable_text = html.xpath("string(.)") if html else "" - return 'Powered by FileList' in printable_text def _parse_site_page(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/gazelle.py b/app/modules/indexer/parser/gazelle.py index 766d57d9..73616768 100644 --- a/app/modules/indexer/parser/gazelle.py +++ b/app/modules/indexer/parser/gazelle.py @@ -4,23 +4,12 @@ from typing import Optional from lxml import etree -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class GazelleSiteUserInfo(SiteParserBase): schema = SiteSchema.Gazelle - order = SITE_BASE_ORDER - - @classmethod - def match(cls, html_text: str) -> bool: - html = etree.HTML(html_text) - if not html: - return False - - printable_text = html.xpath("string(.)") if html else "" - - return "Powered by Gazelle" in printable_text or "DIC Music" in printable_text def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/ipt_project.py b/app/modules/indexer/parser/ipt_project.py index da0f3325..1e4eb7a5 100644 --- a/app/modules/indexer/parser/ipt_project.py +++ b/app/modules/indexer/parser/ipt_project.py @@ -4,17 +4,12 @@ from typing import Optional from lxml import etree -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class IptSiteUserInfo(SiteParserBase): schema = SiteSchema.Ipt - order = SITE_BASE_ORDER + 35 - - @classmethod - def match(cls, html_text: str) -> bool: - return 'IPTorrents' in html_text def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/mtorrent.py b/app/modules/indexer/parser/mtorrent.py index 996a3ded..5194043e 100644 --- a/app/modules/indexer/parser/mtorrent.py +++ b/app/modules/indexer/parser/mtorrent.py @@ -3,16 +3,13 @@ import json from typing import Optional, Tuple from urllib.parse import urljoin -from lxml import etree - from app.log import logger -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class MTorrentSiteUserInfo(SiteParserBase): schema = SiteSchema.MTorrent - order = SITE_BASE_ORDER + 60 request_mode = "apikey" # 用户级别字典 @@ -37,15 +34,6 @@ class MTorrentSiteUserInfo(SiteParserBase): "18": "Bet memberStaff", } - @classmethod - def match(cls, html_text: str) -> bool: - html = etree.HTML(html_text) - if not html: - return False - if html.xpath("//title/text()") and "M-Team" in html.xpath("//title/text()")[0]: - return True - return False - def _parse_site_page(self, html_text: str): """ 获取站点页面地址 diff --git a/app/modules/indexer/parser/nexus_audiences.py b/app/modules/indexer/parser/nexus_audiences.py index d1d6866c..7590572e 100644 --- a/app/modules/indexer/parser/nexus_audiences.py +++ b/app/modules/indexer/parser/nexus_audiences.py @@ -1,17 +1,12 @@ # -*- coding: utf-8 -*- from urllib.parse import urljoin -from app.modules.indexer.parser import SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteSchema from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo): schema = SiteSchema.NexusAudiences - order = SITE_BASE_ORDER + 5 - - @classmethod - def match(cls, html_text: str) -> bool: - return 'audiences.me' in html_text def _parse_site_page(self, html_text: str): super()._parse_site_page(html_text) diff --git a/app/modules/indexer/parser/nexus_hhanclub.py b/app/modules/indexer/parser/nexus_hhanclub.py index d60f5c9c..af3b57e0 100644 --- a/app/modules/indexer/parser/nexus_hhanclub.py +++ b/app/modules/indexer/parser/nexus_hhanclub.py @@ -3,18 +3,13 @@ import re from lxml import etree -from app.modules.indexer.parser import SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteSchema from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo from app.utils.string import StringUtils class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo): schema = SiteSchema.NexusHhanclub - order = SITE_BASE_ORDER + 20 - - @classmethod - def match(cls, html_text: str) -> bool: - return 'hhanclub.top' in html_text def _parse_user_traffic_info(self, html_text): super()._parse_user_traffic_info(html_text) diff --git a/app/modules/indexer/parser/nexus_php.py b/app/modules/indexer/parser/nexus_php.py index 121c536a..33c8651f 100644 --- a/app/modules/indexer/parser/nexus_php.py +++ b/app/modules/indexer/parser/nexus_php.py @@ -5,22 +5,12 @@ from typing import Optional from lxml import etree from app.log import logger -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class NexusPhpSiteUserInfo(SiteParserBase): schema = SiteSchema.NexusPhp - order = SITE_BASE_ORDER * 2 - - @classmethod - def match(cls, html_text: str) -> bool: - """ - 默认使用NexusPhp解析 - :param html_text: - :return: - """ - return True def _parse_site_page(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/nexus_project.py b/app/modules/indexer/parser/nexus_project.py index 153ba92c..1c854d6b 100644 --- a/app/modules/indexer/parser/nexus_project.py +++ b/app/modules/indexer/parser/nexus_project.py @@ -1,17 +1,12 @@ # -*- coding: utf-8 -*- import re -from app.modules.indexer.parser import SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteSchema from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo class NexusProjectSiteUserInfo(NexusPhpSiteUserInfo): schema = SiteSchema.NexusProject - order = SITE_BASE_ORDER + 25 - - @classmethod - def match(cls, html_text: str) -> bool: - return 'Nexus Project' in html_text def _parse_site_page(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/nexus_rabbit.py b/app/modules/indexer/parser/nexus_rabbit.py index 394eabc2..f5a24fdc 100644 --- a/app/modules/indexer/parser/nexus_rabbit.py +++ b/app/modules/indexer/parser/nexus_rabbit.py @@ -2,25 +2,13 @@ import json from typing import Optional -from lxml import etree - from app.log import logger -from app.modules.indexer.parser import SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteSchema from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo class NexusRabbitSiteUserInfo(NexusPhpSiteUserInfo): schema = SiteSchema.NexusRabbit - order = SITE_BASE_ORDER + 5 - - @classmethod - def match(cls, html_text: str) -> bool: - html = etree.HTML(html_text) - if not html: - return False - - printable_text = html.xpath("string(.)") if html else "" - return 'Style by Rabbit' in printable_text def _parse_site_page(self, html_text: str): super()._parse_site_page(html_text) diff --git a/app/modules/indexer/parser/small_horse.py b/app/modules/indexer/parser/small_horse.py index 1db46a6f..31968144 100644 --- a/app/modules/indexer/parser/small_horse.py +++ b/app/modules/indexer/parser/small_horse.py @@ -4,17 +4,12 @@ from typing import Optional from lxml import etree -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class SmallHorseSiteUserInfo(SiteParserBase): schema = SiteSchema.SmallHorse - order = SITE_BASE_ORDER + 30 - - @classmethod - def match(cls, html_text: str) -> bool: - return 'Small Horse' in html_text def _parse_site_page(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/tnode.py b/app/modules/indexer/parser/tnode.py index f3b04d7b..5aad758c 100644 --- a/app/modules/indexer/parser/tnode.py +++ b/app/modules/indexer/parser/tnode.py @@ -3,17 +3,12 @@ import json import re from typing import Optional -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class TNodeSiteUserInfo(SiteParserBase): schema = SiteSchema.TNode - order = SITE_BASE_ORDER + 60 - - @classmethod - def match(cls, html_text: str) -> bool: - return 'Powered By TNode' in html_text def _parse_site_page(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/torrent_leech.py b/app/modules/indexer/parser/torrent_leech.py index 9158a45a..4bd44c1e 100644 --- a/app/modules/indexer/parser/torrent_leech.py +++ b/app/modules/indexer/parser/torrent_leech.py @@ -4,17 +4,12 @@ from typing import Optional from lxml import etree -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class TorrentLeechSiteUserInfo(SiteParserBase): schema = SiteSchema.TorrentLeech - order = SITE_BASE_ORDER + 40 - - @classmethod - def match(cls, html_text: str) -> bool: - return 'TorrentLeech' in html_text def _parse_site_page(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/unit3d.py b/app/modules/indexer/parser/unit3d.py index 21a24ea4..98e48e02 100644 --- a/app/modules/indexer/parser/unit3d.py +++ b/app/modules/indexer/parser/unit3d.py @@ -4,17 +4,12 @@ from typing import Optional from lxml import etree -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class Unit3dSiteUserInfo(SiteParserBase): schema = SiteSchema.Unit3d - order = SITE_BASE_ORDER + 15 - - @classmethod - def match(cls, html_text: str) -> bool: - return "unit3d.js" in html_text def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) diff --git a/app/modules/indexer/parser/yema.py b/app/modules/indexer/parser/yema.py index 885fab76..03d0e74e 100644 --- a/app/modules/indexer/parser/yema.py +++ b/app/modules/indexer/parser/yema.py @@ -2,17 +2,12 @@ import json from typing import Optional, Tuple -from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER +from app.modules.indexer.parser import SiteParserBase, SiteSchema from app.utils.string import StringUtils class TYemaSiteUserInfo(SiteParserBase): schema = SiteSchema.Yema - order = SITE_BASE_ORDER + 60 - - @classmethod - def match(cls, html_text: str) -> bool: - return 'YemaPT' in html_text def _parse_site_page(self, html_text: str): """