nas/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-04-05 03:38:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

add storage

Browse Source
This commit is contained in:
jxxghp
2024-06-30 08:59:12 +08:00
parent 77632880d1
commit 63ca5ee313
24 changed files with 367 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/api/endpoints/aliyun.py
View File

@@ -9,7 +9,7 @@ from app.chain.transfer import TransferChain
from app.core.config import settings
from app.core.metainfo import MetaInfoPath
from app.core.security import verify_token, verify_uri_token
from app.helper.aliyun import AliyunHelper
from app.modules.filetransfer.storage.alipan import AliyunHelper
from app.helper.progress import ProgressHelper
from app.schemas.types import ProgressKey

2
app/api/endpoints/u115.py
View File

@@ -10,7 +10,7 @@ from app.core.config import settings
from app.core.metainfo import MetaInfoPath
from app.core.security import verify_token, verify_uri_token
from app.helper.progress import ProgressHelper
from app.helper.u115 import U115Helper
from app.modules.filetransfer.storage.u115 import U115Helper
from app.schemas.types import ProgressKey
from app.utils.http import RequestUtils

4
app/chain/media.py
View File

@@ -11,8 +11,8 @@ from app.core.context import Context, MediaInfo
from app.core.event import eventmanager, Event
from app.core.meta import MetaBase
from app.core.metainfo import MetaInfo, MetaInfoPath
from app.helper.aliyun import AliyunHelper
from app.helper.u115 import U115Helper
from app.modules.filetransfer.storage.alipan import AliyunHelper
from app.modules.filetransfer.storage.u115 import U115Helper
from app.log import logger
from app.schemas.types import EventType, MediaType
from app.utils.http import RequestUtils

4
app/chain/transfer.py
View File

@@ -17,11 +17,11 @@ from app.db.models.downloadhistory import DownloadHistory
from app.db.models.transferhistory import TransferHistory
from app.db.systemconfig_oper import SystemConfigOper
from app.db.transferhistory_oper import TransferHistoryOper
from app.helper.aliyun import AliyunHelper
from app.modules.filetransfer.storage.alipan import AliyunHelper
from app.helper.directory import DirectoryHelper
from app.helper.format import FormatParser
from app.helper.progress import ProgressHelper
from app.helper.u115 import U115Helper
from app.modules.filetransfer.storage.u115 import U115Helper
from app.log import logger
from app.schemas import TransferInfo, TransferTorrent, Notification, EpisodeFormat
from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey, NotificationType, MessageChannel, \

75
app/modules/filetransfer/storage/__init__.py Normal file
View File

@@ -0,0 +1,75 @@
from abc import ABCMeta, abstractmethod
from pathlib import Path
from typing import Optional, List, Any
from app import schemas
class StorageBase(metaclass=ABCMeta):
"""
存储基类
"""
@abstractmethod
def check(self) -> bool:
"""
检查存储是否可用
"""
pass
@abstractmethod
def list(self, fileitm: schemas.FileItem) -> Optional[List[schemas.FileItem]]:
"""
浏览文件
"""
pass
@abstractmethod
def create_folder(self, fileitm: schemas.FileItem, name: str) -> Optional[schemas.FileItem]:
"""
创建目录
"""
pass
@abstractmethod
def delete(self, fileitm: schemas.FileItem) -> bool:
"""
删除文件
"""
pass
@abstractmethod
def rename(self, fileitm: schemas.FileItem, name: str) -> bool:
"""
重命名文件
"""
pass
@abstractmethod
def download(self, fileitm: schemas.FileItem) -> Any:
"""
下载链接
"""
pass
@abstractmethod
def move(self, fileitm: schemas.FileItem, target_dir: schemas.FileItem) -> bool:
"""
移动文件
"""
pass
@abstractmethod
def upload(self, fileitm: schemas.FileItem, path: Path) -> Optional[schemas.FileItem]:
"""
上传文件
"""
pass
@abstractmethod
def detail(self, fileitm: schemas.FileItem) -> Optional[schemas.FileItem]:
"""
获取文件详情
"""
pass

69
app/helper/aliyun.py → app/modules/filetransfer/storage/alipan.py
View File

@@ -11,13 +11,14 @@ from app import schemas
from app.core.config import settings
from app.db.systemconfig_oper import SystemConfigOper
from app.log import logger
from app.modules.filetransfer.storage import StorageBase
from app.schemas.types import SystemConfigKey
from app.utils.http import RequestUtils
from app.utils.string import StringUtils
from app.utils.system import SystemUtils
class AliyunHelper:
class AliPan(StorageBase):
"""
阿里云相关操作
"""
@@ -297,6 +298,12 @@ class AliyunHelper:
"x-signature": self._X_SIGNATURE
}
def check(self) -> bool:
"""
检查存储是否可用
"""
pass
def user_info(self) -> dict:
"""
获取用户信息（drive_id等）
@@ -398,7 +405,7 @@ class AliyunHelper:
drive_id=fileinfo.get("drive_id"),
) for fileinfo in ret_items]
def create_folder(self, drive_id: str, parent_file_id: str, name: str, path: str = "/") -> Optional[schemas.FileItem]:
def create_folder(self, fileitem: schemas.FileItem, name: str) -> Optional[schemas.FileItem]:
"""
创建目录
"""
@@ -407,8 +414,8 @@ class AliyunHelper:
return None
headers = self.__get_headers(params)
res = RequestUtils(headers=headers, timeout=10).post_res(self.create_folder_file_url, json={
"drive_id": drive_id,
"parent_file_id": parent_file_id,
"drive_id": fileitem.drive_id,
"parent_file_id": fileitem.parent_fileid,
"name": name,
"check_name_mode": "refuse",
"type": "folder"
@@ -432,13 +439,13 @@ class AliyunHelper:
parent_fileid=result.get("parent_file_id"),
type=result.get("type"),
name=result.get("file_name"),
path=f"{path}{result.get('file_name')}",
path=f"{fileitem.path}{result.get('file_name')}",
)
else:
self.__handle_error(res, "创建目录")
return None
def delete(self, drive_id: str, file_id: str) -> bool:
def delete(self, fileitem: schemas.FileItem) -> bool:
"""
删除文件
"""
@@ -447,8 +454,8 @@ class AliyunHelper:
return False
headers = self.__get_headers(params)
res = RequestUtils(headers=headers, timeout=10).post_res(self.delete_file_url, json={
"drive_id": drive_id,
"file_id": file_id
"drive_id": fileitem.drive_id,
"file_id": fileitem.fileid
})
if res:
return True
@@ -456,7 +463,7 @@ class AliyunHelper:
self.__handle_error(res, "删除文件")
return False
def detail(self, drive_id: str, file_id: str, path: str = "/") -> Optional[schemas.FileItem]:
def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
"""
获取文件详情
"""
@@ -465,8 +472,8 @@ class AliyunHelper:
return None
headers = self.__get_headers(params)
res = RequestUtils(headers=headers, timeout=10).post_res(self.file_detail_url, json={
"drive_id": drive_id,
"file_id": file_id
"drive_id": fileitem.drive_id,
"file_id": fileitem.fileid
})
if res:
result = res.json()
@@ -480,13 +487,13 @@ class AliyunHelper:
extension=result.get("file_extension"),
modify_time=StringUtils.str_to_timestamp(result.get("updated_at")),
thumbnail=result.get("thumbnail"),
path=f"{path}{result.get('name')}"
path=f"{fileitem.path}{result.get('name')}"
)
else:
self.__handle_error(res, "获取文件详情")
return None
def rename(self, drive_id: str, file_id: str, name: str) -> bool:
def rename(self, fileitem: schemas.FileItem, name: str) -> bool:
"""
重命名文件
"""
@@ -495,8 +502,8 @@ class AliyunHelper:
return False
headers = self.__get_headers(params)
res = RequestUtils(headers=headers, timeout=10).post_res(self.rename_file_url, json={
"drive_id": drive_id,
"file_id": file_id,
"drive_id": fileitem.drive_id,
"file_id": fileitem.fileid,
"name": name,
"check_name_mode": "refuse"
})
@@ -506,7 +513,7 @@ class AliyunHelper:
self.__handle_error(res, "重命名文件")
return False
def download(self, drive_id: str, file_id: str) -> Optional[str]:
def download(self, fileitem: schemas.FileItem) -> Optional[str]:
"""
获取下载链接
"""
@@ -515,8 +522,8 @@ class AliyunHelper:
return None
headers = self.__get_headers(params)
res = RequestUtils(headers=headers, timeout=10).post_res(self.download_url, json={
"drive_id": drive_id,
"file_id": file_id
"drive_id": fileitem.drive_id,
"file_id": fileitem.fileid
})
if res:
return res.json().get("url")
@@ -524,7 +531,7 @@ class AliyunHelper:
self.__handle_error(res, "获取下载链接")
return None
def move(self, drive_id: str, file_id: str, target_id: str) -> bool:
def move(self, fileitem: schemas.FileItem, target_dir: schemas.FileItem) -> bool:
"""
移动文件
"""
@@ -533,9 +540,9 @@ class AliyunHelper:
return False
headers = self.__get_headers(params)
res = RequestUtils(headers=headers, timeout=10).post_res(self.move_file_url, json={
"drive_id": drive_id,
"file_id": file_id,
"to_parent_file_id": target_id,
"drive_id": fileitem.drive_id,
"file_id": fileitem.fileid,
"to_parent_file_id": target_dir.fileid,
"check_name_mode": "refuse"
})
if res:
@@ -544,7 +551,7 @@ class AliyunHelper:
self.__handle_error(res, "移动文件")
return False
def upload(self, drive_id: str, parent_file_id: str, file_path: Path) -> Optional[schemas.FileItem]:
def upload(self, fileitem: schemas.FileItem, path: Path) -> Optional[schemas.FileItem]:
"""
上传文件，并标记完成
"""
@@ -553,9 +560,9 @@ class AliyunHelper:
return None
headers = self.__get_headers(params)
res = RequestUtils(headers=headers, timeout=10).post_res(self.create_folder_file_url, json={
"drive_id": drive_id,
"parent_file_id": parent_file_id,
"name": file_path.name,
"drive_id": fileitem.drive_id,
"parent_file_id": fileitem.parent_fileid,
"name": path.name,
"check_name_mode": "refuse",
"create_scene": "file_upload",
"type": "file",
@@ -564,7 +571,7 @@ class AliyunHelper:
"part_number": 1
}
],
"size": file_path.stat().st_size
"size": path.stat().st_size
})
if not res:
self.__handle_error(res, "创建文件")
@@ -579,7 +586,7 @@ class AliyunHelper:
parent_fileid=result.get("parent_file_id"),
type="file",
name=result.get("file_name"),
path=f"{file_path.parent}/{result.get('file_name')}"
path=f"{fileitem.path}{result.get('file_name')}"
)
file_id = result.get("file_id")
upload_id = result.get("upload_id")
@@ -593,13 +600,13 @@ class AliyunHelper:
"User-Agent": settings.USER_AGENT,
"Referer": "https://www.alipan.com/",
"Accept": "*/*",
}).put_res(upload_url, data=file_path.read_bytes())
}).put_res(upload_url, data=path.read_bytes())
if not res:
self.__handle_error(res, "上传文件")
return None
# 标记文件上传完毕
res = RequestUtils(headers=headers, timeout=10).post_res(self.upload_file_complete_url, json={
"drive_id": drive_id,
"drive_id": fileitem.drive_id,
"file_id": file_id,
"upload_id": upload_id
})
@@ -613,7 +620,7 @@ class AliyunHelper:
parent_fileid=result.get("parent_file_id"),
type="file",
name=result.get("name"),
path=f"{file_path.parent}/{result.get('name')}",
path=f"{fileitem.path}{result.get('name')}",
)
else:
logger.warn("上传文件失败：无法获取上传地址！")

200
app/modules/filetransfer/storage/local.py Normal file
View File

@@ -0,0 +1,200 @@
import shutil
from pathlib import Path
from typing import Optional, List
from starlette.responses import FileResponse, Response
from app import schemas
from app.log import logger
from app.modules.filetransfer.storage import StorageBase
from app.utils.system import SystemUtils
class LocalStorage(StorageBase):
"""
本地文件操作
"""
def check(self) -> bool:
"""
检查存储是否可用
"""
return True
def list(self, fileitem: schemas.FileItem) -> Optional[List[schemas.FileItem]]:
"""
浏览文件
"""
# 返回结果
ret_items = []
path = fileitem.path
if not fileitem.path or fileitem.path == "/":
if SystemUtils.is_windows():
partitions = SystemUtils.get_windows_drives() or ["C:/"]
for partition in partitions:
ret_items.append(schemas.FileItem(
type="dir",
path=partition + "/",
name=partition,
basename=partition
))
return ret_items
else:
path = "/"
else:
if SystemUtils.is_windows():
path = path.lstrip("/")
elif not path.startswith("/"):
path = "/" + path
# 遍历目录
path_obj = Path(path)
if not path_obj.exists():
logger.warn(f"目录不存在：{path}")
return []
# 如果是文件
if path_obj.is_file():
ret_items.append(schemas.FileItem(
type="file",
path=str(path_obj).replace("\\", "/"),
name=path_obj.name,
basename=path_obj.stem,
extension=path_obj.suffix[1:],
size=path_obj.stat().st_size,
modify_time=path_obj.stat().st_mtime,
))
return ret_items
# 扁历所有目录
for item in SystemUtils.list_sub_directory(path_obj):
ret_items.append(schemas.FileItem(
type="dir",
path=str(item).replace("\\", "/") + "/",
name=item.name,
basename=item.stem,
modify_time=item.stat().st_mtime,
))
# 遍历所有文件，不含子目录
for item in SystemUtils.list_sub_all(path_obj):
ret_items.append(schemas.FileItem(
type="file",
path=str(item).replace("\\", "/"),
name=item.name,
basename=item.stem,
extension=item.suffix[1:],
size=item.stat().st_size,
modify_time=item.stat().st_mtime,
))
return ret_items
def create_folder(self, fileitem: schemas.FileItem, name: str) -> Optional[schemas.FileItem]:
"""
创建目录
"""
if not fileitem.path:
return None
path_obj = Path(fileitem.path) / name
if path_obj.exists():
return None
path_obj.mkdir(parents=True, exist_ok=True)
return schemas.FileItem(
type="dir",
path=str(path_obj).replace("\\", "/") + "/",
name=name,
basename=name,
modify_time=path_obj.stat().st_mtime,
)
def detail(self, fileitm: schemas.FileItem) -> Optional[schemas.FileItem]:
"""
获取文件详情
"""
path_obj = Path(fileitm.path)
return schemas.FileItem(
type="file",
path=str(path_obj).replace("\\", "/"),
name=path_obj.name,
basename=path_obj.stem,
extension=path_obj.suffix[1:],
size=path_obj.stat().st_size,
modify_time=path_obj.stat().st_mtime,
)
def delete(self, fileitem: schemas.FileItem) -> bool:
"""
删除文件
"""
if not fileitem.path:
return False
path_obj = Path(fileitem.path)
if not path_obj.exists():
return False
if path_obj.is_file():
path_obj.unlink()
else:
shutil.rmtree(path_obj, ignore_errors=True)
return True
def rename(self, fileitem: schemas.FileItem, name: str) -> bool:
"""
重命名文件
"""
path_obj = Path(fileitem.path)
if not path_obj.exists():
return False
path_obj.rename(path_obj.parent / name)
def download(self, fileitem: schemas.FileItem) -> Optional[Response]:
"""
下载文件
"""
if not fileitem.path:
return None
path_obj = Path(fileitem.path)
if not path_obj.exists():
return None
if path_obj.is_file():
# 做为文件流式下载
return FileResponse(path_obj)
else:
# 做为压缩包下载
shutil.make_archive(base_name=path_obj.stem, format="zip", root_dir=path_obj)
reponse = Response(content=path_obj.read_bytes(), media_type="application/zip")
# 删除压缩包
Path(f"{path_obj.stem}.zip").unlink()
return reponse
def move(self, fileitem: schemas.FileItem, target_dir: schemas.FileItem) -> bool:
"""
移动文件
"""
if not fileitem.path or not target_dir.path:
return False
path_obj = Path(fileitem.path)
target_obj = Path(target_dir.path)
if not path_obj.exists() or not target_obj.exists():
return False
path_obj.rename(target_obj / path_obj.name)
return True
def upload(self, fileitem: schemas.FileItem, path: Path) -> Optional[schemas.FileItem]:
"""
上传文件
"""
if not fileitem.path:
return None
path_obj = Path(fileitem.path)
if not path_obj.exists():
return None
shutil.copy(path, path_obj / path.name)
return schemas.FileItem(
type="file",
path=str(path_obj / path.name).replace("\\", "/"),
name=path.name,
basename=path.stem,
extension=path.suffix[1:],
size=path.stat().st_size,
modify_time=path.stat().st_mtime,
)

53
app/helper/u115.py → app/modules/filetransfer/storage/u115.py
View File

@@ -10,11 +10,12 @@ from py115.types import LoginTarget, QrcodeSession, QrcodeStatus, Credential, Do
from app import schemas
from app.db.systemconfig_oper import SystemConfigOper
from app.log import logger
from app.modules.filetransfer.storage import StorageBase
from app.schemas.types import SystemConfigKey
from app.utils.singleton import Singleton
class U115Helper(metaclass=Singleton):
class U115Pan(StorageBase, metaclass=Singleton):
"""
115相关操作
"""
@@ -138,19 +139,25 @@ class U115Helper(metaclass=Singleton):
logger.error(f"获取115存储空间失败：{str(e)}")
return None
def list(self, parent_file_id: str = '0', path: str = "/") -> Optional[List[schemas.FileItem]]:
def check(self) -> bool:
"""
检查存储是否可用
"""
pass
def list(self, fileitem: schemas.FileItem) -> Optional[List[schemas.FileItem]]:
"""
浏览文件
"""
if not self.__init_cloud():
return None
try:
items = self.cloud.storage().list(dir_id=parent_file_id)
items = self.cloud.storage().list(dir_id=fileitem.parent_fileid)
return [schemas.FileItem(
fileid=item.file_id,
parent_fileid=item.parent_id,
type="dir" if item.is_dir else "file",
path=f"{path}{item.name}" + ("/" if item.is_dir else ""),
path=f"{fileitem.path}{item.name}" + ("/" if item.is_dir else ""),
name=item.name,
size=item.size,
extension=Path(item.name).suffix[1:],
@@ -161,19 +168,19 @@ class U115Helper(metaclass=Singleton):
logger.error(f"浏览115文件失败：{str(e)}")
return None
def create_folder(self, parent_file_id: str, name: str, path: str = "/") -> Optional[schemas.FileItem]:
def create_folder(self, fileitem: schemas.FileItem, name: str) -> Optional[schemas.FileItem]:
"""
创建目录
"""
if not self.__init_cloud():
return None
try:
result = self.cloud.storage().make_dir(parent_file_id, name)
result = self.cloud.storage().make_dir(fileitem.parent_fileid, name)
return schemas.FileItem(
fileid=result.file_id,
parent_fileid=result.parent_id,
type="dir",
path=f"{path}{name}/",
path=f"{fileitem.path}{name}/",
name=name,
modify_time=result.modified_time.timestamp() if result.modified_time else 0,
pickcode=result.pickcode
@@ -182,65 +189,71 @@ class U115Helper(metaclass=Singleton):
logger.error(f"创建115目录失败：{str(e)}")
return None
def delete(self, file_id: str) -> bool:
def detail(self, fileitm: schemas.FileItem) -> Optional[schemas.FileItem]:
"""
获取文件详情
"""
pass
def delete(self, fileitem: schemas.FileItem) -> bool:
"""
删除文件
"""
if not self.__init_cloud():
return False
try:
self.cloud.storage().delete(file_id)
self.cloud.storage().delete(fileitem.fileid)
return True
except Exception as e:
logger.error(f"删除115文件失败：{str(e)}")
return False
def rename(self, file_id: str, name: str) -> bool:
def rename(self, fileitem: schemas.FileItem, name: str) -> bool:
"""
重命名文件
"""
if not self.__init_cloud():
return False
try:
self.cloud.storage().rename(file_id, name)
self.cloud.storage().rename(fileitem.fileid, name)
return True
except Exception as e:
logger.error(f"重命名115文件失败：{str(e)}")
return False
def download(self, pickcode: str) -> Optional[DownloadTicket]:
def download(self, fileitem: schemas.FileItem) -> Optional[DownloadTicket]:
"""
获取下载链接
"""
if not self.__init_cloud():
return None
try:
return self.cloud.storage().request_download(pickcode)
return self.cloud.storage().request_download(fileitem.pickcode)
except Exception as e:
logger.error(f"115下载失败：{str(e)}")
return None
def move(self, file_id: str, target_id: str) -> bool:
def move(self, fileitem: schemas.FileItem, target_dir: schemas.FileItem) -> bool:
"""
移动文件
"""
if not self.__init_cloud():
return False
try:
self.cloud.storage().move(file_id, target_id)
self.cloud.storage().move(fileitem.fileid, target_dir.fileid)
return True
except Exception as e:
logger.error(f"移动115文件失败：{str(e)}")
return False
def upload(self, parent_file_id: str, file_path: Path) -> Optional[schemas.FileItem]:
def upload(self, fileitem: schemas.FileItem, path: Path) -> Optional[schemas.FileItem]:
"""
上传文件
"""
if not self.__init_cloud():
return None
try:
ticket = self.cloud.storage().request_upload(dir_id=parent_file_id, file_path=str(file_path))
ticket = self.cloud.storage().request_upload(dir_id=fileitem.fileid, file_path=str(path))
if ticket is None:
logger.warn(f"115请求上传出错")
return None
@@ -256,7 +269,7 @@ class U115Helper(metaclass=Singleton):
)
por = bucket.put_object_from_file(
key=ticket.object_key,
filename=str(file_path),
filename=str(path),
headers=ticket.headers,
)
result = por.resp.response.json()
@@ -265,10 +278,10 @@ class U115Helper(metaclass=Singleton):
logger.info(f"115上传文件成功：{fileitem}")
return schemas.FileItem(
fileid=fileitem.get('file_id'),
parent_fileid=parent_file_id,
parent_fileid=fileitem.fileid,
type="file",
name=fileitem.get('file_name'),
path=f"{file_path / fileitem.get('file_name')}",
path=f"{fileitem.path}{fileitem.get('file_name')}",
size=fileitem.get('file_size'),
extension=Path(fileitem.get('file_name')).suffix[1:],
pickcode=fileitem.get('pickcode')

13
app/modules/indexer/parser/__init__.py
View File

@@ -14,8 +14,6 @@ from app.log import logger
from app.utils.http import RequestUtils
from app.utils.site import SiteUtils
SITE_BASE_ORDER = 1000
# 站点框架
class SiteSchema(Enum):
@@ -39,8 +37,6 @@ class SiteSchema(Enum):
class SiteParserBase(metaclass=ABCMeta):
# 站点模版
schema = SiteSchema.NexusPhp
# 站点解析时判断顺序，值越小越先解析
order = SITE_BASE_ORDER
# 请求模式 cookie/apikey
request_mode = "cookie"
@@ -155,15 +151,6 @@ class SiteParserBase(metaclass=ABCMeta):
"""
return self.schema
@classmethod
def match(cls, html_text: str) -> bool:
"""
是否匹配当前解析模型
:param html_text: 站点首页html
:return: 是否匹配
"""
pass
def parse(self):
"""
解析站点信息

11
app/modules/indexer/parser/discuz.py
View File

@@ -4,21 +4,12 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class DiscuzUserInfo(SiteParserBase):
schema = SiteSchema.DiscuzX
order = SITE_BASE_ORDER + 10
@classmethod
def match(cls, html_text: str) -> bool:
html = etree.HTML(html_text)
if not html:
return False
printable_text = html.xpath("string(.)") if html else ""
return 'Powered by Discuz!' in printable_text
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)

12
app/modules/indexer/parser/file_list.py
View File

@@ -4,22 +4,12 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class FileListSiteUserInfo(SiteParserBase):
schema = SiteSchema.FileList
order = SITE_BASE_ORDER + 50
@classmethod
def match(cls, html_text: str) -> bool:
html = etree.HTML(html_text)
if not html:
return False
printable_text = html.xpath("string(.)") if html else ""
return 'Powered by FileList' in printable_text
def _parse_site_page(self, html_text: str):
html_text = self._prepare_html_text(html_text)

13
app/modules/indexer/parser/gazelle.py
View File

@@ -4,23 +4,12 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class GazelleSiteUserInfo(SiteParserBase):
schema = SiteSchema.Gazelle
order = SITE_BASE_ORDER
@classmethod
def match(cls, html_text: str) -> bool:
html = etree.HTML(html_text)
if not html:
return False
printable_text = html.xpath("string(.)") if html else ""
return "Powered by Gazelle" in printable_text or "DIC Music" in printable_text
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)

7
app/modules/indexer/parser/ipt_project.py
View File

@@ -4,17 +4,12 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class IptSiteUserInfo(SiteParserBase):
schema = SiteSchema.Ipt
order = SITE_BASE_ORDER + 35
@classmethod
def match(cls, html_text: str) -> bool:
return 'IPTorrents' in html_text
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)

14
app/modules/indexer/parser/mtorrent.py
View File

@@ -3,16 +3,13 @@ import json
from typing import Optional, Tuple
from urllib.parse import urljoin
from lxml import etree
from app.log import logger
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class MTorrentSiteUserInfo(SiteParserBase):
schema = SiteSchema.MTorrent
order = SITE_BASE_ORDER + 60
request_mode = "apikey"
# 用户级别字典
@@ -37,15 +34,6 @@ class MTorrentSiteUserInfo(SiteParserBase):
"18": "Bet memberStaff",
}
@classmethod
def match(cls, html_text: str) -> bool:
html = etree.HTML(html_text)
if not html:
return False
if html.xpath("//title/text()") and "M-Team" in html.xpath("//title/text()")[0]:
return True
return False
def _parse_site_page(self, html_text: str):
"""
获取站点页面地址

7
app/modules/indexer/parser/nexus_audiences.py
View File

@@ -1,17 +1,12 @@
# -*- coding: utf-8 -*-
from urllib.parse import urljoin
from app.modules.indexer.parser import SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteSchema
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo):
schema = SiteSchema.NexusAudiences
order = SITE_BASE_ORDER + 5
@classmethod
def match(cls, html_text: str) -> bool:
return 'audiences.me' in html_text
def _parse_site_page(self, html_text: str):
super()._parse_site_page(html_text)

7
app/modules/indexer/parser/nexus_hhanclub.py
View File

@@ -3,18 +3,13 @@ import re
from lxml import etree
from app.modules.indexer.parser import SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteSchema
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
from app.utils.string import StringUtils
class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo):
schema = SiteSchema.NexusHhanclub
order = SITE_BASE_ORDER + 20
@classmethod
def match(cls, html_text: str) -> bool:
return 'hhanclub.top' in html_text
def _parse_user_traffic_info(self, html_text):
super()._parse_user_traffic_info(html_text)

12
app/modules/indexer/parser/nexus_php.py
View File

@@ -5,22 +5,12 @@ from typing import Optional
from lxml import etree
from app.log import logger
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class NexusPhpSiteUserInfo(SiteParserBase):
schema = SiteSchema.NexusPhp
order = SITE_BASE_ORDER * 2
@classmethod
def match(cls, html_text: str) -> bool:
"""
默认使用NexusPhp解析
:param html_text:
:return:
"""
return True
def _parse_site_page(self, html_text: str):
html_text = self._prepare_html_text(html_text)

7
app/modules/indexer/parser/nexus_project.py
View File

@@ -1,17 +1,12 @@
# -*- coding: utf-8 -*-
import re
from app.modules.indexer.parser import SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteSchema
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
class NexusProjectSiteUserInfo(NexusPhpSiteUserInfo):
schema = SiteSchema.NexusProject
order = SITE_BASE_ORDER + 25
@classmethod
def match(cls, html_text: str) -> bool:
return 'Nexus Project' in html_text
def _parse_site_page(self, html_text: str):
html_text = self._prepare_html_text(html_text)

14
app/modules/indexer/parser/nexus_rabbit.py
View File

@@ -2,25 +2,13 @@
import json
from typing import Optional
from lxml import etree
from app.log import logger
from app.modules.indexer.parser import SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteSchema
from app.modules.indexer.parser.nexus_php import NexusPhpSiteUserInfo
class NexusRabbitSiteUserInfo(NexusPhpSiteUserInfo):
schema = SiteSchema.NexusRabbit
order = SITE_BASE_ORDER + 5
@classmethod
def match(cls, html_text: str) -> bool:
html = etree.HTML(html_text)
if not html:
return False
printable_text = html.xpath("string(.)") if html else ""
return 'Style by Rabbit' in printable_text
def _parse_site_page(self, html_text: str):
super()._parse_site_page(html_text)

7
app/modules/indexer/parser/small_horse.py
View File

@@ -4,17 +4,12 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class SmallHorseSiteUserInfo(SiteParserBase):
schema = SiteSchema.SmallHorse
order = SITE_BASE_ORDER + 30
@classmethod
def match(cls, html_text: str) -> bool:
return 'Small Horse' in html_text
def _parse_site_page(self, html_text: str):
html_text = self._prepare_html_text(html_text)

7
app/modules/indexer/parser/tnode.py
View File

@@ -3,17 +3,12 @@ import json
import re
from typing import Optional
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class TNodeSiteUserInfo(SiteParserBase):
schema = SiteSchema.TNode
order = SITE_BASE_ORDER + 60
@classmethod
def match(cls, html_text: str) -> bool:
return 'Powered By TNode' in html_text
def _parse_site_page(self, html_text: str):
html_text = self._prepare_html_text(html_text)

7
app/modules/indexer/parser/torrent_leech.py
View File

@@ -4,17 +4,12 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class TorrentLeechSiteUserInfo(SiteParserBase):
schema = SiteSchema.TorrentLeech
order = SITE_BASE_ORDER + 40
@classmethod
def match(cls, html_text: str) -> bool:
return 'TorrentLeech' in html_text
def _parse_site_page(self, html_text: str):
html_text = self._prepare_html_text(html_text)

7
app/modules/indexer/parser/unit3d.py
View File

@@ -4,17 +4,12 @@ from typing import Optional
from lxml import etree
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class Unit3dSiteUserInfo(SiteParserBase):
schema = SiteSchema.Unit3d
order = SITE_BASE_ORDER + 15
@classmethod
def match(cls, html_text: str) -> bool:
return "unit3d.js" in html_text
def _parse_user_base_info(self, html_text: str):
html_text = self._prepare_html_text(html_text)

7
app/modules/indexer/parser/yema.py
View File

@@ -2,17 +2,12 @@
import json
from typing import Optional, Tuple
from app.modules.indexer.parser import SiteParserBase, SiteSchema, SITE_BASE_ORDER
from app.modules.indexer.parser import SiteParserBase, SiteSchema
from app.utils.string import StringUtils
class TYemaSiteUserInfo(SiteParserBase):
schema = SiteSchema.Yema
order = SITE_BASE_ORDER + 60
@classmethod
def match(cls, html_text: str) -> bool:
return '<title>YemaPT</title>' in html_text
def _parse_site_page(self, html_text: str):
"""