diff --git a/backend/src/module/network/proxy.py b/backend/src/module/network/proxy.py new file mode 100644 index 00000000..54ff2c9a --- /dev/null +++ b/backend/src/module/network/proxy.py @@ -0,0 +1,16 @@ +from module.conf import settings + + +@property +def set_proxy(): + auth = f"{settings.proxy.username}:{settings.proxy.password}@" \ + if settings.proxy.username else \ + "" + if "http" in settings.proxy.type: + proxy = f"{settings.proxy.type}://{auth}{settings.proxy.host}:{settings.proxy.port}" + elif settings.proxy.type == "socks5": + proxy = f"socks5://{auth}{settings.proxy.host}:{settings.proxy.port}" + else: + proxy = None + logger.error(f"[Network] Unsupported proxy type: {settings.proxy.type}") + return proxy diff --git a/backend/src/module/network/request_contents.py b/backend/src/module/network/request_contents.py index b307de2a..1d7f880c 100644 --- a/backend/src/module/network/request_contents.py +++ b/backend/src/module/network/request_contents.py @@ -11,17 +11,22 @@ from .site import rss_parser logger = logging.getLogger(__name__) +@property +def gen_filter(): + return "|".join(settings.rss.filter) + + class RequestContent(RequestURL): async def get_torrents( self, _url: str, - _filter: str = "|".join(settings.rss_parser.filter), + _filter: str = gen_filter, limit: int = None, retry: int = 3, ) -> list[Torrent]: - soup = await self.get_xml(_url, retry) - if soup: - torrent_titles, torrent_urls, torrent_homepage = rss_parser(soup) + feeds = await self.get_xml(_url, retry) + if feeds: + torrent_titles, torrent_urls, torrent_homepage = rss_parser(feeds) torrents: list[Torrent] = [] for _title, torrent_url, homepage in zip( torrent_titles, torrent_urls, torrent_homepage @@ -30,12 +35,9 @@ class RequestContent(RequestURL): torrents.append( Torrent(name=_title, url=torrent_url, homepage=homepage) ) - if isinstance(limit, int): - if len(torrents) >= limit: - break - return torrents + return torrents if limit is None else torrents[:limit] else: - logger.warning(f"[Network] Failed to get torrents: {_url}") + logger.error(f"[Network] Torrents list is empty: {_url}") return [] async def get_xml(self, _url, retry: int = 3) -> xml.etree.ElementTree.Element: @@ -49,14 +51,8 @@ class RequestContent(RequestURL): if req: return req.json() - async def post_json(self, _url, data: dict) -> dict: - return await self.post_url(_url, data) - - async def post_data(self, _url, data: dict) -> dict: - return await self.post_json(_url, data) - - async def post_files(self, _url, data: dict, files: dict) -> dict: - return await self.post_form(_url, data, files) + async def post_data(self, _url, data: dict, files: dict[str, bytes]) -> dict: + return await self.post_url(_url, data, files) async def get_html(self, _url): req = await self.get_url(_url) diff --git a/backend/src/module/network/request_url.py b/backend/src/module/network/request_url.py index 79520613..26c6d1d9 100644 --- a/backend/src/module/network/request_url.py +++ b/backend/src/module/network/request_url.py @@ -1,9 +1,10 @@ import asyncio import logging -import time import httpx +from .proxy import set_proxy + from module.conf import settings logger = logging.getLogger(__name__) @@ -23,13 +24,12 @@ def retry_async(times=3): except httpx.RequestError: if _ < times - 1: await asyncio.sleep(5) # 延迟5秒后重试 - logger.warning( + logger.debug( f"[Network] Cannot connect to {url}. Wait for 5 seconds." ) except Exception as e: logger.debug(e) - logger.error(f"[Network] Failed connecting to {url}") - logger.warning("[Network] Please check DNS/Connection settings") + logger.error(f"[Network] Cannot connect to {url}") break return None @@ -41,6 +41,7 @@ def retry_async(times=3): class RequestURL: def __init__(self): self.header = {"user-agent": "Mozilla/5.0", "Accept": "application/xml"} + self.proxy = set_proxy if settings.proxy.enable else None @retry_async() async def get_url(self, url): @@ -49,8 +50,8 @@ class RequestURL: return req @retry_async() - async def post_url(self, url: str, data: dict): - req = await self.client.post(url=url, data=data) + async def post_url(self, url: str, data: dict, files: dict[str, bytes] = None): + req = await self.client.post(url=url, data=data, files=files) req.raise_for_status() return req @@ -65,31 +66,9 @@ class RequestURL: logger.debug(f"[Network] Cannot connect to {url}.") return False - async def post_form(self, url: str, data: dict, files): - try: - req = await self.client.post(url=url, data=data, files=files) - req.raise_for_status() - return req - except httpx.RequestError: - logger.warning(f"[Network] Cannot connect to {url}.") - return None - async def __aenter__(self): - proxy = None - if settings.proxy.enable: - auth = ( - f"{settings.proxy.username}:{settings.proxy.password}@" - if settings.proxy.username - else "" - ) - if "http" in settings.proxy.type: - proxy = f"{settings.proxy.type}://{auth}{settings.proxy.host}:{settings.proxy.port}" - elif settings.proxy.type == "socks5": - proxy = f"socks5://{auth}{settings.proxy.host}:{settings.proxy.port}" - else: - logger.error(f"[Network] Unsupported proxy type: {settings.proxy.type}") self.client = httpx.AsyncClient( - http2=True, proxies=proxy, headers=self.header, timeout=5 + http2=True, proxies=self.proxy, headers=self.header, timeout=5 ) return self