diff --git a/backend/requirements.txt b/backend/requirements.txt index 02ed2157..d36de2eb 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -9,7 +9,7 @@ idna==3.4 pydantic~=1.10 PySocks==1.7.1 qbittorrent-api==2023.9.53 -requests==2.31.0 +httpx[http2,socks]==0.25.0 six==1.16.0 sniffio==1.3.0 soupsieve==2.4.1 diff --git a/backend/src/module/network/request_contents.py b/backend/src/module/network/request_contents.py index 05abca02..b307de2a 100644 --- a/backend/src/module/network/request_contents.py +++ b/backend/src/module/network/request_contents.py @@ -12,14 +12,14 @@ logger = logging.getLogger(__name__) class RequestContent(RequestURL): - def get_torrents( + async def get_torrents( self, _url: str, _filter: str = "|".join(settings.rss_parser.filter), limit: int = None, retry: int = 3, ) -> list[Torrent]: - soup = self.get_xml(_url, retry) + soup = await self.get_xml(_url, retry) if soup: torrent_titles, torrent_urls, torrent_homepage = rss_parser(soup) torrents: list[Torrent] = [] @@ -38,38 +38,40 @@ class RequestContent(RequestURL): logger.warning(f"[Network] Failed to get torrents: {_url}") return [] - def get_xml(self, _url, retry: int = 3) -> xml.etree.ElementTree.Element: - req = self.get_url(_url, retry) + async def get_xml(self, _url, retry: int = 3) -> xml.etree.ElementTree.Element: + req = await self.get_url(_url, retry) if req: return xml.etree.ElementTree.fromstring(req.text) # API JSON - def get_json(self, _url) -> dict: - req = self.get_url(_url) + async def get_json(self, _url) -> dict: + req = await self.get_url(_url) if req: return req.json() - def post_json(self, _url, data: dict) -> dict: - return self.post_url(_url, data).json() + async def post_json(self, _url, data: dict) -> dict: + return await self.post_url(_url, data) - def post_data(self, _url, data: dict) -> dict: - return self.post_url(_url, data) + async def post_data(self, _url, data: dict) -> dict: + return await self.post_json(_url, data) - def post_files(self, _url, data: dict, files: dict) -> dict: - return self.post_form(_url, data, files) + async def post_files(self, _url, data: dict, files: dict) -> dict: + return await self.post_form(_url, data, files) - def get_html(self, _url): - return self.get_url(_url).text + async def get_html(self, _url): + req = await self.get_url(_url) + if req: + return req.text - def get_content(self, _url): - req = self.get_url(_url) + async def get_content(self, _url): + req = await self.get_url(_url) if req: return req.content - def check_connection(self, _url): - return self.check_url(_url) + async def check_connection(self, _url): + return await self.check_url(_url) - def get_rss_title(self, _url): - soup = self.get_xml(_url) + async def get_rss_title(self, _url): + soup = await self.get_xml(_url) if soup: return soup.find("./channel/title").text diff --git a/backend/src/module/network/request_url.py b/backend/src/module/network/request_url.py index 0b85e77e..11191739 100644 --- a/backend/src/module/network/request_url.py +++ b/backend/src/module/network/request_url.py @@ -1,9 +1,7 @@ import logging -import socket import time -import requests -import socks +import httpx from module.conf import settings @@ -13,40 +11,15 @@ logger = logging.getLogger(__name__) class RequestURL: def __init__(self): self.header = {"user-agent": "Mozilla/5.0", "Accept": "application/xml"} - self._socks5_proxy = False - def get_url(self, url, retry=3): + async def get_url(self, url, retry=3): try_time = 0 while True: try: - req = self.session.get(url=url, headers=self.header, timeout=5) - logger.debug(f"[Network] Successfully connected to {url}. Status: {req.status_code}") + req = await self.client.get(url=url, headers=self.header, timeout=5) req.raise_for_status() return req - except requests.RequestException: - logger.debug( - f"[Network] Cannot connect to {url}. Wait for 5 seconds." - ) - try_time += 1 - if try_time >= retry: - break - time.sleep(5) - except Exception as e: - logger.debug(e) - break - logger.error(f"[Network] Unable to connect to {url}, Please check your network settings") - return None - - def post_url(self, url: str, data: dict, retry=3): - try_time = 0 - while True: - try: - req = self.session.post( - url=url, headers=self.header, data=data, timeout=5 - ) - req.raise_for_status() - return req - except requests.RequestException: + except httpx.RequestError: logger.warning( f"[Network] Cannot connect to {url}. Wait for 5 seconds." ) @@ -61,64 +34,69 @@ class RequestURL: logger.warning("[Network] Please check DNS/Connection settings") return None - def check_url(self, url: str): + async def post_url(self, url: str, data: dict, retry=3): + try_time = 0 + while True: + try: + req = await self.client.post( + url=url, headers=self.header, data=data, timeout=5 + ) + req.raise_for_status() + return req + except httpx.RequestError: + logger.warning( + f"[Network] Cannot connect to {url}. Wait for 5 seconds." + ) + try_time += 1 + if try_time >= retry: + break + time.sleep(5) + except Exception as e: + logger.debug(e) + break + logger.error(f"[Network] Failed connecting to {url}") + logger.warning("[Network] Please check DNS/Connection settings") + return None + + async def check_url(self, url: str): if "://" not in url: url = f"http://{url}" try: - req = requests.head(url=url, headers=self.header, timeout=5) + req = await self.client.get(url=url, headers=self.header, timeout=5) req.raise_for_status() return True - except requests.RequestException: + except httpx.RequestError: logger.debug(f"[Network] Cannot connect to {url}.") return False - def post_form(self, url: str, data: dict, files): + async def post_form(self, url: str, data: dict, files): try: - req = self.session.post( + req = await self.client.post( url=url, headers=self.header, data=data, files=files, timeout=5 ) req.raise_for_status() return req - except requests.RequestException: + except httpx.RequestError: logger.warning(f"[Network] Cannot connect to {url}.") return None - def __enter__(self): - self.session = requests.Session() + async def __aenter__(self): + proxy = None if settings.proxy.enable: + auth = f"{settings.proxy.username}:{settings.proxy.password}@" \ + if settings.proxy.username else \ + "" if "http" in settings.proxy.type: - if settings.proxy.username: - username=settings.proxy.username - password=settings.proxy.password - url = f"http://{username}:{password}@{settings.proxy.host}:{settings.proxy.port}" - self.session.proxies = { - "http": url, - "https": url, - } - else: - url = f"http://{settings.proxy.host}:{settings.proxy.port}" - self.session.proxies = { - "http": url, - "https": url, - } + proxy = f"{settings.proxy.type}://{auth}{settings.proxy.host}:{settings.proxy.port}" elif settings.proxy.type == "socks5": - self._socks5_proxy = True - socks.set_default_proxy( - socks.SOCKS5, - addr=settings.proxy.host, - port=settings.proxy.port, - rdns=True, - username=settings.proxy.username, - password=settings.proxy.password, - ) - socket.socket = socks.socksocket + proxy = f"socks5://{auth}{settings.proxy.host}:{settings.proxy.port}" else: logger.error(f"[Network] Unsupported proxy type: {settings.proxy.type}") + self.client = httpx.AsyncClient( + http2=True, + proxies=proxy, + ) return self - def __exit__(self, exc_type, exc_val, exc_tb): - if self._socks5_proxy: - socks.set_default_proxy() - socket.socket = socks.socksocket - self._socks5_proxy = False - self.session.close() + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.client.aclose()