From ee95f821aff4b23dfbee352fdce815eacbc617b2 Mon Sep 17 00:00:00 2001 From: EstrellaXD Date: Tue, 2 Jan 2024 22:05:35 +0800 Subject: [PATCH] remove: retry decorator, fix test bug. --- .../src/module/network/request_contents.py | 8 +- backend/src/module/network/request_url.py | 74 +++++++++---------- backend/src/test/test_rss_engine.py | 2 +- pyrightconfig.json | 8 -- 4 files changed, 39 insertions(+), 53 deletions(-) delete mode 100644 pyrightconfig.json diff --git a/backend/src/module/network/request_contents.py b/backend/src/module/network/request_contents.py index 1d7f880c..13909e6f 100644 --- a/backend/src/module/network/request_contents.py +++ b/backend/src/module/network/request_contents.py @@ -11,20 +11,16 @@ from .site import rss_parser logger = logging.getLogger(__name__) -@property -def gen_filter(): - return "|".join(settings.rss.filter) - - class RequestContent(RequestURL): async def get_torrents( self, _url: str, - _filter: str = gen_filter, + _filter: str = None, limit: int = None, retry: int = 3, ) -> list[Torrent]: feeds = await self.get_xml(_url, retry) + _filter = _filter if _filter else "|".join(settings.rss_parser.filter) if feeds: torrent_titles, torrent_urls, torrent_homepage = rss_parser(feeds) torrents: list[Torrent] = [] diff --git a/backend/src/module/network/request_url.py b/backend/src/module/network/request_url.py index 26c6d1d9..4dba690b 100644 --- a/backend/src/module/network/request_url.py +++ b/backend/src/module/network/request_url.py @@ -10,50 +10,48 @@ from module.conf import settings logger = logging.getLogger(__name__) -def retry_async(times=3): - def decorator(func): - async def wrapper(*args, **kwargs): - url = kwargs.get("url", None) - if url is None: - url = args[0] - for _ in range(times): - try: - resp = await func(*args, **kwargs) - logger.debug(f"[Network] Successfully connected to {url}") - return resp - except httpx.RequestError: - if _ < times - 1: - await asyncio.sleep(5) # 延迟5秒后重试 - logger.debug( - f"[Network] Cannot connect to {url}. Wait for 5 seconds." - ) - except Exception as e: - logger.debug(e) - logger.error(f"[Network] Cannot connect to {url}") - break - return None - - return wrapper - - return decorator - - class RequestURL: def __init__(self): self.header = {"user-agent": "Mozilla/5.0", "Accept": "application/xml"} self.proxy = set_proxy if settings.proxy.enable else None - @retry_async() - async def get_url(self, url): - req = await self.client.get(url=url) - req.raise_for_status() - return req + async def get_url(self, url, retry=3): + for _ in range(retry): + try: + req = await self.client.get(url=url) + return req + except httpx.RequestError: + logger.debug( + f"[Network] Cannot connect to {url}. Wait for 5 seconds." + ) + except httpx.TimeoutException: + logger.debug( + f"[Network] Timeout. Cannot connect to {url}. Wait for 5 seconds." + ) + except Exception as e: + logger.debug(e) + logger.error(f"[Network] Cannot connect to {url}") + break + await asyncio.sleep(5) - @retry_async() - async def post_url(self, url: str, data: dict, files: dict[str, bytes] = None): - req = await self.client.post(url=url, data=data, files=files) - req.raise_for_status() - return req + async def post_url(self, url: str, data: dict, files: dict[str, bytes] = None, retry: int = 3): + for _ in range(retry): + try: + req = await self.client.post(url=url, data=data, files=files) + return req + except httpx.RequestError: + logger.debug( + f"[Network] Cannot connect to {url}. Wait for 5 seconds." + ) + except httpx.TimeoutException: + logger.debug( + f"[Network] Timeout. Cannot connect to {url}. Wait for 5 seconds." + ) + except Exception as e: + logger.debug(e) + logger.error(f"[Network] Cannot connect to {url}") + break + await asyncio.sleep(5) async def check_url(self, url: str): if "://" not in url: diff --git a/backend/src/test/test_rss_engine.py b/backend/src/test/test_rss_engine.py index 549db9ee..2c58e32e 100644 --- a/backend/src/test/test_rss_engine.py +++ b/backend/src/test/test_rss_engine.py @@ -13,7 +13,7 @@ async def test_rss_engine(): assert resp.status result = engine.rss.search_active() - assert result[0].name == "Mikan Project - 无职转生~到了异世界就拿出真本事~" + assert result[1].name == "Mikan Project - 无职转生~到了异世界就拿出真本事~" new_torrents = await engine.pull_rss(result[1]) torrent = new_torrents[0] diff --git a/pyrightconfig.json b/pyrightconfig.json deleted file mode 100644 index 4521b494..00000000 --- a/pyrightconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "pythonPath": "/opt/homebrew/Caskroom/miniforge/base/envs/auto_bangumi/bin/python", - "root": "backend/src", - "venvPath": "/opt/homebrew/Caskroom/miniforge/base/envs", - "venv": "auto_bangumi", - "typeCheckingMode": "basic", - "reportMissingImports": true -}