mirror of
https://github.com/EstrellaXD/Auto_Bangumi.git
synced 2026-03-20 11:57:46 +08:00
41298f2f8e
Database: - Add error handling and per-step version tracking in migrations - Enable SQLite foreign keys via PRAGMA on connect - Fix SQLAlchemy .is_(None) usage, add session.merge() for detached - Batch commit for semantic alias merges - Quote table/field names in fill-null-defaults SQL - Guard against empty user data in migration Parsers: - TMDB: bounded LRU cache (512), asyncio.gather for parallel season fetches, fix season regex \d -> \d+, null-safe year, fix id shadowing - Raw parser: re.escape() for group/prefix regex, None guard on match - OpenAI: handle Pydantic model_dump, catch ValueError Network: - Null-safe get_html() return - Error handling per RSS item in mikan parser - Progressive retry delays (5/15/45/120/300s) with specific exceptions - Platform detection via sys.platform instead of path heuristic - Move filter cache to instance variable Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
84 lines
2.6 KiB
Python
84 lines
2.6 KiB
Python
import logging
|
|
import re
|
|
import xml.etree.ElementTree
|
|
|
|
from module.conf import settings
|
|
from module.models import Torrent
|
|
|
|
from .request_url import RequestURL
|
|
from .site import rss_parser
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class RequestContent(RequestURL):
|
|
async def get_torrents(
|
|
self,
|
|
_url: str,
|
|
_filter: str = None,
|
|
limit: int = None,
|
|
retry: int = 3,
|
|
) -> list[Torrent]:
|
|
soup = await self.get_xml(_url, retry)
|
|
if soup:
|
|
parsed_items = rss_parser(soup)
|
|
torrents: list[Torrent] = []
|
|
if _filter is None:
|
|
_filter = "|".join(settings.rss_parser.filter)
|
|
for _title, torrent_url, homepage in parsed_items:
|
|
if re.search(_filter, _title) is None:
|
|
torrents.append(
|
|
Torrent(name=_title, url=torrent_url, homepage=homepage)
|
|
)
|
|
if isinstance(limit, int):
|
|
if len(torrents) >= limit:
|
|
break
|
|
return torrents
|
|
else:
|
|
logger.warning(f"[Network] Failed to get torrents: {_url}")
|
|
return []
|
|
|
|
async def get_xml(self, _url, retry: int = 3) -> xml.etree.ElementTree.Element:
|
|
req = await self.get_url(_url, retry)
|
|
if req:
|
|
try:
|
|
return xml.etree.ElementTree.fromstring(req.text)
|
|
except xml.etree.ElementTree.ParseError as e:
|
|
logger.warning(f"[Network] Failed to parse XML from {_url}: {e}")
|
|
return None
|
|
|
|
# API JSON
|
|
async def get_json(self, _url) -> dict:
|
|
req = await self.get_url(_url)
|
|
if req:
|
|
return req.json()
|
|
|
|
async def post_json(self, _url, data: dict) -> dict:
|
|
resp = await self.post_url(_url, data)
|
|
return resp.json()
|
|
|
|
async def post_data(self, _url, data: dict):
|
|
return await self.post_url(_url, data)
|
|
|
|
async def post_files(self, _url, data: dict, files: dict):
|
|
return await self.post_form(_url, data, files)
|
|
|
|
async def get_html(self, _url):
|
|
resp = await self.get_url(_url)
|
|
return resp.text if resp else None
|
|
|
|
async def get_content(self, _url):
|
|
req = await self.get_url(_url)
|
|
if req:
|
|
return req.content
|
|
logger.warning(f"[Network] Failed to get content from {_url}")
|
|
return None
|
|
|
|
async def check_connection(self, _url):
|
|
return await self.check_url(_url)
|
|
|
|
async def get_rss_title(self, _url):
|
|
soup = await self.get_xml(_url)
|
|
if soup:
|
|
return soup.find("./channel/title").text
|