feat: new async rss poller.

This commit is contained in:
EstrellaXD
2024-01-03 21:18:49 +08:00
parent b1707e6276
commit 62d99eb24e
3 changed files with 13 additions and 9 deletions

View File

@@ -21,7 +21,7 @@ class RSSThread(ProgramStatus):
async def __loop_mission(self):
async with RSSEngine() as engine:
await engine.rss_checker(self.analyser, self.stop_event)
await engine.rss_poller(self.analyser, self.stop_event)
def rss_loop(self):
asyncio.run(self.__loop_mission())

View File

@@ -17,16 +17,17 @@ class RSSEngine:
self._to_refresh = False
self.db_status = False
async def rss_checker(self, callback: Callable = None):
while 1:
async def rss_poller(self, callback: Callable = None):
# Main RSS Loop
while True:
with Database() as database:
rss_items = database.rss.search_active()
if rss_items:
tasks = []
for item in rss_items:
tasks.append(self.pull_rss(item, database, callback))
await asyncio.gather(*tasks)
await asyncio.sleep(settings.rss.interval)
await asyncio.gather(*tasks)
await asyncio.sleep(settings.program.rss_time)
@staticmethod
async def _get_torrents(rss: RSSItem) -> list[Torrent]:
@@ -38,12 +39,13 @@ class RSSEngine:
return torrents
async def pull_rss(
self, rss_item: RSSItem, database: Database, callback: Callable = None, **kwargs
self, rss_item: RSSItem, database: Database = None, callback: Callable = None, **kwargs
) -> list[Torrent]:
torrents = await self._get_torrents(rss_item)
new_torrents = database.torrent.check_new(torrents)
if callback:
await callback(rss_item, new_torrents, **kwargs)
database.torrent.add_all(new_torrents)
return new_torrents
@staticmethod
@@ -118,3 +120,5 @@ class RSSEngine:
@staticmethod
def __connect_database():
return Database(engine)

View File

@@ -2,13 +2,13 @@ import re
from urllib3.util import parse_url
from module.rss import RSSEngine
from module.rss import RSSManager
from module.utils import save_image
from module.network import RequestContent
async def from_30_to_31():
async with RSSEngine() as db:
async with RSSManager() as db:
db.migrate()
# Update poster link
bangumis = db.bangumi.search_all()
@@ -33,7 +33,7 @@ async def from_30_to_31():
async def cache_image():
async with RSSEngine() as db, RequestContent() as req:
async with RSSManager() as db, RequestContent() as req:
bangumis = db.bangumi.search_all()
for bangumi in bangumis:
if bangumi.poster_link: