From d336bcbf1f60d40412e03322db377bd727468110 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Tue, 8 Jul 2025 11:00:38 +0800 Subject: [PATCH] fix etree --- app/chain/site.py | 22 +- app/core/meta/streamingplatform.py | 1 - app/helper/cookie.py | 121 +++--- app/modules/indexer/parser/discuz.py | 149 +++---- app/modules/indexer/parser/file_list.py | 107 ++--- app/modules/indexer/parser/gazelle.py | 173 ++++---- app/modules/indexer/parser/ipt_project.py | 104 ++--- app/modules/indexer/parser/nexus_audiences.py | 22 +- app/modules/indexer/parser/nexus_hhanclub.py | 48 ++- app/modules/indexer/parser/nexus_php.py | 371 ++++++++++-------- app/modules/indexer/parser/nexus_rabbit.py | 86 ++-- app/modules/indexer/parser/small_horse.py | 101 ++--- app/modules/indexer/parser/torrent_leech.py | 90 +++-- app/modules/indexer/parser/unit3d.py | 129 +++--- app/modules/subtitle/__init__.py | 32 +- app/modules/themoviedb/tmdbapi.py | 3 + app/utils/site.py | 87 ++-- 17 files changed, 897 insertions(+), 749 deletions(-) diff --git a/app/chain/site.py b/app/chain/site.py index 75a8d2e8..86e72b8e 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -271,16 +271,20 @@ class SiteChain(ChainBase): logger.error(f"获取站点页面失败：{url}") return favicon_url, None html = etree.HTML(html_text) - if StringUtils.is_valid_html_element(html): - fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href') - if fav_link: - favicon_url = urljoin(url, fav_link[0]) + try: + if StringUtils.is_valid_html_element(html): + fav_link = html.xpath('//head/link[contains(@rel, "icon")]/@href') + if fav_link: + favicon_url = urljoin(url, fav_link[0]) - res = RequestUtils(cookies=cookie, timeout=15, ua=ua).get_res(url=favicon_url) - if res: - return favicon_url, base64.b64encode(res.content).decode() - else: - logger.error(f"获取站点图标失败：{favicon_url}") + res = RequestUtils(cookies=cookie, timeout=15, ua=ua).get_res(url=favicon_url) + if res: + return favicon_url, base64.b64encode(res.content).decode() + else: + logger.error(f"获取站点图标失败：{favicon_url}") + finally: + if html is not None: + del html return favicon_url, None def sync_cookies(self, manual=False) -> Tuple[bool, str]: diff --git a/app/core/meta/streamingplatform.py b/app/core/meta/streamingplatform.py index 450f5ecb..e63b3a1d 100644 --- a/app/core/meta/streamingplatform.py +++ b/app/core/meta/streamingplatform.py @@ -312,4 +312,3 @@ class StreamingPlatforms(metaclass=Singleton): if name is None: return False return name.upper() in self._lookup_cache - diff --git a/app/helper/cookie.py b/app/helper/cookie.py index 0f51c41f..07bc536f 100644 --- a/app/helper/cookie.py +++ b/app/helper/cookie.py @@ -96,53 +96,58 @@ class CookieHelper: return None, None, "获取源码失败" # 查找用户名输入框 html = etree.HTML(html_text) - username_xpath = None - for xpath in self._SITE_LOGIN_XPATH.get("username"): - if html.xpath(xpath): - username_xpath = xpath - break - if not username_xpath: - return None, None, "未找到用户名输入框" - # 查找密码输入框 - password_xpath = None - for xpath in self._SITE_LOGIN_XPATH.get("password"): - if html.xpath(xpath): - password_xpath = xpath - break - if not password_xpath: - return None, None, "未找到密码输入框" - # 处理二步验证码 - otp_code = TwoFactorAuth(two_step_code).get_code() - # 查找二步验证码输入框 - twostep_xpath = None - if otp_code: - for xpath in self._SITE_LOGIN_XPATH.get("twostep"): + try: + username_xpath = None + for xpath in self._SITE_LOGIN_XPATH.get("username"): if html.xpath(xpath): - twostep_xpath = xpath + username_xpath = xpath break - # 查找验证码输入框 - captcha_xpath = None - for xpath in self._SITE_LOGIN_XPATH.get("captcha"): - if html.xpath(xpath): - captcha_xpath = xpath - break - # 查找验证码图片 - captcha_img_url = None - if captcha_xpath: - for xpath in self._SITE_LOGIN_XPATH.get("captcha_img"): + if not username_xpath: + return None, None, "未找到用户名输入框" + # 查找密码输入框 + password_xpath = None + for xpath in self._SITE_LOGIN_XPATH.get("password"): if html.xpath(xpath): - captcha_img_url = html.xpath(xpath)[0] + password_xpath = xpath break - if not captcha_img_url: - return None, None, "未找到验证码图片" - # 查找登录按钮 - submit_xpath = None - for xpath in self._SITE_LOGIN_XPATH.get("submit"): - if html.xpath(xpath): - submit_xpath = xpath - break - if not submit_xpath: - return None, None, "未找到登录按钮" + if not password_xpath: + return None, None, "未找到密码输入框" + # 处理二步验证码 + otp_code = TwoFactorAuth(two_step_code).get_code() + # 查找二步验证码输入框 + twostep_xpath = None + if otp_code: + for xpath in self._SITE_LOGIN_XPATH.get("twostep"): + if html.xpath(xpath): + twostep_xpath = xpath + break + # 查找验证码输入框 + captcha_xpath = None + for xpath in self._SITE_LOGIN_XPATH.get("captcha"): + if html.xpath(xpath): + captcha_xpath = xpath + break + # 查找验证码图片 + captcha_img_url = None + if captcha_xpath: + for xpath in self._SITE_LOGIN_XPATH.get("captcha_img"): + if html.xpath(xpath): + captcha_img_url = html.xpath(xpath)[0] + break + if not captcha_img_url: + return None, None, "未找到验证码图片" + # 查找登录按钮 + submit_xpath = None + for xpath in self._SITE_LOGIN_XPATH.get("submit"): + if html.xpath(xpath): + submit_xpath = xpath + break + if not submit_xpath: + return None, None, "未找到登录按钮" + finally: + if html is not None: + del html + # 点击登录按钮 try: # 等待登录按钮准备好 @@ -185,19 +190,23 @@ class CookieHelper: if not otp_code: return None, None, "需要二次验证码" html = etree.HTML(page.content()) - for xpath in self._SITE_LOGIN_XPATH.get("twostep"): - if html.xpath(xpath): - try: - # 刷新一下 2fa code - otp_code = TwoFactorAuth(two_step_code).get_code() - page.fill(xpath, otp_code) - # 登录按钮 xpath 理论上相同，不再重复查找 - page.click(submit_xpath) - page.wait_for_load_state("networkidle", timeout=30 * 1000) - except Exception as e: - logger.error(f"二次验证码输入失败：{str(e)}") - return None, None, f"二次验证码输入失败：{str(e)}" - break + try: + for xpath in self._SITE_LOGIN_XPATH.get("twostep"): + if html.xpath(xpath): + try: + # 刷新一下 2fa code + otp_code = TwoFactorAuth(two_step_code).get_code() + page.fill(xpath, otp_code) + # 登录按钮 xpath 理论上相同，不再重复查找 + page.click(submit_xpath) + page.wait_for_load_state("networkidle", timeout=30 * 1000) + except Exception as e: + logger.error(f"二次验证码输入失败：{str(e)}") + return None, None, f"二次验证码输入失败：{str(e)}" + break + finally: + if html is not None: + del html # 登录后的源码 html_text = page.content() if not html_text: diff --git a/app/modules/indexer/parser/discuz.py b/app/modules/indexer/parser/discuz.py index 7d28378b..cb56b1cb 100644 --- a/app/modules/indexer/parser/discuz.py +++ b/app/modules/indexer/parser/discuz.py @@ -14,15 +14,18 @@ class DiscuzUserInfo(SiteParserBase): def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) - - user_info = html.xpath('//a[contains(@href, "&uid=")]') - if user_info: - user_id_match = re.search(r"&uid=(\d+)", user_info[0].attrib['href']) - if user_id_match and user_id_match.group().strip(): - self.userid = user_id_match.group(1) - self._torrent_seeding_page = f"forum.php?&mod=torrents&cat_5up=on" - self._user_detail_page = user_info[0].attrib['href'] - self.username = user_info[0].text.strip() + try: + user_info = html.xpath('//a[contains(@href, "&uid=")]') + if user_info: + user_id_match = re.search(r"&uid=(\d+)", user_info[0].attrib['href']) + if user_id_match and user_id_match.group().strip(): + self.userid = user_id_match.group(1) + self._torrent_seeding_page = f"forum.php?&mod=torrents&cat_5up=on" + self._user_detail_page = user_info[0].attrib['href'] + self.username = user_info[0].text.strip() + finally: + if html is not None: + del html def _parse_site_page(self, html_text: str): pass @@ -34,40 +37,44 @@ class DiscuzUserInfo(SiteParserBase): :return: """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - # 用户等级 - user_levels_text = html.xpath('//a[contains(@href, "usergroup")]/text()') - if user_levels_text: - self.user_level = user_levels_text[-1].strip() + # 用户等级 + user_levels_text = html.xpath('//a[contains(@href, "usergroup")]/text()') + if user_levels_text: + self.user_level = user_levels_text[-1].strip() - # 加入日期 - join_at_text = html.xpath('//li[em[text()="注册时间"]]/text()') - if join_at_text: - self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip()) + # 加入日期 + join_at_text = html.xpath('//li[em[text()="注册时间"]]/text()') + if join_at_text: + self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip()) - # 分享率 - ratio_text = html.xpath('//li[contains(.//text(), "分享率")]//text()') - if ratio_text: - ratio_match = re.search(r"\(([\d,.]+)\)", ratio_text[0]) - if ratio_match and ratio_match.group(1).strip(): - self.bonus = StringUtils.str_float(ratio_match.group(1)) + # 分享率 + ratio_text = html.xpath('//li[contains(.//text(), "分享率")]//text()') + if ratio_text: + ratio_match = re.search(r"\(([\d,.]+)\)", ratio_text[0]) + if ratio_match and ratio_match.group(1).strip(): + self.bonus = StringUtils.str_float(ratio_match.group(1)) - # 积分 - bouns_text = html.xpath('//li[em[text()="积分"]]/text()') - if bouns_text: - self.bonus = StringUtils.str_float(bouns_text[0].strip()) + # 积分 + bouns_text = html.xpath('//li[em[text()="积分"]]/text()') + if bouns_text: + self.bonus = StringUtils.str_float(bouns_text[0].strip()) - # 上传 - upload_text = html.xpath('//li[em[contains(text(),"上传量")]]/text()') - if upload_text: - self.upload = StringUtils.num_filesize(upload_text[0].strip().split('/')[-1]) + # 上传 + upload_text = html.xpath('//li[em[contains(text(),"上传量")]]/text()') + if upload_text: + self.upload = StringUtils.num_filesize(upload_text[0].strip().split('/')[-1]) - # 下载 - download_text = html.xpath('//li[em[contains(text(),"下载量")]]/text()') - if download_text: - self.download = StringUtils.num_filesize(download_text[0].strip().split('/')[-1]) + # 下载 + download_text = html.xpath('//li[em[contains(text(),"下载量")]]/text()') + if download_text: + self.download = StringUtils.num_filesize(download_text[0].strip().split('/')[-1]) + finally: + if html is not None: + del html def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]: """ @@ -77,44 +84,48 @@ class DiscuzUserInfo(SiteParserBase): :return: 下页地址 """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - size_col = 3 - seeders_col = 4 - # 搜索size列 - if html.xpath('//tr[position()=1]/td[.//img[@class="size"] and .//img[@alt="size"]]'): - size_col = len(html.xpath('//tr[position()=1]/td[.//img[@class="size"] ' - 'and .//img[@alt="size"]]/preceding-sibling::td')) + 1 - # 搜索seeders列 - if html.xpath('//tr[position()=1]/td[.//img[@class="seeders"] and .//img[@alt="seeders"]]'): - seeders_col = len(html.xpath('//tr[position()=1]/td[.//img[@class="seeders"] ' - 'and .//img[@alt="seeders"]]/preceding-sibling::td')) + 1 + size_col = 3 + seeders_col = 4 + # 搜索size列 + if html.xpath('//tr[position()=1]/td[.//img[@class="size"] and .//img[@alt="size"]]'): + size_col = len(html.xpath('//tr[position()=1]/td[.//img[@class="size"] ' + 'and .//img[@alt="size"]]/preceding-sibling::td')) + 1 + # 搜索seeders列 + if html.xpath('//tr[position()=1]/td[.//img[@class="seeders"] and .//img[@alt="seeders"]]'): + seeders_col = len(html.xpath('//tr[position()=1]/td[.//img[@class="seeders"] ' + 'and .//img[@alt="seeders"]]/preceding-sibling::td')) + 1 - page_seeding = 0 - page_seeding_size = 0 - page_seeding_info = [] - seeding_sizes = html.xpath(f'//tr[position()>1]/td[{size_col}]') - seeding_seeders = html.xpath(f'//tr[position()>1]/td[{seeders_col}]//text()') - if seeding_sizes and seeding_seeders: - page_seeding = len(seeding_sizes) + page_seeding = 0 + page_seeding_size = 0 + page_seeding_info = [] + seeding_sizes = html.xpath(f'//tr[position()>1]/td[{size_col}]') + seeding_seeders = html.xpath(f'//tr[position()>1]/td[{seeders_col}]//text()') + if seeding_sizes and seeding_seeders: + page_seeding = len(seeding_sizes) - for i in range(0, len(seeding_sizes)): - size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = StringUtils.str_int(seeding_seeders[i]) + for i in range(0, len(seeding_sizes)): + size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = StringUtils.str_int(seeding_seeders[i]) - page_seeding_size += size - page_seeding_info.append([seeders, size]) + page_seeding_size += size + page_seeding_info.append([seeders, size]) - self.seeding += page_seeding - self.seeding_size += page_seeding_size - self.seeding_info.extend(page_seeding_info) + self.seeding += page_seeding + self.seeding_size += page_seeding_size + self.seeding_info.extend(page_seeding_info) - # 是否存在下页数据 - next_page = None - next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href') - if next_page_text: - next_page = next_page_text[-1].strip() + # 是否存在下页数据 + next_page = None + next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href') + if next_page_text: + next_page = next_page_text[-1].strip() + finally: + if html is not None: + del html return next_page diff --git a/app/modules/indexer/parser/file_list.py b/app/modules/indexer/parser/file_list.py index 05bf5f8d..06730f13 100644 --- a/app/modules/indexer/parser/file_list.py +++ b/app/modules/indexer/parser/file_list.py @@ -24,10 +24,13 @@ class FileListSiteUserInfo(SiteParserBase): def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) - - ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()') - if ret: - self.username = str(ret[0]) + try: + ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()') + if ret: + self.username = str(ret[0]) + finally: + if html is not None: + del html def _parse_user_traffic_info(self, html_text: str): """ @@ -40,39 +43,41 @@ class FileListSiteUserInfo(SiteParserBase): def _parse_user_detail_info(self, html_text: str): html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) + try: + upload_html = html.xpath('//table//tr/td[text()="Uploaded"]/following-sibling::td//text()') + if upload_html: + self.upload = StringUtils.num_filesize(upload_html[0]) + download_html = html.xpath('//table//tr/td[text()="Downloaded"]/following-sibling::td//text()') + if download_html: + self.download = StringUtils.num_filesize(download_html[0]) - upload_html = html.xpath('//table//tr/td[text()="Uploaded"]/following-sibling::td//text()') - if upload_html: - self.upload = StringUtils.num_filesize(upload_html[0]) - download_html = html.xpath('//table//tr/td[text()="Downloaded"]/following-sibling::td//text()') - if download_html: - self.download = StringUtils.num_filesize(download_html[0]) + ratio_html = html.xpath('//table//tr/td[text()="Share ratio"]/following-sibling::td//text()') + if ratio_html: + share_ratio = StringUtils.str_float(ratio_html[0]) + else: + share_ratio = 0 + self.ratio = 0 if self.download == 0 else share_ratio - ratio_html = html.xpath('//table//tr/td[text()="Share ratio"]/following-sibling::td//text()') - if ratio_html: - share_ratio = StringUtils.str_float(ratio_html[0]) - else: - share_ratio = 0 - self.ratio = 0 if self.download == 0 else share_ratio + seed_html = html.xpath('//table//tr/td[text()="Seed bonus"]/following-sibling::td//text()') + if seed_html: + self.seeding = StringUtils.str_int(seed_html[1]) + self.seeding_size = StringUtils.num_filesize(seed_html[3]) - seed_html = html.xpath('//table//tr/td[text()="Seed bonus"]/following-sibling::td//text()') - if seed_html: - self.seeding = StringUtils.str_int(seed_html[1]) - self.seeding_size = StringUtils.num_filesize(seed_html[3]) + user_level_html = html.xpath('//table//tr/td[text()="Class"]/following-sibling::td//text()') + if user_level_html: + self.user_level = user_level_html[0].strip() - user_level_html = html.xpath('//table//tr/td[text()="Class"]/following-sibling::td//text()') - if user_level_html: - self.user_level = user_level_html[0].strip() + join_at_html = html.xpath('//table//tr/td[contains(text(), "Join")]/following-sibling::td//text()') + if join_at_html: + join_at = (join_at_html[0].split("("))[0].strip() + self.join_at = StringUtils.unify_datetime_str(join_at) - join_at_html = html.xpath('//table//tr/td[contains(text(), "Join")]/following-sibling::td//text()') - if join_at_html: - join_at = (join_at_html[0].split("("))[0].strip() - self.join_at = StringUtils.unify_datetime_str(join_at) - - bonus_html = html.xpath('//a[contains(@href, "shop.php")]') - if bonus_html: - self.bonus = StringUtils.str_float(bonus_html[0].xpath("string(.)").strip()) - pass + bonus_html = html.xpath('//a[contains(@href, "shop.php")]') + if bonus_html: + self.bonus = StringUtils.str_float(bonus_html[0].xpath("string(.)").strip()) + finally: + if html is not None: + del html def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]: """ @@ -82,28 +87,32 @@ class FileListSiteUserInfo(SiteParserBase): :return: 下页地址 """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - size_col = 6 - seeders_col = 7 + size_col = 6 + seeders_col = 7 - page_seeding_size = 0 - page_seeding_info = [] - seeding_sizes = html.xpath(f'//table/tr[position()>1]/td[{size_col}]') - seeding_seeders = html.xpath(f'//table/tr[position()>1]/td[{seeders_col}]') - if seeding_sizes and seeding_seeders: - for i in range(0, len(seeding_sizes)): - size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip()) + page_seeding_size = 0 + page_seeding_info = [] + seeding_sizes = html.xpath(f'//table/tr[position()>1]/td[{size_col}]') + seeding_seeders = html.xpath(f'//table/tr[position()>1]/td[{seeders_col}]') + if seeding_sizes and seeding_seeders: + for i in range(0, len(seeding_sizes)): + size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip()) - page_seeding_size += size - page_seeding_info.append([seeders, size]) + page_seeding_size += size + page_seeding_info.append([seeders, size]) - self.seeding_info.extend(page_seeding_info) + self.seeding_info.extend(page_seeding_info) - # 是否存在下页数据 - next_page = None + # 是否存在下页数据 + next_page = None + finally: + if html is not None: + del html return next_page diff --git a/app/modules/indexer/parser/gazelle.py b/app/modules/indexer/parser/gazelle.py index ea7a867f..71508589 100644 --- a/app/modules/indexer/parser/gazelle.py +++ b/app/modules/indexer/parser/gazelle.py @@ -14,46 +14,49 @@ class GazelleSiteUserInfo(SiteParserBase): def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) + try: + tmps = html.xpath('//a[contains(@href, "user.php?id=")]') + if tmps: + user_id_match = re.search(r"user.php\?id=(\d+)", tmps[0].attrib['href']) + if user_id_match and user_id_match.group().strip(): + self.userid = user_id_match.group(1) + self._torrent_seeding_page = f"torrents.php?type=seeding&userid={self.userid}" + self._user_detail_page = f"user.php?id={self.userid}" + self.username = tmps[0].text.strip() - tmps = html.xpath('//a[contains(@href, "user.php?id=")]') - if tmps: - user_id_match = re.search(r"user.php\?id=(\d+)", tmps[0].attrib['href']) - if user_id_match and user_id_match.group().strip(): - self.userid = user_id_match.group(1) - self._torrent_seeding_page = f"torrents.php?type=seeding&userid={self.userid}" - self._user_detail_page = f"user.php?id={self.userid}" - self.username = tmps[0].text.strip() - - tmps = html.xpath('//*[@id="header-uploaded-value"]/@data-value') - if tmps: - self.upload = StringUtils.num_filesize(tmps[0]) - else: - tmps = html.xpath('//li[@id="stats_seeding"]/span/text()') + tmps = html.xpath('//*[@id="header-uploaded-value"]/@data-value') if tmps: self.upload = StringUtils.num_filesize(tmps[0]) + else: + tmps = html.xpath('//li[@id="stats_seeding"]/span/text()') + if tmps: + self.upload = StringUtils.num_filesize(tmps[0]) - tmps = html.xpath('//*[@id="header-downloaded-value"]/@data-value') - if tmps: - self.download = StringUtils.num_filesize(tmps[0]) - else: - tmps = html.xpath('//li[@id="stats_leeching"]/span/text()') + tmps = html.xpath('//*[@id="header-downloaded-value"]/@data-value') if tmps: self.download = StringUtils.num_filesize(tmps[0]) + else: + tmps = html.xpath('//li[@id="stats_leeching"]/span/text()') + if tmps: + self.download = StringUtils.num_filesize(tmps[0]) - self.ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3) + self.ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3) - tmps = html.xpath('//a[contains(@href, "bonus.php")]/@data-tooltip') - if tmps: - bonus_match = re.search(r"([\d,.]+)", tmps[0]) - if bonus_match and bonus_match.group(1).strip(): - self.bonus = StringUtils.str_float(bonus_match.group(1)) - else: - tmps = html.xpath('//a[contains(@href, "bonus.php")]') + tmps = html.xpath('//a[contains(@href, "bonus.php")]/@data-tooltip') if tmps: - bonus_text = tmps[0].xpath("string(.)") - bonus_match = re.search(r"([\d,.]+)", bonus_text) + bonus_match = re.search(r"([\d,.]+)", tmps[0]) if bonus_match and bonus_match.group(1).strip(): self.bonus = StringUtils.str_float(bonus_match.group(1)) + else: + tmps = html.xpath('//a[contains(@href, "bonus.php")]') + if tmps: + bonus_text = tmps[0].xpath("string(.)") + bonus_match = re.search(r"([\d,.]+)", bonus_text) + if bonus_match and bonus_match.group(1).strip(): + self.bonus = StringUtils.str_float(bonus_match.group(1)) + finally: + if html is not None: + del html def _parse_site_page(self, html_text: str): pass @@ -65,27 +68,31 @@ class GazelleSiteUserInfo(SiteParserBase): :return: """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - # 用户等级 - user_levels_text = html.xpath('//*[@id="class-value"]/@data-value') - if user_levels_text: - self.user_level = user_levels_text[0].strip() - else: - user_levels_text = html.xpath('//li[contains(text(), "用户等级")]/text()') + # 用户等级 + user_levels_text = html.xpath('//*[@id="class-value"]/@data-value') if user_levels_text: - self.user_level = user_levels_text[0].split(':')[1].strip() + self.user_level = user_levels_text[0].strip() + else: + user_levels_text = html.xpath('//li[contains(text(), "用户等级")]/text()') + if user_levels_text: + self.user_level = user_levels_text[0].split(':')[1].strip() - # 加入日期 - join_at_text = html.xpath('//*[@id="join-date-value"]/@data-value') - if join_at_text: - self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip()) - else: - join_at_text = html.xpath( - '//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()') + # 加入日期 + join_at_text = html.xpath('//*[@id="join-date-value"]/@data-value') if join_at_text: self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip()) + else: + join_at_text = html.xpath( + '//div[contains(@class, "box_userinfo_stats")]//li[contains(text(), "加入时间")]/span/text()') + if join_at_text: + self.join_at = StringUtils.unify_datetime_str(join_at_text[0].strip()) + finally: + if html is not None: + del html def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]: """ @@ -95,48 +102,52 @@ class GazelleSiteUserInfo(SiteParserBase): :return: 下页地址 """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - size_col = 3 - # 搜索size列 - if html.xpath('//table[contains(@id, "torrent")]//tr[1]/td'): - size_col = len(html.xpath('//table[contains(@id, "torrent")]//tr[1]/td')) - 3 - # 搜索seeders列 - seeders_col = size_col + 2 + size_col = 3 + # 搜索size列 + if html.xpath('//table[contains(@id, "torrent")]//tr[1]/td'): + size_col = len(html.xpath('//table[contains(@id, "torrent")]//tr[1]/td')) - 3 + # 搜索seeders列 + seeders_col = size_col + 2 - page_seeding = 0 - page_seeding_size = 0 - page_seeding_info = [] - seeding_sizes = html.xpath(f'//table[contains(@id, "torrent")]//tr[position()>1]/td[{size_col}]') - seeding_seeders = html.xpath(f'//table[contains(@id, "torrent")]//tr[position()>1]/td[{seeders_col}]/text()') - if seeding_sizes and seeding_seeders: - page_seeding = len(seeding_sizes) + page_seeding = 0 + page_seeding_size = 0 + page_seeding_info = [] + seeding_sizes = html.xpath(f'//table[contains(@id, "torrent")]//tr[position()>1]/td[{size_col}]') + seeding_seeders = html.xpath(f'//table[contains(@id, "torrent")]//tr[position()>1]/td[{seeders_col}]/text()') + if seeding_sizes and seeding_seeders: + page_seeding = len(seeding_sizes) - for i in range(0, len(seeding_sizes)): - size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = int(seeding_seeders[i]) + for i in range(0, len(seeding_sizes)): + size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = int(seeding_seeders[i]) - page_seeding_size += size - page_seeding_info.append([seeders, size]) + page_seeding_size += size + page_seeding_info.append([seeders, size]) - if multi_page: - self.seeding += page_seeding - self.seeding_size += page_seeding_size - self.seeding_info.extend(page_seeding_info) - else: - if not self.seeding: - self.seeding = page_seeding - if not self.seeding_size: - self.seeding_size = page_seeding_size - if not self.seeding_info: - self.seeding_info = page_seeding_info + if multi_page: + self.seeding += page_seeding + self.seeding_size += page_seeding_size + self.seeding_info.extend(page_seeding_info) + else: + if not self.seeding: + self.seeding = page_seeding + if not self.seeding_size: + self.seeding_size = page_seeding_size + if not self.seeding_info: + self.seeding_info = page_seeding_info - # 是否存在下页数据 - next_page = None - next_page_text = html.xpath('//a[contains(.//text(), "Next") or contains(.//text(), "下一页")]/@href') - if next_page_text: - next_page = next_page_text[-1].strip() + # 是否存在下页数据 + next_page = None + next_page_text = html.xpath('//a[contains(.//text(), "Next") or contains(.//text(), "下一页")]/@href') + if next_page_text: + next_page = next_page_text[-1].strip() + finally: + if html is not None: + del html return next_page diff --git a/app/modules/indexer/parser/ipt_project.py b/app/modules/indexer/parser/ipt_project.py index 643cd0b3..63fb607c 100644 --- a/app/modules/indexer/parser/ipt_project.py +++ b/app/modules/indexer/parser/ipt_project.py @@ -14,67 +14,79 @@ class IptSiteUserInfo(SiteParserBase): def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) - tmps = html.xpath('//a[contains(@href, "/u/")]//text()') - tmps_id = html.xpath('//a[contains(@href, "/u/")]/@href') - if tmps: - self.username = str(tmps[-1]) - if tmps_id: - user_id_match = re.search(r"/u/(\d+)", tmps_id[0]) - if user_id_match and user_id_match.group().strip(): - self.userid = user_id_match.group(1) - self._user_detail_page = f"user.php?u={self.userid}" - self._torrent_seeding_page = f"peers?u={self.userid}" + try: + tmps = html.xpath('//a[contains(@href, "/u/")]//text()') + tmps_id = html.xpath('//a[contains(@href, "/u/")]/@href') + if tmps: + self.username = str(tmps[-1]) + if tmps_id: + user_id_match = re.search(r"/u/(\d+)", tmps_id[0]) + if user_id_match and user_id_match.group().strip(): + self.userid = user_id_match.group(1) + self._user_detail_page = f"user.php?u={self.userid}" + self._torrent_seeding_page = f"peers?u={self.userid}" - tmps = html.xpath('//div[@class = "stats"]/div/div') - if tmps: - self.upload = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[1]).strip()) - self.download = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[2]).strip()) - self.seeding = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[0]) - self.leeching = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[1]) - self.ratio = StringUtils.str_float(str(tmps[0].xpath('span/text()')[0]).strip().replace('-', '0')) - self.bonus = StringUtils.str_float(tmps[0].xpath('a')[3].xpath('text()')[0]) + tmps = html.xpath('//div[@class = "stats"]/div/div') + if tmps: + self.upload = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[1]).strip()) + self.download = StringUtils.num_filesize(str(tmps[0].xpath('span/text()')[2]).strip()) + self.seeding = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[0]) + self.leeching = StringUtils.str_int(tmps[0].xpath('a')[2].xpath('text()')[1]) + self.ratio = StringUtils.str_float(str(tmps[0].xpath('span/text()')[0]).strip().replace('-', '0')) + self.bonus = StringUtils.str_float(tmps[0].xpath('a')[3].xpath('text()')[0]) + finally: + if html is not None: + del html def _parse_site_page(self, html_text: str): pass def _parse_user_detail_info(self, html_text: str): html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return + try: + if not StringUtils.is_valid_html_element(html): + return - user_levels_text = html.xpath('//tr/th[text()="Class"]/following-sibling::td[1]/text()') - if user_levels_text: - self.user_level = user_levels_text[0].strip() + user_levels_text = html.xpath('//tr/th[text()="Class"]/following-sibling::td[1]/text()') + if user_levels_text: + self.user_level = user_levels_text[0].strip() - # 加入日期 - join_at_text = html.xpath('//tr/th[text()="Join date"]/following-sibling::td[1]/text()') - if join_at_text: - self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0]) + # 加入日期 + join_at_text = html.xpath('//tr/th[text()="Join date"]/following-sibling::td[1]/text()') + if join_at_text: + self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0]) + finally: + if html is not None: + del html def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]: html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None - # seeding start - seeding_end_pos = 3 - if html.xpath('//tr/td[text() = "Leechers"]'): - seeding_end_pos = len(html.xpath('//tr/td[text() = "Leechers"]/../preceding-sibling::tr')) + 1 - seeding_end_pos = seeding_end_pos - 3 + try: + if not StringUtils.is_valid_html_element(html): + return None + # seeding start + seeding_end_pos = 3 + if html.xpath('//tr/td[text() = "Leechers"]'): + seeding_end_pos = len(html.xpath('//tr/td[text() = "Leechers"]/../preceding-sibling::tr')) + 1 + seeding_end_pos = seeding_end_pos - 3 - page_seeding = 0 - page_seeding_size = 0 - seeding_torrents = html.xpath('//tr/td[text() = "Seeders"]/../following-sibling::tr/td[position()=6]/text()') - if seeding_torrents: - page_seeding = seeding_end_pos - for per_size in seeding_torrents[:seeding_end_pos]: - if '(' in per_size and ')' in per_size: - per_size = per_size.split('(')[-1] - per_size = per_size.split(')')[0] + page_seeding = 0 + page_seeding_size = 0 + seeding_torrents = html.xpath('//tr/td[text() = "Seeders"]/../following-sibling::tr/td[position()=6]/text()') + if seeding_torrents: + page_seeding = seeding_end_pos + for per_size in seeding_torrents[:seeding_end_pos]: + if '(' in per_size and ')' in per_size: + per_size = per_size.split('(')[-1] + per_size = per_size.split(')')[0] - page_seeding_size += StringUtils.num_filesize(per_size) + page_seeding_size += StringUtils.num_filesize(per_size) - self.seeding = page_seeding - self.seeding_size = page_seeding_size + self.seeding = page_seeding + self.seeding_size = page_seeding_size + finally: + if html is not None: + del html def _parse_user_traffic_info(self, html_text: str): pass diff --git a/app/modules/indexer/parser/nexus_audiences.py b/app/modules/indexer/parser/nexus_audiences.py index 7b391780..5d255af2 100644 --- a/app/modules/indexer/parser/nexus_audiences.py +++ b/app/modules/indexer/parser/nexus_audiences.py @@ -23,12 +23,16 @@ class NexusAudiencesSiteUserInfo(NexusPhpSiteUserInfo): if not html_text: return html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return - total_row = html.xpath('//table[@class="table table-bordered"]//tr[td[1][normalize-space()="Total"]]') - if not total_row: - return - seeding_count = total_row[0].xpath('./td[2]/text()') - seeding_size = total_row[0].xpath('./td[3]/text()') - self.seeding = StringUtils.str_int(seeding_count[0]) if seeding_count else 0 - self.seeding_size = StringUtils.num_filesize(seeding_size[0].strip()) if seeding_size else 0 + try: + if not StringUtils.is_valid_html_element(html): + return + total_row = html.xpath('//table[@class="table table-bordered"]//tr[td[1][normalize-space()="Total"]]') + if not total_row: + return + seeding_count = total_row[0].xpath('./td[2]/text()') + seeding_size = total_row[0].xpath('./td[3]/text()') + self.seeding = StringUtils.str_int(seeding_count[0]) if seeding_count else 0 + self.seeding_size = StringUtils.num_filesize(seeding_size[0].strip()) if seeding_size else 0 + finally: + if html is not None: + del html diff --git a/app/modules/indexer/parser/nexus_hhanclub.py b/app/modules/indexer/parser/nexus_hhanclub.py index 4667933e..e0b19d7d 100644 --- a/app/modules/indexer/parser/nexus_hhanclub.py +++ b/app/modules/indexer/parser/nexus_hhanclub.py @@ -17,21 +17,25 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo): html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) - # 上传、下载、分享率 - upload_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", - html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[4]/text()')[0]) - download_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", - html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[5]/text()')[0]) - ratio_match = re.search(r"分享率][:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+)", - html.xpath('//*[@id="user-info-panel"]/div[2]/div[1]/div[1]/div/text()')[0]) + try: + # 上传、下载、分享率 + upload_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", + html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[4]/text()')[0]) + download_match = re.search(r"[_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+[KMGTPI]*B)", + html.xpath('//*[@id="user-info-panel"]/div[2]/div[2]/div[5]/text()')[0]) + ratio_match = re.search(r"分享率][:：_<>/a-zA-Z-=\"'\s#;]+([\d,.\s]+)", + html.xpath('//*[@id="user-info-panel"]/div[2]/div[1]/div[1]/div/text()')[0]) - # 计算分享率 - self.upload = StringUtils.num_filesize(upload_match.group(1).strip()) if upload_match else 0 - self.download = StringUtils.num_filesize(download_match.group(1).strip()) if download_match else 0 - # 优先使用页面上的分享率 - calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3) - self.ratio = StringUtils.str_float(ratio_match.group(1)) if ( - ratio_match and ratio_match.group(1).strip()) else calc_ratio + # 计算分享率 + self.upload = StringUtils.num_filesize(upload_match.group(1).strip()) if upload_match else 0 + self.download = StringUtils.num_filesize(download_match.group(1).strip()) if download_match else 0 + # 优先使用页面上的分享率 + calc_ratio = 0.0 if self.download <= 0.0 else round(self.upload / self.download, 3) + self.ratio = StringUtils.str_float(ratio_match.group(1)) if ( + ratio_match and ratio_match.group(1).strip()) else calc_ratio + finally: + if html is not None: + del html def _parse_user_detail_info(self, html_text: str): """ @@ -42,12 +46,16 @@ class NexusHhanclubSiteUserInfo(NexusPhpSiteUserInfo): super()._parse_user_detail_info(html_text) html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return - # 加入时间 - join_at_text = html.xpath('//*[@id="mainContent"]/div/div[2]/div[4]/div[3]/span[2]/text()[1]') - if join_at_text: - self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip()) + try: + if not StringUtils.is_valid_html_element(html): + return + # 加入时间 + join_at_text = html.xpath('//*[@id="mainContent"]/div/div[2]/div[4]/div[3]/span[2]/text()[1]') + if join_at_text: + self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip()) + finally: + if html is not None: + del html def _get_user_level(self, html): super()._get_user_level(html) diff --git a/app/modules/indexer/parser/nexus_php.py b/app/modules/indexer/parser/nexus_php.py index cecd01ad..5851895c 100644 --- a/app/modules/indexer/parser/nexus_php.py +++ b/app/modules/indexer/parser/nexus_php.py @@ -34,21 +34,25 @@ class NexusPhpSiteUserInfo(SiteParserBase): :return: """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return + try: + if not StringUtils.is_valid_html_element(html): + return - message_labels = html.xpath('//a[@href="messages.php"]/..') - message_labels.extend(html.xpath('//a[contains(@href, "messages.php")]/..')) - if message_labels: - message_text = message_labels[0].xpath("string(.)") + message_labels = html.xpath('//a[@href="messages.php"]/..') + message_labels.extend(html.xpath('//a[contains(@href, "messages.php")]/..')) + if message_labels: + message_text = message_labels[0].xpath("string(.)") - logger.debug(f"{self._site_name} 消息原始信息 {message_text}") - message_unread_match = re.findall(r"[^Date](信息箱\s*|\((?![^)]*:)|你有\xa0)(\d+)", message_text) + logger.debug(f"{self._site_name} 消息原始信息 {message_text}") + message_unread_match = re.findall(r"[^Date](信息箱\s*|\((?![^)]*:)|你有\xa0)(\d+)", message_text) - if message_unread_match and len(message_unread_match[-1]) == 2: - self.message_unread = StringUtils.str_int(message_unread_match[-1][1]) - elif message_text.isdigit(): - self.message_unread = StringUtils.str_int(message_text) + if message_unread_match and len(message_unread_match[-1]) == 2: + self.message_unread = StringUtils.str_int(message_unread_match[-1][1]) + elif message_text.isdigit(): + self.message_unread = StringUtils.str_int(message_text) + finally: + if html is not None: + del html def _parse_user_base_info(self, html_text: str): """ @@ -61,18 +65,23 @@ class NexusPhpSiteUserInfo(SiteParserBase): self._parse_message_unread(html_text) html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return + try: + if not StringUtils.is_valid_html_element(html): + return - ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//b//text()') - if ret: - self.username = str(ret[0]) - return - ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()') - if ret: - self.username = str(ret[0]) + ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//b//text()') + if ret: + self.username = str(ret[0]) + return + ret = html.xpath(f'//a[contains(@href, "userdetails") and contains(@href, "{self.userid}")]//text()') + if ret: + self.username = str(ret[0]) + + ret = html.xpath('//a[contains(@href, "userdetails")]//strong//text()') + finally: + if html is not None: + del html - ret = html.xpath('//a[contains(@href, "userdetails")]//strong//text()') if ret: self.username = str(ret[0]) return @@ -98,28 +107,32 @@ class NexusPhpSiteUserInfo(SiteParserBase): self.leeching = StringUtils.str_int(leeching_match.group(2)) if leeching_match and leeching_match.group( 2).strip() else 0 html = etree.HTML(html_text) - has_ucoin, self.bonus = self._parse_ucoin(html) - if has_ucoin: - return - tmps = html.xpath('//a[contains(@href,"mybonus")]/text()') if html else None - if tmps: - bonus_text = str(tmps[0]).strip() - bonus_match = re.search(r"([\d,.]+)", bonus_text) - if bonus_match and bonus_match.group(1).strip(): - self.bonus = StringUtils.str_float(bonus_match.group(1)) - return - bonus_match = re.search(r"mybonus.[\[\]:：<>/a-zA-Z_\-=\"'\s#;.(使用魔力值豆]+\s*([\d,.]+)[<()&\s]", html_text) try: - if bonus_match and bonus_match.group(1).strip(): - self.bonus = StringUtils.str_float(bonus_match.group(1)) + has_ucoin, self.bonus = self._parse_ucoin(html) + if has_ucoin: return - bonus_match = re.search(r"[魔力值|\]][\[\]:：<>/a-zA-Z_\-=\"'\s#;]+\s*([\d,.]+|\"[\d,.]+\")[<>()&\s]", - html_text, - flags=re.S) - if bonus_match and bonus_match.group(1).strip(): - self.bonus = StringUtils.str_float(bonus_match.group(1).strip('"')) - except Exception as err: - logger.error(f"{self._site_name} 解析魔力值出错, 错误信息: {str(err)}") + tmps = html.xpath('//a[contains(@href,"mybonus")]/text()') if html else None + if tmps: + bonus_text = str(tmps[0]).strip() + bonus_match = re.search(r"([\d,.]+)", bonus_text) + if bonus_match and bonus_match.group(1).strip(): + self.bonus = StringUtils.str_float(bonus_match.group(1)) + return + bonus_match = re.search(r"mybonus.[\[\]:：<>/a-zA-Z_\-=\"'\s#;.(使用魔力值豆]+\s*([\d,.]+)[<()&\s]", html_text) + try: + if bonus_match and bonus_match.group(1).strip(): + self.bonus = StringUtils.str_float(bonus_match.group(1)) + return + bonus_match = re.search(r"[魔力值|\]][\[\]:：<>/a-zA-Z_\-=\"'\s#;]+\s*([\d,.]+|\"[\d,.]+\")[<>()&\s]", + html_text, + flags=re.S) + if bonus_match and bonus_match.group(1).strip(): + self.bonus = StringUtils.str_float(bonus_match.group(1).strip('"')) + except Exception as err: + logger.error(f"{self._site_name} 解析魔力值出错, 错误信息: {str(err)}") + finally: + if html is not None: + del html @staticmethod def _parse_ucoin(html): @@ -155,72 +168,76 @@ class NexusPhpSiteUserInfo(SiteParserBase): :return: 下页地址 """ html = etree.HTML(str(html_text).replace(r'\/', '/')) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - # 首页存在扩展链接，使用扩展链接 - seeding_url_text = html.xpath('//a[contains(@href,"torrents.php") ' - 'and contains(@href,"seeding")]/@href') - if multi_page is False and seeding_url_text and seeding_url_text[0].strip(): - self._torrent_seeding_page = seeding_url_text[0].strip() - return self._torrent_seeding_page + # 首页存在扩展链接，使用扩展链接 + seeding_url_text = html.xpath('//a[contains(@href,"torrents.php") ' + 'and contains(@href,"seeding")]/@href') + if multi_page is False and seeding_url_text and seeding_url_text[0].strip(): + self._torrent_seeding_page = seeding_url_text[0].strip() + return self._torrent_seeding_page - size_col = 3 - seeders_col = 4 - # 搜索size列 - size_col_xpath = '//tr[position()=1]/' \ - 'td[(img[@class="size"] and img[@alt="size"])' \ - ' or (text() = "大小")' \ - ' or (a/img[@class="size" and @alt="size"])]' - if html.xpath(size_col_xpath): - size_col = len(html.xpath(f'{size_col_xpath}/preceding-sibling::td')) + 1 - # 搜索seeders列 - seeders_col_xpath = '//tr[position()=1]/' \ - 'td[(img[@class="seeders"] and img[@alt="seeders"])' \ - ' or (text() = "在做种")' \ - ' or (a/img[@class="seeders" and @alt="seeders"])]' - if html.xpath(seeders_col_xpath): - seeders_col = len(html.xpath(f'{seeders_col_xpath}/preceding-sibling::td')) + 1 + size_col = 3 + seeders_col = 4 + # 搜索size列 + size_col_xpath = '//tr[position()=1]/' \ + 'td[(img[@class="size"] and img[@alt="size"])' \ + ' or (text() = "大小")' \ + ' or (a/img[@class="size" and @alt="size"])]' + if html.xpath(size_col_xpath): + size_col = len(html.xpath(f'{size_col_xpath}/preceding-sibling::td')) + 1 + # 搜索seeders列 + seeders_col_xpath = '//tr[position()=1]/' \ + 'td[(img[@class="seeders"] and img[@alt="seeders"])' \ + ' or (text() = "在做种")' \ + ' or (a/img[@class="seeders" and @alt="seeders"])]' + if html.xpath(seeders_col_xpath): + seeders_col = len(html.xpath(f'{seeders_col_xpath}/preceding-sibling::td')) + 1 - page_seeding = 0 - page_seeding_size = 0 - page_seeding_info = [] - # 如果 table class="torrents"，则增加table[@class="torrents"] - table_class = '//table[@class="torrents"]' if html.xpath('//table[@class="torrents"]') else '' - seeding_sizes = html.xpath(f'{table_class}//tr[position()>1]/td[{size_col}]') - seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]/b/a/text()') - if not seeding_seeders: - seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]//text()') - if seeding_sizes and seeding_seeders: - page_seeding = len(seeding_sizes) + page_seeding = 0 + page_seeding_size = 0 + page_seeding_info = [] + # 如果 table class="torrents"，则增加table[@class="torrents"] + table_class = '//table[@class="torrents"]' if html.xpath('//table[@class="torrents"]') else '' + seeding_sizes = html.xpath(f'{table_class}//tr[position()>1]/td[{size_col}]') + seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]/b/a/text()') + if not seeding_seeders: + seeding_seeders = html.xpath(f'{table_class}//tr[position()>1]/td[{seeders_col}]//text()') + if seeding_sizes and seeding_seeders: + page_seeding = len(seeding_sizes) - for i in range(0, len(seeding_sizes)): - size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = StringUtils.str_int(seeding_seeders[i]) + for i in range(0, len(seeding_sizes)): + size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = StringUtils.str_int(seeding_seeders[i]) - page_seeding_size += size - page_seeding_info.append([seeders, size]) + page_seeding_size += size + page_seeding_info.append([seeders, size]) - self.seeding += page_seeding - self.seeding_size += page_seeding_size - self.seeding_info.extend(page_seeding_info) + self.seeding += page_seeding + self.seeding_size += page_seeding_size + self.seeding_info.extend(page_seeding_info) - # 是否存在下页数据 - next_page = None - next_page_text = html.xpath( - '//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href') - - # 防止识别到详情页 - while next_page_text: - next_page = next_page_text.pop().strip() - if not next_page.startswith('details.php'): - break + # 是否存在下页数据 next_page = None + next_page_text = html.xpath( + '//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁") or contains(.//text(), ">")]/@href') - # fix up page url - if next_page: - if self.userid not in next_page: - next_page = f'{next_page}&userid={self.userid}&type=seeding' + # 防止识别到详情页 + while next_page_text: + next_page = next_page_text.pop().strip() + if not next_page.startswith('details.php'): + break + next_page = None + + # fix up page url + if next_page: + if self.userid not in next_page: + next_page = f'{next_page}&userid={self.userid}&type=seeding' + finally: + if html is not None: + del html return next_page @@ -231,57 +248,61 @@ class NexusPhpSiteUserInfo(SiteParserBase): :return: """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return + try: + if not StringUtils.is_valid_html_element(html): + return - self._get_user_level(html) + self._get_user_level(html) - self._fixup_traffic_info(html) + self._fixup_traffic_info(html) - # 加入日期 - join_at_text = html.xpath( - '//tr/td[text()="加入日期" or text()="注册日期" or *[text()="加入日期"]]/following-sibling::td[1]//text()' - '|//div/b[text()="加入日期"]/../text()') - if join_at_text: - self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip()) + # 加入日期 + join_at_text = html.xpath( + '//tr/td[text()="加入日期" or text()="注册日期" or *[text()="加入日期"]]/following-sibling::td[1]//text()' + '|//div/b[text()="加入日期"]/../text()') + if join_at_text: + self.join_at = StringUtils.unify_datetime_str(join_at_text[0].split(' (')[0].strip()) - # 做种体积 & 做种数 - # seeding 页面获取不到的话，此处再获取一次 - seeding_sizes = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//' - 'table[tr[1][td[4 and text()="尺寸"]]]//tr[position()>1]/td[4]') - seeding_seeders = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//' - 'table[tr[1][td[5 and text()="做种者"]]]//tr[position()>1]/td[5]//text()') - tmp_seeding = len(seeding_sizes) - tmp_seeding_size = 0 - tmp_seeding_info = [] - for i in range(0, len(seeding_sizes)): - size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = StringUtils.str_int(seeding_seeders[i]) + # 做种体积 & 做种数 + # seeding 页面获取不到的话，此处再获取一次 + seeding_sizes = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//' + 'table[tr[1][td[4 and text()="尺寸"]]]//tr[position()>1]/td[4]') + seeding_seeders = html.xpath('//tr/td[text()="当前上传"]/following-sibling::td[1]//' + 'table[tr[1][td[5 and text()="做种者"]]]//tr[position()>1]/td[5]//text()') + tmp_seeding = len(seeding_sizes) + tmp_seeding_size = 0 + tmp_seeding_info = [] + for i in range(0, len(seeding_sizes)): + size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = StringUtils.str_int(seeding_seeders[i]) - tmp_seeding_size += size - tmp_seeding_info.append([seeders, size]) + tmp_seeding_size += size + tmp_seeding_info.append([seeders, size]) - if not self.seeding_size: - self.seeding_size = tmp_seeding_size - if not self.seeding: - self.seeding = tmp_seeding - if not self.seeding_info: - self.seeding_info = tmp_seeding_info + if not self.seeding_size: + self.seeding_size = tmp_seeding_size + if not self.seeding: + self.seeding = tmp_seeding + if not self.seeding_info: + self.seeding_info = tmp_seeding_info - seeding_sizes = html.xpath('//tr/td[text()="做种统计"]/following-sibling::td[1]//text()') - if seeding_sizes: - seeding_match = re.search(r"总做种数:\s+(\d+)", seeding_sizes[0], re.IGNORECASE) - seeding_size_match = re.search(r"总做种体积:\s+([\d,.\s]+[KMGTPI]*B)", seeding_sizes[0], re.IGNORECASE) - tmp_seeding = StringUtils.str_int(seeding_match.group(1)) if ( - seeding_match and seeding_match.group(1)) else 0 - tmp_seeding_size = StringUtils.num_filesize( - seeding_size_match.group(1).strip()) if seeding_size_match else 0 - if not self.seeding_size: - self.seeding_size = tmp_seeding_size - if not self.seeding: - self.seeding = tmp_seeding + seeding_sizes = html.xpath('//tr/td[text()="做种统计"]/following-sibling::td[1]//text()') + if seeding_sizes: + seeding_match = re.search(r"总做种数:\s+(\d+)", seeding_sizes[0], re.IGNORECASE) + seeding_size_match = re.search(r"总做种体积:\s+([\d,.\s]+[KMGTPI]*B)", seeding_sizes[0], re.IGNORECASE) + tmp_seeding = StringUtils.str_int(seeding_match.group(1)) if ( + seeding_match and seeding_match.group(1)) else 0 + tmp_seeding_size = StringUtils.num_filesize( + seeding_size_match.group(1).strip()) if seeding_size_match else 0 + if not self.seeding_size: + self.seeding_size = tmp_seeding_size + if not self.seeding: + self.seeding = tmp_seeding - self._fixup_torrent_seeding_page(html) + self._fixup_torrent_seeding_page(html) + finally: + if html is not None: + del html def _fixup_torrent_seeding_page(self, html): """ @@ -348,43 +369,51 @@ class NexusPhpSiteUserInfo(SiteParserBase): def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]: html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - message_links = html.xpath('//tr[not(./td/img[@alt="Read"])]/td/a[contains(@href, "viewmessage")]/@href') - msg_links.extend(message_links) - # 是否存在下页数据 - next_page = None - next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href') - if next_page_text: - next_page = next_page_text[-1].strip() + message_links = html.xpath('//tr[not(./td/img[@alt="Read"])]/td/a[contains(@href, "viewmessage")]/@href') + msg_links.extend(message_links) + # 是否存在下页数据 + next_page = None + next_page_text = html.xpath('//a[contains(.//text(), "下一页") or contains(.//text(), "下一頁")]/@href') + if next_page_text: + next_page = next_page_text[-1].strip() + finally: + if html is not None: + del html return next_page def _parse_message_content(self, html_text): html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None, None, None - # 标题 - message_head_text = None - message_head = html.xpath('//h1/text()' - '|//div[@class="layui-card-header"]/span[1]/text()') - if message_head: - message_head_text = message_head[-1].strip() + try: + if not StringUtils.is_valid_html_element(html): + return None, None, None + # 标题 + message_head_text = None + message_head = html.xpath('//h1/text()' + '|//div[@class="layui-card-header"]/span[1]/text()') + if message_head: + message_head_text = message_head[-1].strip() - # 消息时间 - message_date_text = None - message_date = html.xpath('//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[2]/td[2]' - '|//div[@class="layui-card-header"]/span[2]/span[2]') - if message_date: - message_date_text = message_date[0].xpath("string(.)").strip() + # 消息时间 + message_date_text = None + message_date = html.xpath('//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[2]/td[2]' + '|//div[@class="layui-card-header"]/span[2]/span[2]') + if message_date: + message_date_text = message_date[0].xpath("string(.)").strip() - # 消息内容 - message_content_text = None - message_content = html.xpath('//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[3]/td' - '|//div[contains(@class,"layui-card-body")]') - if message_content: - message_content_text = message_content[0].xpath("string(.)").strip() + # 消息内容 + message_content_text = None + message_content = html.xpath('//h1/following-sibling::table[.//tr/td[@class="colhead"]]//tr[3]/td' + '|//div[contains(@class,"layui-card-body")]') + if message_content: + message_content_text = message_content[0].xpath("string(.)").strip() + finally: + if html is not None: + del html return message_head_text, message_date_text, message_content_text diff --git a/app/modules/indexer/parser/nexus_rabbit.py b/app/modules/indexer/parser/nexus_rabbit.py index bf72c8eb..e93b08d3 100644 --- a/app/modules/indexer/parser/nexus_rabbit.py +++ b/app/modules/indexer/parser/nexus_rabbit.py @@ -114,48 +114,56 @@ class NexusRabbitSiteUserInfo(SiteParserBase): def _parse_user_base_info(self, html_text: str): """只有奶糖余额才需要在 base 中获取，其它均可以在详情页拿到""" html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return - bonus = html.xpath( - '//div[contains(text(), "奶糖余额")]/following-sibling::div[1]/text()' - ) - if bonus: - self.bonus = StringUtils.str_float(bonus[0].strip()) + try: + if not StringUtils.is_valid_html_element(html): + return + bonus = html.xpath( + '//div[contains(text(), "奶糖余额")]/following-sibling::div[1]/text()' + ) + if bonus: + self.bonus = StringUtils.str_float(bonus[0].strip()) + finally: + if html is not None: + del html def _parse_user_detail_info(self, html_text: str): html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return - # 缩小一下查找范围，所有的信息都在这个 div 里 - user_info = html.xpath('//div[contains(@class, "layui-hares-user-info-right")]') - if not user_info: - return - user_info = user_info[0] - # 用户名 - if username := user_info.xpath( - './/span[contains(text(), "用户名")]/a/span/text()' - ): - self.username = username[0].strip() - # 等级 - if user_level := user_info.xpath('.//span[contains(text(), "等级")]/b/text()'): - self.user_level = user_level[0].strip() - # 加入日期 - if join_date := user_info.xpath('.//span[contains(text(), "注册日期")]/text()'): - join_date = join_date[0].strip().split("\r")[0].removeprefix("注册日期：") - self.join_at = StringUtils.unify_datetime_str(join_date) - # 上传量 - if upload := user_info.xpath('.//span[contains(text(), "上传量")]/text()'): - self.upload = StringUtils.num_filesize( - upload[0].strip().removeprefix("上传量：") - ) - # 下载量 - if download := user_info.xpath('.//span[contains(text(), "下载量")]/text()'): - self.download = StringUtils.num_filesize( - download[0].strip().removeprefix("下载量：") - ) - # 分享率 - if ratio := user_info.xpath('.//span[contains(text(), "分享率")]/em/text()'): - self.ratio = StringUtils.str_float(ratio[0].strip()) + try: + if not StringUtils.is_valid_html_element(html): + return + # 缩小一下查找范围，所有的信息都在这个 div 里 + user_info = html.xpath('//div[contains(@class, "layui-hares-user-info-right")]') + if not user_info: + return + user_info = user_info[0] + # 用户名 + if username := user_info.xpath( + './/span[contains(text(), "用户名")]/a/span/text()' + ): + self.username = username[0].strip() + # 等级 + if user_level := user_info.xpath('.//span[contains(text(), "等级")]/b/text()'): + self.user_level = user_level[0].strip() + # 加入日期 + if join_date := user_info.xpath('.//span[contains(text(), "注册日期")]/text()'): + join_date = join_date[0].strip().split("\r")[0].removeprefix("注册日期：") + self.join_at = StringUtils.unify_datetime_str(join_date) + # 上传量 + if upload := user_info.xpath('.//span[contains(text(), "上传量")]/text()'): + self.upload = StringUtils.num_filesize( + upload[0].strip().removeprefix("上传量：") + ) + # 下载量 + if download := user_info.xpath('.//span[contains(text(), "下载量")]/text()'): + self.download = StringUtils.num_filesize( + download[0].strip().removeprefix("下载量：") + ) + # 分享率 + if ratio := user_info.xpath('.//span[contains(text(), "分享率")]/em/text()'): + self.ratio = StringUtils.str_float(ratio[0].strip()) + finally: + if html is not None: + del html def _parse_message_content(self, html_text): """ diff --git a/app/modules/indexer/parser/small_horse.py b/app/modules/indexer/parser/small_horse.py index 745592fc..a1307b71 100644 --- a/app/modules/indexer/parser/small_horse.py +++ b/app/modules/indexer/parser/small_horse.py @@ -24,9 +24,13 @@ class SmallHorseSiteUserInfo(SiteParserBase): def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) - ret = html.xpath('//a[contains(@href, "user.php")]//text()') - if ret: - self.username = str(ret[0]) + try: + ret = html.xpath('//a[contains(@href, "user.php")]//text()') + if ret: + self.username = str(ret[0]) + finally: + if html is not None: + del html def _parse_user_traffic_info(self, html_text: str): """ @@ -36,21 +40,25 @@ class SmallHorseSiteUserInfo(SiteParserBase): """ html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) - tmps = html.xpath('//ul[@class = "stats nobullet"]') - if tmps: - if tmps[1].xpath("li") and tmps[1].xpath("li")[0].xpath("span//text()"): - self.join_at = StringUtils.unify_datetime_str(tmps[1].xpath("li")[0].xpath("span//text()")[0]) - self.upload = StringUtils.num_filesize(str(tmps[1].xpath("li")[2].xpath("text()")[0]).split(":")[1].strip()) - self.download = StringUtils.num_filesize( - str(tmps[1].xpath("li")[3].xpath("text()")[0]).split(":")[1].strip()) - if tmps[1].xpath("li")[4].xpath("span//text()"): - self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[4].xpath("span//text()")[0]).replace('∞', '0')) - else: - self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1]) - self.bonus = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1]) - self.user_level = str(tmps[3].xpath("li")[0].xpath("text()")[0]).split(":")[1].strip() - self.leeching = StringUtils.str_int( - (tmps[4].xpath("li")[6].xpath("text()")[0]).split(":")[1].replace("[", "")) + try: + tmps = html.xpath('//ul[@class = "stats nobullet"]') + if tmps: + if tmps[1].xpath("li") and tmps[1].xpath("li")[0].xpath("span//text()"): + self.join_at = StringUtils.unify_datetime_str(tmps[1].xpath("li")[0].xpath("span//text()")[0]) + self.upload = StringUtils.num_filesize(str(tmps[1].xpath("li")[2].xpath("text()")[0]).split(":")[1].strip()) + self.download = StringUtils.num_filesize( + str(tmps[1].xpath("li")[3].xpath("text()")[0]).split(":")[1].strip()) + if tmps[1].xpath("li")[4].xpath("span//text()"): + self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[4].xpath("span//text()")[0]).replace('∞', '0')) + else: + self.ratio = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1]) + self.bonus = StringUtils.str_float(str(tmps[1].xpath("li")[5].xpath("text()")[0]).split(":")[1]) + self.user_level = str(tmps[3].xpath("li")[0].xpath("text()")[0]).split(":")[1].strip() + self.leeching = StringUtils.str_int( + (tmps[4].xpath("li")[6].xpath("text()")[0]).split(":")[1].replace("[", "")) + finally: + if html is not None: + del html def _parse_user_detail_info(self, html_text: str): pass @@ -63,39 +71,42 @@ class SmallHorseSiteUserInfo(SiteParserBase): :return: 下页地址 """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - size_col = 6 - seeders_col = 8 + size_col = 6 + seeders_col = 8 - page_seeding = 0 - page_seeding_size = 0 - page_seeding_info = [] - seeding_sizes = html.xpath(f'//table[@id="torrent_table"]//tr[position()>1]/td[{size_col}]') - seeding_seeders = html.xpath(f'//table[@id="torrent_table"]//tr[position()>1]/td[{seeders_col}]') - if seeding_sizes and seeding_seeders: - page_seeding = len(seeding_sizes) + page_seeding = 0 + page_seeding_size = 0 + page_seeding_info = [] + seeding_sizes = html.xpath(f'//table[@id="torrent_table"]//tr[position()>1]/td[{size_col}]') + seeding_seeders = html.xpath(f'//table[@id="torrent_table"]//tr[position()>1]/td[{seeders_col}]') + if seeding_sizes and seeding_seeders: + page_seeding = len(seeding_sizes) - for i in range(0, len(seeding_sizes)): - size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip()) + for i in range(0, len(seeding_sizes)): + size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip()) - page_seeding_size += size - page_seeding_info.append([seeders, size]) + page_seeding_size += size + page_seeding_info.append([seeders, size]) - self.seeding += page_seeding - self.seeding_size += page_seeding_size - self.seeding_info.extend(page_seeding_info) - - # 是否存在下页数据 - next_page = None - next_pages = html.xpath('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li') - if next_pages and len(next_pages) > 1: - page_num = next_pages[0].xpath("string(.)").strip() - if page_num.isdigit(): - next_page = f"{self._torrent_seeding_page}&page={page_num}" + self.seeding += page_seeding + self.seeding_size += page_seeding_size + self.seeding_info.extend(page_seeding_info) + # 是否存在下页数据 + next_page = None + next_pages = html.xpath('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li') + if next_pages and len(next_pages) > 1: + page_num = next_pages[0].xpath("string(.)").strip() + if page_num.isdigit(): + next_page = f"{self._torrent_seeding_page}&page={page_num}" + finally: + if html is not None: + del html return next_page def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]: diff --git a/app/modules/indexer/parser/torrent_leech.py b/app/modules/indexer/parser/torrent_leech.py index 55a246ce..e7d85ace 100644 --- a/app/modules/indexer/parser/torrent_leech.py +++ b/app/modules/indexer/parser/torrent_leech.py @@ -32,29 +32,33 @@ class TorrentLeechSiteUserInfo(SiteParserBase): """ html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) - upload_html = html.xpath('//div[contains(@class,"profile-uploaded")]//span/text()') - if upload_html: - self.upload = StringUtils.num_filesize(upload_html[0]) - download_html = html.xpath('//div[contains(@class,"profile-downloaded")]//span/text()') - if download_html: - self.download = StringUtils.num_filesize(download_html[0]) - ratio_html = html.xpath('//div[contains(@class,"profile-ratio")]//span/text()') - if ratio_html: - self.ratio = StringUtils.str_float(ratio_html[0].replace('∞', '0')) + try: + upload_html = html.xpath('//div[contains(@class,"profile-uploaded")]//span/text()') + if upload_html: + self.upload = StringUtils.num_filesize(upload_html[0]) + download_html = html.xpath('//div[contains(@class,"profile-downloaded")]//span/text()') + if download_html: + self.download = StringUtils.num_filesize(download_html[0]) + ratio_html = html.xpath('//div[contains(@class,"profile-ratio")]//span/text()') + if ratio_html: + self.ratio = StringUtils.str_float(ratio_html[0].replace('∞', '0')) - user_level_html = html.xpath('//table[contains(@class, "profileViewTable")]' - '//tr/td[text()="Class"]/following-sibling::td/text()') - if user_level_html: - self.user_level = user_level_html[0].strip() + user_level_html = html.xpath('//table[contains(@class, "profileViewTable")]' + '//tr/td[text()="Class"]/following-sibling::td/text()') + if user_level_html: + self.user_level = user_level_html[0].strip() - join_at_html = html.xpath('//table[contains(@class, "profileViewTable")]' - '//tr/td[text()="Registration date"]/following-sibling::td/text()') - if join_at_html: - self.join_at = StringUtils.unify_datetime_str(join_at_html[0].strip()) + join_at_html = html.xpath('//table[contains(@class, "profileViewTable")]' + '//tr/td[text()="Registration date"]/following-sibling::td/text()') + if join_at_html: + self.join_at = StringUtils.unify_datetime_str(join_at_html[0].strip()) - bonus_html = html.xpath('//span[contains(@class, "total-TL-points")]/text()') - if bonus_html: - self.bonus = StringUtils.str_float(bonus_html[0].strip()) + bonus_html = html.xpath('//span[contains(@class, "total-TL-points")]/text()') + if bonus_html: + self.bonus = StringUtils.str_float(bonus_html[0].strip()) + finally: + if html is not None: + del html def _parse_user_detail_info(self, html_text: str): pass @@ -67,33 +71,37 @@ class TorrentLeechSiteUserInfo(SiteParserBase): :return: 下页地址 """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - size_col = 2 - seeders_col = 7 + size_col = 2 + seeders_col = 7 - page_seeding = 0 - page_seeding_size = 0 - page_seeding_info = [] - seeding_sizes = html.xpath(f'//tbody/tr/td[{size_col}]') - seeding_seeders = html.xpath(f'//tbody/tr/td[{seeders_col}]/text()') - if seeding_sizes and seeding_seeders: - page_seeding = len(seeding_sizes) + page_seeding = 0 + page_seeding_size = 0 + page_seeding_info = [] + seeding_sizes = html.xpath(f'//tbody/tr/td[{size_col}]') + seeding_seeders = html.xpath(f'//tbody/tr/td[{seeders_col}]/text()') + if seeding_sizes and seeding_seeders: + page_seeding = len(seeding_sizes) - for i in range(0, len(seeding_sizes)): - size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = StringUtils.str_int(seeding_seeders[i]) + for i in range(0, len(seeding_sizes)): + size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = StringUtils.str_int(seeding_seeders[i]) - page_seeding_size += size - page_seeding_info.append([seeders, size]) + page_seeding_size += size + page_seeding_info.append([seeders, size]) - self.seeding += page_seeding - self.seeding_size += page_seeding_size - self.seeding_info.extend(page_seeding_info) + self.seeding += page_seeding + self.seeding_size += page_seeding_size + self.seeding_info.extend(page_seeding_info) - # 是否存在下页数据 - next_page = None + # 是否存在下页数据 + next_page = None + finally: + if html is not None: + del html return next_page diff --git a/app/modules/indexer/parser/unit3d.py b/app/modules/indexer/parser/unit3d.py index b0d3c55a..bb9ebdf6 100644 --- a/app/modules/indexer/parser/unit3d.py +++ b/app/modules/indexer/parser/unit3d.py @@ -14,21 +14,24 @@ class Unit3dSiteUserInfo(SiteParserBase): def _parse_user_base_info(self, html_text: str): html_text = self._prepare_html_text(html_text) html = etree.HTML(html_text) + try: + tmps = html.xpath('//a[contains(@href, "/users/") and contains(@href, "settings")]/@href') + if tmps: + user_name_match = re.search(r"/users/(.+)/settings", tmps[0]) + if user_name_match and user_name_match.group().strip(): + self.username = user_name_match.group(1) + self._torrent_seeding_page = f"/users/{self.username}/active?perPage=100&client=&seeding=include" + self._user_detail_page = f"/users/{self.username}" - tmps = html.xpath('//a[contains(@href, "/users/") and contains(@href, "settings")]/@href') - if tmps: - user_name_match = re.search(r"/users/(.+)/settings", tmps[0]) - if user_name_match and user_name_match.group().strip(): - self.username = user_name_match.group(1) - self._torrent_seeding_page = f"/users/{self.username}/active?perPage=100&client=&seeding=include" - self._user_detail_page = f"/users/{self.username}" - - tmps = html.xpath('//a[contains(@href, "bonus/earnings")]') - if tmps: - bonus_text = tmps[0].xpath("string(.)") - bonus_match = re.search(r"([\d,.]+)", bonus_text) - if bonus_match and bonus_match.group(1).strip(): - self.bonus = StringUtils.str_float(bonus_match.group(1)) + tmps = html.xpath('//a[contains(@href, "bonus/earnings")]') + if tmps: + bonus_text = tmps[0].xpath("string(.)") + bonus_match = re.search(r"([\d,.]+)", bonus_text) + if bonus_match and bonus_match.group(1).strip(): + self.bonus = StringUtils.str_float(bonus_match.group(1)) + finally: + if html is not None: + del html def _parse_site_page(self, html_text: str): pass @@ -40,21 +43,25 @@ class Unit3dSiteUserInfo(SiteParserBase): :return: """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - # 用户等级 - user_levels_text = html.xpath('//div[contains(@class, "content")]//span[contains(@class, "badge-user")]/text()') - if user_levels_text: - self.user_level = user_levels_text[0].strip() + # 用户等级 + user_levels_text = html.xpath('//div[contains(@class, "content")]//span[contains(@class, "badge-user")]/text()') + if user_levels_text: + self.user_level = user_levels_text[0].strip() - # 加入日期 - join_at_text = html.xpath('//div[contains(@class, "content")]//h4[contains(text(), "注册日期") ' - 'or contains(text(), "註冊日期") ' - 'or contains(text(), "Registration date")]/text()') - if join_at_text: - self.join_at = StringUtils.unify_datetime_str( - join_at_text[0].replace('注册日期', '').replace('註冊日期', '').replace('Registration date', '')) + # 加入日期 + join_at_text = html.xpath('//div[contains(@class, "content")]//h4[contains(text(), "注册日期") ' + 'or contains(text(), "註冊日期") ' + 'or contains(text(), "Registration date")]/text()') + if join_at_text: + self.join_at = StringUtils.unify_datetime_str( + join_at_text[0].replace('注册日期', '').replace('註冊日期', '').replace('Registration date', '')) + finally: + if html is not None: + del html def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: Optional[bool] = False) -> Optional[str]: """ @@ -64,44 +71,48 @@ class Unit3dSiteUserInfo(SiteParserBase): :return: 下页地址 """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return None + try: + if not StringUtils.is_valid_html_element(html): + return None - size_col = 9 - seeders_col = 2 - # 搜索size列 - if html.xpath('//thead//th[contains(@class,"size")]'): - size_col = len(html.xpath('//thead//th[contains(@class,"size")][1]/preceding-sibling::th')) + 1 - # 搜索seeders列 - if html.xpath('//thead//th[contains(@class,"seeders")]'): - seeders_col = len(html.xpath('//thead//th[contains(@class,"seeders")]/preceding-sibling::th')) + 1 + size_col = 9 + seeders_col = 2 + # 搜索size列 + if html.xpath('//thead//th[contains(@class,"size")]'): + size_col = len(html.xpath('//thead//th[contains(@class,"size")][1]/preceding-sibling::th')) + 1 + # 搜索seeders列 + if html.xpath('//thead//th[contains(@class,"seeders")]'): + seeders_col = len(html.xpath('//thead//th[contains(@class,"seeders")]/preceding-sibling::th')) + 1 - page_seeding = 0 - page_seeding_size = 0 - page_seeding_info = [] - seeding_sizes = html.xpath(f'//tr[position()]/td[{size_col}]') - seeding_seeders = html.xpath(f'//tr[position()]/td[{seeders_col}]') - if seeding_sizes and seeding_seeders: - page_seeding = len(seeding_sizes) + page_seeding = 0 + page_seeding_size = 0 + page_seeding_info = [] + seeding_sizes = html.xpath(f'//tr[position()]/td[{size_col}]') + seeding_seeders = html.xpath(f'//tr[position()]/td[{seeders_col}]') + if seeding_sizes and seeding_seeders: + page_seeding = len(seeding_sizes) - for i in range(0, len(seeding_sizes)): - size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) - seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip()) + for i in range(0, len(seeding_sizes)): + size = StringUtils.num_filesize(seeding_sizes[i].xpath("string(.)").strip()) + seeders = StringUtils.str_int(seeding_seeders[i].xpath("string(.)").strip()) - page_seeding_size += size - page_seeding_info.append([seeders, size]) + page_seeding_size += size + page_seeding_info.append([seeders, size]) - self.seeding += page_seeding - self.seeding_size += page_seeding_size - self.seeding_info.extend(page_seeding_info) + self.seeding += page_seeding + self.seeding_size += page_seeding_size + self.seeding_info.extend(page_seeding_info) - # 是否存在下页数据 - next_page = None - next_pages = html.xpath('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li') - if next_pages and len(next_pages) > 1: - page_num = next_pages[0].xpath("string(.)").strip() - if page_num.isdigit(): - next_page = f"{self._torrent_seeding_page}&page={page_num}" + # 是否存在下页数据 + next_page = None + next_pages = html.xpath('//ul[@class="pagination"]/li[contains(@class,"active")]/following-sibling::li') + if next_pages and len(next_pages) > 1: + page_num = next_pages[0].xpath("string(.)").strip() + if page_num.isdigit(): + next_page = f"{self._torrent_seeding_page}&page={page_num}" + finally: + if html is not None: + del html return next_page diff --git a/app/modules/subtitle/__init__.py b/app/modules/subtitle/__init__.py index 3971d148..acb65afc 100644 --- a/app/modules/subtitle/__init__.py +++ b/app/modules/subtitle/__init__.py @@ -104,20 +104,24 @@ class SubtitleModule(_ModuleBase): logger.warn(f"读取页面代码失败：{torrent.page_url}") return html = etree.HTML(res.text) - sublink_list = [] - for xpath in self._SITE_SUBTITLE_XPATH: - sublinks = html.xpath(xpath) - if sublinks: - for sublink in sublinks: - if not sublink: - continue - if not sublink.startswith("http"): - base_url = StringUtils.get_base_url(torrent.page_url) - if sublink.startswith("/"): - sublink = "%s%s" % (base_url, sublink) - else: - sublink = "%s/%s" % (base_url, sublink) - sublink_list.append(sublink) + try: + sublink_list = [] + for xpath in self._SITE_SUBTITLE_XPATH: + sublinks = html.xpath(xpath) + if sublinks: + for sublink in sublinks: + if not sublink: + continue + if not sublink.startswith("http"): + base_url = StringUtils.get_base_url(torrent.page_url) + if sublink.startswith("/"): + sublink = "%s%s" % (base_url, sublink) + else: + sublink = "%s/%s" % (base_url, sublink) + sublink_list.append(sublink) + finally: + if html is not None: + del html # 下载所有字幕文件 for sublink in sublink_list: logger.info(f"找到字幕下载链接：{sublink}，开始下载...") diff --git a/app/modules/themoviedb/tmdbapi.py b/app/modules/themoviedb/tmdbapi.py index 8da7c1ed..77b6ab35 100644 --- a/app/modules/themoviedb/tmdbapi.py +++ b/app/modules/themoviedb/tmdbapi.py @@ -563,6 +563,9 @@ class TmdbApi: except Exception as err: logger.error(f"从TheDbMovie网站查询出错：{str(err)}") return {} + finally: + if html is not None: + del html return {} def get_info(self, diff --git a/app/utils/site.py b/app/utils/site.py index f1831683..2d594d37 100644 --- a/app/utils/site.py +++ b/app/utils/site.py @@ -13,27 +13,31 @@ class SiteUtils: :return: """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): + try: + if not StringUtils.is_valid_html_element(html): + return False + # 存在明显的密码输入框，说明未登录 + if html.xpath("//input[@type='password']"): + return False + # 是否存在登出和用户面板等链接 + xpaths = [ + '//a[contains(@href, "logout")' + ' or contains(@data-url, "logout")' + ' or contains(@href, "mybonus") ' + ' or contains(@onclick, "logout")' + ' or contains(@href, "usercp")' + ' or contains(@lay-on, "logout")]', + '//form[contains(@action, "logout")]', + '//div[@class="user-info-side"]', + '//a[@id="myitem"]' + ] + for xpath in xpaths: + if html.xpath(xpath): + return True return False - # 存在明显的密码输入框，说明未登录 - if html.xpath("//input[@type='password']"): - return False - # 是否存在登出和用户面板等链接 - xpaths = [ - '//a[contains(@href, "logout")' - ' or contains(@data-url, "logout")' - ' or contains(@href, "mybonus") ' - ' or contains(@onclick, "logout")' - ' or contains(@href, "usercp")' - ' or contains(@lay-on, "logout")]', - '//form[contains(@action, "logout")]', - '//div[@class="user-info-side"]', - '//a[@id="myitem"]' - ] - for xpath in xpaths: - if html.xpath(xpath): - return True - return False + finally: + if html is not None: + del html @classmethod def is_checkin(cls, html_text: str) -> bool: @@ -42,24 +46,27 @@ class SiteUtils: :return True已签到 False未签到 """ html = etree.HTML(html_text) - if not StringUtils.is_valid_html_element(html): - return False - # 站点签到支持的识别XPATH - xpaths = [ - '//a[@id="signed"]', - '//a[contains(@href, "attendance")]', - '//a[contains(text(), "签到")]', - '//a/b[contains(text(), "签 到")]', - '//span[@id="sign_in"]/a', - '//a[contains(@href, "addbonus")]', - '//input[@class="dt_button"][contains(@value, "打卡")]', - '//a[contains(@href, "sign_in")]', - '//a[contains(@onclick, "do_signin")]', - '//a[@id="do-attendance"]', - '//shark-icon-button[@href="attendance.php"]' - ] - for xpath in xpaths: - if html.xpath(xpath): + try: + if not StringUtils.is_valid_html_element(html): return False - - return True + # 站点签到支持的识别XPATH + xpaths = [ + '//a[@id="signed"]', + '//a[contains(@href, "attendance")]', + '//a[contains(text(), "签到")]', + '//a/b[contains(text(), "签 到")]', + '//span[@id="sign_in"]/a', + '//a[contains(@href, "addbonus")]', + '//input[@class="dt_button"][contains(@value, "打卡")]', + '//a[contains(@href, "sign_in")]', + '//a[contains(@onclick, "do_signin")]', + '//a[@id="do-attendance"]', + '//shark-icon-button[@href="attendance.php"]' + ] + for xpath in xpaths: + if html.xpath(xpath): + return False + return True + finally: + if html is not None: + del html