diff --git a/app/modules/indexer/parser/__init__.py b/app/modules/indexer/parser/__init__.py index 1a17fd58..55ba05e8 100644 --- a/app/modules/indexer/parser/__init__.py +++ b/app/modules/indexer/parser/__init__.py @@ -156,53 +156,57 @@ class SiteParserBase(metaclass=ABCMeta): 解析站点信息 :return: """ - # Cookie模式时,获取站点首页html - if self.request_mode == "apikey": - if not self.apikey and not self.token: - logger.warn(f"{self._site_name} 未设置cookie 或 apikey/token,跳过后续操作") - return - self._index_html = {} - else: - # 检查是否已经登录 - self._index_html = self._get_page_content(url=self._site_url) - if not self._parse_logged_in(self._index_html): - return - # 解析站点页面 - self._parse_site_page(self._index_html) - # 解析用户基础信息 - if self._user_basic_page: - self._parse_user_base_info( - self._get_page_content( - url=urljoin(self._base_url, self._user_basic_page), - params=self._user_basic_params, - headers=self._user_basic_headers + try: + # Cookie模式时,获取站点首页html + if self.request_mode == "apikey": + if not self.apikey and not self.token: + logger.warn(f"{self._site_name} 未设置cookie 或 apikey/token,跳过后续操作") + return + self._index_html = {} + else: + # 检查是否已经登录 + self._index_html = self._get_page_content(url=self._site_url) + if not self._parse_logged_in(self._index_html): + return + # 解析站点页面 + self._parse_site_page(self._index_html) + # 解析用户基础信息 + if self._user_basic_page: + self._parse_user_base_info( + self._get_page_content( + url=urljoin(self._base_url, self._user_basic_page), + params=self._user_basic_params, + headers=self._user_basic_headers + ) ) - ) - else: - self._parse_user_base_info(self._index_html) - # 解析用户详细信息 - if self._user_detail_page: - self._parse_user_detail_info( - self._get_page_content( - url=urljoin(self._base_url, self._user_detail_page), - params=self._user_detail_params, - headers=self._user_detail_headers + else: + self._parse_user_base_info(self._index_html) + # 解析用户详细信息 + if self._user_detail_page: + self._parse_user_detail_info( + self._get_page_content( + url=urljoin(self._base_url, self._user_detail_page), + params=self._user_detail_params, + headers=self._user_detail_headers + ) ) - ) - # 解析用户未读消息 - if settings.SITE_MESSAGE: - self._pase_unread_msgs() - # 解析用户上传、下载、分享率等信息 - if self._user_traffic_page: - self._parse_user_traffic_info( - self._get_page_content( - url=urljoin(self._base_url, self._user_traffic_page), - params=self._user_traffic_params, - headers=self._user_traffic_headers + # 解析用户未读消息 + if settings.SITE_MESSAGE: + self._pase_unread_msgs() + # 解析用户上传、下载、分享率等信息 + if self._user_traffic_page: + self._parse_user_traffic_info( + self._get_page_content( + url=urljoin(self._base_url, self._user_traffic_page), + params=self._user_traffic_params, + headers=self._user_traffic_headers + ) ) - ) - # 解析用户做种信息 - self._parse_seeding_pages() + # 解析用户做种信息 + self._parse_seeding_pages() + finally: + # 关闭连接 + self.close() def _pase_unread_msgs(self): """ @@ -430,6 +434,14 @@ class SiteParserBase(metaclass=ABCMeta): """ pass + def close(self): + """ + 关闭会话 + """ + if self._session: + self._session.close() + self._session = None + def to_dict(self): """ 转化为字典