From 388564a7230c84b5bbb40ac32d1305709012ce5a Mon Sep 17 00:00:00 2001 From: ngfchl Date: Wed, 30 Nov 2022 02:30:29 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=93=B7=E5=99=A8=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=8E=B7=E5=8F=96=E4=B8=8E=E7=AD=BE=E5=88=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pt.json | 12 +++---- pt_site/UtilityTool.py | 72 ++++++++++++++++++++++++++---------------- pt_site_site.json | 12 +++---- 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/pt.json b/pt.json index e354c46..51cab83 100644 --- a/pt.json +++ b/pt.json @@ -4349,14 +4349,14 @@ "pk": 64, "fields": { "created_at": "2022-10-04T10:55:26.478", - "updated_at": "2022-11-29T22:00:29.937", + "updated_at": "2022-11-30T01:53:33.738", "url": "https://hdchina.org/", "name": "瓷器", "logo": "https://hdchina.org/favicon.ico", "tracker": "http://hdchina.org", - "sign_in_support": false, + "sign_in_support": true, "get_torrent_support": false, - "get_userinfo_support": false, + "get_userinfo_support": true, "search_support": false, "page_default": "torrents.php", "page_sign_in": "plugin_sign-in.php?cmd=signin", @@ -4367,7 +4367,7 @@ "page_search": "torrents.php?search={}", "page_leeching": "getusertorrentlistajax.php?userid={}&type=leeching", "page_uploaded": "getusertorrentlistajax.php?userid={}&type=uploaded", - "page_seeding": "getusertorrentlistajax.php?userid={}&type=seeding", + "page_seeding": "ajax_getusertorrentlist.php?userid={}&type=seeding", "page_completed": "getusertorrentlistajax.php?userid={}&type=completed", "page_mybonus": "mybonus.php", "page_viewfilelist": "viewfilelist.php?id={}", @@ -4375,11 +4375,11 @@ "sign_in_method": "post", "sign_in_captcha": false, "sign_in_params": "{}", - "sign_in_headers": "{}", + "sign_in_headers": "{'connection': 'close'}", "hr": false, "hr_rate": 2, "hr_time": 10, - "sp_full": 100.0, + "sp_full": 70.0, "limit_speed": 100, "torrents_rule": "//table[@class=\"torrents\"]/tr", "name_rule": ".//td[@class=\"embedded\"]/a/b/text()", diff --git a/pt_site/UtilityTool.py b/pt_site/UtilityTool.py index 75b0412..c1522ab 100644 --- a/pt_site/UtilityTool.py +++ b/pt_site/UtilityTool.py @@ -3,6 +3,7 @@ import json import logging import random import re +import ssl import threading import time import traceback @@ -28,6 +29,10 @@ from auto_pt.models import Notify, OCR from pt_site.models import MySite, SignIn, TorrentInfo, SiteStatus, Site from ptools.base import TorrentBaseInfo, PushConfig, CommonResponse, StatusCodeEnum, DownloaderCategory +import urllib3.util.ssl_ + +urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL' + def cookie2dict(source_str: str): """ @@ -187,27 +192,28 @@ class PtSpider: proxies: dict = None): site = my_site.site scraper = self.get_scraper(delay=delay) - self.headers = { + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + _RESTRICTED_SERVER_CIPHERS = 'ALL' + ssl_context.set_ciphers(_RESTRICTED_SERVER_CIPHERS) + ssl_context.check_hostname = False + ssl_context.verify_mode = False + scraper.ssl_context = ssl_context + headers = { 'User-Agent': my_site.user_agent, } for k, v in eval(site.sign_in_headers).items(): - self.headers[k] = v + headers[k] = v # logger.info(self.headers) - - if method.lower() == 'post': - return scraper.post( - url=url, - headers=self.headers, - cookies=self.cookies2dict(my_site.cookie), - data=data, - timeout=timeout, - json=json, - proxies=proxies, - params=params, - ) - return scraper.get( + # if site.url == 'https://hdchina.org/': + # pool = urllib3.HTTPSConnectionPool(host=site.url, port=443, check_hostname=False) + # res = pool.request(method=method, url=url, headers=headers, data=data, params=params) + # logger.info(res) + # return res + # scraper.ssl_context = ssl_ctx + return scraper.request( url=url, - headers=self.headers, + method=method, + headers=headers, cookies=self.cookies2dict(my_site.cookie), data=data, timeout=timeout, @@ -565,7 +571,7 @@ class PtSpider: if len(sign_str) >= 1: return CommonResponse.success(msg=site.name + '已签到，请勿重复操作！！') csrf = ''.join(self.parse(result, '//meta[@name="x-csrf"]/@content')) - logger.info('CSRF字符串{}'.format(csrf)) + logger.info('CSRF字符串：{}'.format(csrf)) sign_res = self.send_request( my_site=my_site, url=site.url + site.page_sign_in, @@ -574,7 +580,7 @@ class PtSpider: 'csrf': csrf } ).json() - logger.info('签到返回结果{}'.format(sign_res)) + logger.info('签到返回结果：{}'.format(sign_res)) if sign_res.get('state') == 'success': msg = "签到成功，您已连续签到{}天，本次增加魔力:{}。".format(sign_res.get('signindays'), sign_res.get('integral')) @@ -1417,14 +1423,26 @@ class PtSpider: if 'totheglory' in site.url: # ttg的信息都是直接加载的，不需要再访问其他网页，直接解析就好 - details_html = etree.HTML(user_detail_res.content) + details_html = etree.HTML(user_detail_res.text) seeding_html = details_html.xpath('//div[@id="ka2"]/table')[0] elif 'greatposterwall' in site.url or 'dicmusic' in site.url: details_html = user_detail_res.json() seeding_html = self.send_request(my_site=my_site, url=site.url + site.page_mybonus).json() + elif 'lemonhd.org' in site.url: + logger.info(site.url) + details_html = etree.HTML(converter.convert(user_detail_res.text)) + seeding_html = details_html else: - details_html = etree.HTML(converter.convert(user_detail_res.content)) - if 'btschool' in site.url: + details_html = etree.HTML(converter.convert(user_detail_res.text)) + if 'hdchina.org' in site.url: + csrf = details_html.xpath('//meta[@name="x-csrf"]/@content') + seeding_detail_res = self.send_request(my_site=my_site, url=seeding_detail_url, method='post', data={ + 'userid': my_site.user_id, + 'type': 'seeding', + 'csrf': ''.join(csrf) + }) + # seeding_html = etree.HTML(converter.convert(seeding_detail_res.text)) + elif 'btschool' in site.url: text = details_html.xpath('//script/text()') logger.info('学校：{}'.format(text)) if len(text) > 0: @@ -1438,19 +1456,16 @@ class PtSpider: except Exception as e: logger.info('BT学校获取做种信息有误！') pass - if 'lemonhd.org' in site.url: - logger.info(site.url) - seeding_html = details_html else: seeding_detail_res = self.send_request(my_site=my_site, url=seeding_detail_url, delay=25) - logger.info('做种信息：{}'.format(seeding_detail_res.content)) + logger.info('做种信息：{}'.format(seeding_detail_res.text)) # leeching_detail_res = self.send_request(my_site=my_site, url=leeching_detail_url, timeout=25) if seeding_detail_res.status_code != 200: return CommonResponse.error( status=StatusCodeEnum.WEB_CONNECT_ERR, msg='{} 做种信息访问错误，错误码：{}'.format(site.name, str(seeding_detail_res.status_code)) ) - seeding_html = etree.HTML(converter.convert(seeding_detail_res.text)) + seeding_html = etree.HTML(converter.convert(seeding_detail_res.text)) # leeching_html = etree.HTML(leeching_detail_res.text) # logger.info(seeding_detail_res.content.decode('utf8')) return CommonResponse.success(data={ @@ -1636,10 +1651,13 @@ class PtSpider: downloaded = ''.join( details_html.xpath(site.downloaded_rule) ).replace(':', '').replace('\xa0\xa0', '').replace('i', '').strip(' ') - downloaded = FileSizeConvert.parse_2_byte(downloaded) uploaded = ''.join( details_html.xpath(site.uploaded_rule) ).replace(':', '').replace('i', '').strip(' ') + if 'hdchina' in site.url: + downloaded = downloaded.split('(')[0].replace(':', '').strip() + uploaded = uploaded.split('(')[0].replace(':', '').strip() + downloaded = FileSizeConvert.parse_2_byte(downloaded) uploaded = FileSizeConvert.parse_2_byte(uploaded) invitation = ''.join( diff --git a/pt_site_site.json b/pt_site_site.json index f3eead9..b87cfa2 100644 --- a/pt_site_site.json +++ b/pt_site_site.json @@ -4182,13 +4182,13 @@ { "id": 64, "created_at": "2022-10-04 10:55:26.478000", - "updated_at": "2022-11-29 22:00:29.937109", + "updated_at": "2022-11-30 01:53:33.738025", "name": "瓷器", "url": "https://hdchina.org/", "logo": "https://hdchina.org/favicon.ico", - "sign_in_support": 0, + "sign_in_support": 1, "get_torrent_support": 0, - "get_userinfo_support": 0, + "get_userinfo_support": 1, "search_support": 0, "page_default": "torrents.php", "page_sign_in": "plugin_sign-in.php?cmd=signin", @@ -4198,7 +4198,7 @@ "page_search": "torrents.php?search={}", "page_leeching": "getusertorrentlistajax.php?userid={}&type=leeching", "page_uploaded": "getusertorrentlistajax.php?userid={}&type=uploaded", - "page_seeding": "getusertorrentlistajax.php?userid={}&type=seeding", + "page_seeding": "ajax_getusertorrentlist.php?userid={}&type=seeding", "page_completed": "getusertorrentlistajax.php?userid={}&type=completed", "page_mybonus": "mybonus.php", "page_viewfilelist": "viewfilelist.php?id={}", @@ -4206,11 +4206,11 @@ "sign_in_method": "post", "sign_in_captcha": 0, "sign_in_params": "{}", - "sign_in_headers": "{}", + "sign_in_headers": "{'connection': 'close'}", "hr": 0, "hr_rate": 2, "hr_time": 10, - "sp_full": 100, + "sp_full": 70, "limit_speed": 100, "torrents_rule": "//table[@class=\"torrents\"]/tr", "name_rule": ".//td[@class=\"embedded\"]/a/b/text()",