nas/ptools
1
0
Fork 0
mirror of https://github.com/ngfchl/ptools synced 2023-07-10 13:41:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化瓷器数据获取与签到

Browse Source
This commit is contained in:
ngfchl
2022-11-30 02:30:29 +08:00
parent a2f3370738
commit 388564a723
3 changed files with 57 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pt.json
View File

@@ -4349,14 +4349,14 @@
"pk": 64,
"fields": {
"created_at": "2022-10-04T10:55:26.478",
"updated_at": "2022-11-29T22:00:29.937",
"updated_at": "2022-11-30T01:53:33.738",
"url": "https://hdchina.org/",
"name": "瓷器",
"logo": "https://hdchina.org/favicon.ico",
"tracker": "http://hdchina.org",
"sign_in_support": false,
"sign_in_support": true,
"get_torrent_support": false,
"get_userinfo_support": false,
"get_userinfo_support": true,
"search_support": false,
"page_default": "torrents.php",
"page_sign_in": "plugin_sign-in.php?cmd=signin",
@@ -4367,7 +4367,7 @@
"page_search": "torrents.php?search={}",
"page_leeching": "getusertorrentlistajax.php?userid={}&type=leeching",
"page_uploaded": "getusertorrentlistajax.php?userid={}&type=uploaded",
"page_seeding": "getusertorrentlistajax.php?userid={}&type=seeding",
"page_seeding": "ajax_getusertorrentlist.php?userid={}&type=seeding",
"page_completed": "getusertorrentlistajax.php?userid={}&type=completed",
"page_mybonus": "mybonus.php",
"page_viewfilelist": "viewfilelist.php?id={}",
@@ -4375,11 +4375,11 @@
"sign_in_method": "post",
"sign_in_captcha": false,
"sign_in_params": "{}",
"sign_in_headers": "{}",
"sign_in_headers": "{'connection': 'close'}",
"hr": false,
"hr_rate": 2,
"hr_time": 10,
"sp_full": 100.0,
"sp_full": 70.0,
"limit_speed": 100,
"torrents_rule": "//table[@class=\"torrents\"]/tr",
"name_rule": ".//td[@class=\"embedded\"]/a/b/text()",

72
pt_site/UtilityTool.py
View File

@@ -3,6 +3,7 @@ import json
import logging
import random
import re
import ssl
import threading
import time
import traceback
@@ -28,6 +29,10 @@ from auto_pt.models import Notify, OCR
from pt_site.models import MySite, SignIn, TorrentInfo, SiteStatus, Site
from ptools.base import TorrentBaseInfo, PushConfig, CommonResponse, StatusCodeEnum, DownloaderCategory
import urllib3.util.ssl_
urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL'
def cookie2dict(source_str: str):
"""
@@ -187,27 +192,28 @@ class PtSpider:
proxies: dict = None):
site = my_site.site
scraper = self.get_scraper(delay=delay)
self.headers = {
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
_RESTRICTED_SERVER_CIPHERS = 'ALL'
ssl_context.set_ciphers(_RESTRICTED_SERVER_CIPHERS)
ssl_context.check_hostname = False
ssl_context.verify_mode = False
scraper.ssl_context = ssl_context
headers = {
'User-Agent': my_site.user_agent,
}
for k, v in eval(site.sign_in_headers).items():
self.headers[k] = v
headers[k] = v
# logger.info(self.headers)
if method.lower() == 'post':
return scraper.post(
url=url,
headers=self.headers,
cookies=self.cookies2dict(my_site.cookie),
data=data,
timeout=timeout,
json=json,
proxies=proxies,
params=params,
)
return scraper.get(
# if site.url == 'https://hdchina.org/':
# pool = urllib3.HTTPSConnectionPool(host=site.url, port=443, check_hostname=False)
# res = pool.request(method=method, url=url, headers=headers, data=data, params=params)
# logger.info(res)
# return res
# scraper.ssl_context = ssl_ctx
return scraper.request(
url=url,
headers=self.headers,
method=method,
headers=headers,
cookies=self.cookies2dict(my_site.cookie),
data=data,
timeout=timeout,
@@ -565,7 +571,7 @@ class PtSpider:
if len(sign_str) >= 1:
return CommonResponse.success(msg=site.name + '已签到，请勿重复操作！！')
csrf = ''.join(self.parse(result, '//meta[@name="x-csrf"]/@content'))
logger.info('CSRF字符串{}'.format(csrf))
logger.info('CSRF字符串：{}'.format(csrf))
sign_res = self.send_request(
my_site=my_site,
url=site.url + site.page_sign_in,
@@ -574,7 +580,7 @@ class PtSpider:
'csrf': csrf
}
).json()
logger.info('签到返回结果{}'.format(sign_res))
logger.info('签到返回结果：{}'.format(sign_res))
if sign_res.get('state') == 'success':
msg = "签到成功，您已连续签到{}天，本次增加魔力:{}。".format(sign_res.get('signindays'),
sign_res.get('integral'))
@@ -1417,14 +1423,26 @@ class PtSpider:
if 'totheglory' in site.url:
# ttg的信息都是直接加载的，不需要再访问其他网页，直接解析就好
details_html = etree.HTML(user_detail_res.content)
details_html = etree.HTML(user_detail_res.text)
seeding_html = details_html.xpath('//div[@id="ka2"]/table')[0]
elif 'greatposterwall' in site.url or 'dicmusic' in site.url:
details_html = user_detail_res.json()
seeding_html = self.send_request(my_site=my_site, url=site.url + site.page_mybonus).json()
elif 'lemonhd.org' in site.url:
logger.info(site.url)
details_html = etree.HTML(converter.convert(user_detail_res.text))
seeding_html = details_html
else:
details_html = etree.HTML(converter.convert(user_detail_res.content))
if 'btschool' in site.url:
details_html = etree.HTML(converter.convert(user_detail_res.text))
if 'hdchina.org' in site.url:
csrf = details_html.xpath('//meta[@name="x-csrf"]/@content')
seeding_detail_res = self.send_request(my_site=my_site, url=seeding_detail_url, method='post', data={
'userid': my_site.user_id,
'type': 'seeding',
'csrf': ''.join(csrf)
})
# seeding_html = etree.HTML(converter.convert(seeding_detail_res.text))
elif 'btschool' in site.url:
text = details_html.xpath('//script/text()')
logger.info('学校：{}'.format(text))
if len(text) > 0:
@@ -1438,19 +1456,16 @@ class PtSpider:
except Exception as e:
logger.info('BT学校获取做种信息有误！')
pass
if 'lemonhd.org' in site.url:
logger.info(site.url)
seeding_html = details_html
else:
seeding_detail_res = self.send_request(my_site=my_site, url=seeding_detail_url, delay=25)
logger.info('做种信息：{}'.format(seeding_detail_res.content))
logger.info('做种信息：{}'.format(seeding_detail_res.text))
# leeching_detail_res = self.send_request(my_site=my_site, url=leeching_detail_url, timeout=25)
if seeding_detail_res.status_code != 200:
return CommonResponse.error(
status=StatusCodeEnum.WEB_CONNECT_ERR,
msg='{} 做种信息访问错误，错误码：{}'.format(site.name, str(seeding_detail_res.status_code))
)
seeding_html = etree.HTML(converter.convert(seeding_detail_res.text))
seeding_html = etree.HTML(converter.convert(seeding_detail_res.text))
# leeching_html = etree.HTML(leeching_detail_res.text)
# logger.info(seeding_detail_res.content.decode('utf8'))
return CommonResponse.success(data={
@@ -1636,10 +1651,13 @@ class PtSpider:
downloaded = ''.join(
details_html.xpath(site.downloaded_rule)
).replace(':', '').replace('\xa0\xa0', '').replace('i', '').strip(' ')
downloaded = FileSizeConvert.parse_2_byte(downloaded)
uploaded = ''.join(
details_html.xpath(site.uploaded_rule)
).replace(':', '').replace('i', '').strip(' ')
if 'hdchina' in site.url:
downloaded = downloaded.split('(')[0].replace(':', '').strip()
uploaded = uploaded.split('(')[0].replace(':', '').strip()
downloaded = FileSizeConvert.parse_2_byte(downloaded)
uploaded = FileSizeConvert.parse_2_byte(uploaded)
invitation = ''.join(

12
pt_site_site.json
View File

@@ -4182,13 +4182,13 @@
{
"id": 64,
"created_at": "2022-10-04 10:55:26.478000",
"updated_at": "2022-11-29 22:00:29.937109",
"updated_at": "2022-11-30 01:53:33.738025",
"name": "瓷器",
"url": "https://hdchina.org/",
"logo": "https://hdchina.org/favicon.ico",
"sign_in_support": 0,
"sign_in_support": 1,
"get_torrent_support": 0,
"get_userinfo_support": 0,
"get_userinfo_support": 1,
"search_support": 0,
"page_default": "torrents.php",
"page_sign_in": "plugin_sign-in.php?cmd=signin",
@@ -4198,7 +4198,7 @@
"page_search": "torrents.php?search={}",
"page_leeching": "getusertorrentlistajax.php?userid={}&type=leeching",
"page_uploaded": "getusertorrentlistajax.php?userid={}&type=uploaded",
"page_seeding": "getusertorrentlistajax.php?userid={}&type=seeding",
"page_seeding": "ajax_getusertorrentlist.php?userid={}&type=seeding",
"page_completed": "getusertorrentlistajax.php?userid={}&type=completed",
"page_mybonus": "mybonus.php",
"page_viewfilelist": "viewfilelist.php?id={}",
@@ -4206,11 +4206,11 @@
"sign_in_method": "post",
"sign_in_captcha": 0,
"sign_in_params": "{}",
"sign_in_headers": "{}",
"sign_in_headers": "{'connection': 'close'}",
"hr": 0,
"hr_rate": 2,
"hr_time": 10,
"sp_full": 100,
"sp_full": 70,
"limit_speed": 100,
"torrents_rule": "//table[@class=\"torrents\"]/tr",
"name_rule": ".//td[@class=\"embedded\"]/a/b/text()",