调整:我的站点部分字段调整为每日获取记录,便于生成图表

优化:部分页面显示
This commit is contained in:
ngfchl
2023-01-19 11:25:36 +08:00
parent 3991abb254
commit 292afe324e
8 changed files with 174 additions and 73 deletions

View File

@@ -538,7 +538,7 @@ def site_status_api(request):
sign_in_support = site.sign_in_support and my_site.sign_in
if len(site_info_list) <= 0:
logger.info(f'{site.name}: 获取站点信息列表错误,站点尚未获取过数据!')
site_info = {
out_site_info = {
'id': my_site.id,
'name': site.name,
'icon': site.logo,
@@ -546,7 +546,7 @@ def site_status_api(request):
'class': my_site.my_level,
'sign_in_support': sign_in_support,
'sign_in_state': False,
'invite': my_site.invitation,
'invite': 0,
'sp_hour': 0,
'sp_hour_full': '0',
'seeding': 0,
@@ -571,12 +571,12 @@ def site_status_api(request):
site_info = site_info_list.first()
downloaded += site_info.downloaded
uploaded += site_info.uploaded
seeding += my_site.seed
leeching += my_site.leech
seeding += site_info.seed
leeching += site_info.leech
sp += site_info.my_sp
sp_hour += (float(my_site.sp_hour) if my_site.sp_hour != '' else 0)
sp_hour += (float(site_info.sp_hour) if site_info.sp_hour != '' else 0)
bonus += site_info.my_bonus
leeching += my_site.leech
leeching += site_info.leech
seeding_size += site_info.seed_vol
weeks = ((now - my_site.time_join).days // 7) if my_site.time_join else 0
days = ((now - my_site.time_join).days % 7) if my_site.time_join else 0
@@ -586,7 +586,7 @@ def site_status_api(request):
sign_in_state = sign_in_list.first().sign_in_today if len(sign_in_list) > 0 else False
else:
sign_in_state = False
site_info = {
out_site_info = {
'id': my_site.id,
'name': site.name,
'icon': site.logo,
@@ -595,12 +595,12 @@ def site_status_api(request):
'class': my_site.my_level,
'sign_in_support': sign_in_support,
'sign_in_state': sign_in_state,
'invite': my_site.invitation,
'sp_hour': float(my_site.sp_hour) if my_site.sp_hour != '' else 0,
'invite': site_info.invitation,
'sp_hour': float(site_info.sp_hour) if site_info.sp_hour != '' else 0,
'sp_hour_full': '{:.2%}'.format(
float(my_site.sp_hour) / site.sp_full) if site.sp_full != 0 else '0%',
'seeding': my_site.seed,
'leeching': my_site.leech,
float(site_info.sp_hour) / site.sp_full) if site.sp_full != 0 else '0%',
'seeding': site_info.seed,
'leeching': site_info.leech,
'weeks': f'{weeks}{days}',
'time_join': my_site.time_join if my_site.time_join else now,
'hr': my_site.my_hr,
@@ -653,12 +653,12 @@ def site_status_api(request):
}
)
logger.info(f'我的站点id{my_site.id}')
logger.info(f'当前等级:{level_info_dict}')
logger.info(f'下一等级:{next_level_dict}')
# logger.info(f'当前等级:{level_info_dict}')
# logger.info(f'下一等级:{next_level_dict}')
upgrade_day = my_site.time_join + timedelta(days=next_level.days * 7)
logger.info(f'下一等级升级日期:{upgrade_day}')
site_info.update({
out_site_info.update({
'level_info': level_info_dict,
'next_level': next_level_dict,
'upgrade_day': upgrade_day if upgrade_day > datetime.today() else False,
@@ -671,7 +671,7 @@ def site_status_api(request):
logger.info(message)
logger.error(traceback.format_exc(limit=3))
continue
status_list.append(site_info)
status_list.append(out_site_info)
# 按上传量排序
# status_list.sort(key=lambda x: x['mail'], reverse=False)
@@ -730,20 +730,20 @@ def site_data_api(request):
])
date_list = list(date_list)
date_list.sort()
print(f'日期列表:{date_list}')
# print(f'日期列表:{date_list}')
print(f'日期数量:{len(date_list)}')
for my_site in my_site_list:
# 每个站点获取自己站点的所有信息
site_status_list = my_site.sitestatus_set.order_by('created_at').all()
print(f'站点数据条数:{len(site_status_list)}')
# print(f'站点数据条数:{len(site_status_list)}')
info_list = [
{
'uploaded': site_info.uploaded,
'date': site_info.created_at.date().strftime('%Y-%m-%d')
} for site_info in site_status_list
]
print(f'提取完后站点数据条数:{len(info_list)}')
# print(f'提取完后站点数据条数:{len(info_list)}')
# 生成本站点的增量列表,并标注时间
'''
@@ -762,12 +762,12 @@ def site_data_api(request):
(index, info) in enumerate(info_list) if 0 < index < len(info_list)
}
print(f'处理完后站点数据条数:{len(info_list)}')
# print(f'处理完后站点数据条数:{len(info_list)}')
for date in date_list:
if not diff_info_list.get(date):
diff_info_list[date] = 0
# print(diff_info_list)
print(len(diff_info_list))
# print(len(diff_info_list))
diff_info_list = sorted(diff_info_list.items(), key=lambda x: x[0])
diff_list.append({
'name': my_site.site.name,
@@ -776,7 +776,7 @@ def site_data_api(request):
'stack': 'increment',
'data': [value[1] if value[1] > 0 else 0 for value in diff_info_list]
})
print(diff_list)
# print(diff_list)
return JsonResponse(data=CommonResponse.success(
data={
@@ -792,7 +792,7 @@ def site_data_api(request):
msg='访问出错咯!'
).to_dict(), safe=False)
site_info_list = my_site.sitestatus_set.order_by('created_at').all()
logger.info(site_info_list)
# logger.info(site_info_list)
site_status_list = []
site = {
'id': my_site.id,
@@ -800,8 +800,6 @@ def site_data_api(request):
'icon': my_site.site.logo,
'url': my_site.site.url,
'class': my_site.my_level,
'seeding': my_site.seed,
'leeching': my_site.leech,
'last_active': datetime.strftime(my_site.updated_at, '%Y/%m/%d %H:%M:%S'),
}
for site_info in site_info_list:
@@ -812,11 +810,13 @@ def site_data_api(request):
'seedingSize': site_info.seed_vol,
'sp': site_info.my_sp,
'bonus': site_info.my_bonus,
'seeding': site_info.seed,
'leeching': site_info.leech,
'info_date': site_info.created_at.date()
}
site_status_list.append(my_site_status)
logger.info(site)
logger.info(site_status_list)
# logger.info(site_status_list)
return JsonResponse(data=CommonResponse.success(
data={
'site': site,
@@ -873,15 +873,15 @@ def update_site_api(request):
my_site.site.name,
my_site.my_level,
status.my_sp,
my_site.sp_hour,
status.sp_hour,
status.my_bonus,
status.ratio,
FileSizeConvert.parse_2_file_size(status.seed_vol),
FileSizeConvert.parse_2_file_size(status.uploaded),
FileSizeConvert.parse_2_file_size(status.downloaded),
my_site.seed,
my_site.leech,
my_site.invitation,
status.seed,
status.leech,
status.invitation,
my_site.my_hr
)
return JsonResponse(data=CommonResponse.success(
@@ -1077,7 +1077,8 @@ def edit_my_site(request):
my_site_list = MySite.objects.filter(id=my_site_id)
if len(my_site_list) == 1:
my_site = my_site_list.values(
'id', 'site', 'sign_in', 'hr', 'search', 'user_id', 'passkey', 'user_agent', 'cookie', 'time_join'
'id', 'site', 'sign_in', 'get_info', 'hr', 'search', 'user_id', 'passkey', 'user_agent', 'cookie',
'time_join'
).first()
return JsonResponse(CommonResponse.success(data={
'my_site': my_site

View File

@@ -1798,8 +1798,8 @@ class PtSpider:
my_site.latest_active = stats.get('lastAccess')
my_site.my_level = details_response.get('personal').get('class')
community = details_response.get('community')
my_site.seed = community.get('seeding')
my_site.leech = community.get('leeching')
seed = community.get('seeding')
leech = community.get('leeching')
# ajax.php?action=index
if 'greatposterwall' in site.url:
userdata = seeding_response.get('userstats')
@@ -1807,7 +1807,7 @@ class PtSpider:
# if userdata.get('bonusPoints') else 0
seeding_size = userdata.get('seedingSize')
# if userdata.get('seedingSize') else 0
my_site.sp_hour = userdata.get('seedingBonusPointsPerHour')
sp_hour = userdata.get('seedingBonusPointsPerHour')
# if userdata.get('seedingBonusPointsPerHour') else 0
if 'dicmusic' in site.url:
logger.info('海豚')
@@ -1822,7 +1822,7 @@ class PtSpider:
seeding_size = FileSizeConvert.parse_2_byte(''.join(seeding_size_str))
"""
my_sp = 0
my_site.sp_hour = 0
sp_hour = 0
seeding_size = 0
my_site.save()
res_gpw = SiteStatus.objects.update_or_create(
@@ -1836,6 +1836,9 @@ class PtSpider:
'my_bonus': 0,
# 做种体积
'seed_vol': seeding_size,
'seed': seed,
'leech': leech,
'sp_hour': sp_hour,
})
if float(ratio) < 1:
msg = f'{site.name} 分享率 {ratio} 过低,请注意'
@@ -1860,12 +1863,12 @@ class PtSpider:
my_sp = details_html.get(site.my_sp_rule)
ratio = uploaded / downloaded if downloaded > 0 else 'inf'
my_site.time_join = datetime.fromtimestamp(details_html.get(site.time_join_rule))
my_site.invitation = details_html.get(site.invitation_rule)
invitation = details_html.get(site.invitation_rule)
my_site.my_level = details_html.get('class').get('name')
my_site.seed = details_html.get(site.seed_rule)
my_site.leech = details_html.get(site.leech_rule)
seed = details_html.get(site.seed_rule)
leech = details_html.get(site.leech_rule)
my_site.mail = details_html.get(site.mailbox_rule)
my_site.sp_hour = details_html.get(site.hour_sp_rule)
sp_hour = details_html.get(site.hour_sp_rule)
my_site.save()
res_gpw = SiteStatus.objects.update_or_create(
site=my_site,
@@ -1876,6 +1879,10 @@ class PtSpider:
'uploaded': uploaded,
'my_sp': my_sp,
'my_bonus': 0,
'invitation': invitation,
'seed': seed,
'leech': leech,
'sp_hour': sp_hour,
# 做种体积
'seed_vol': seeding_size,
})
@@ -2128,34 +2135,34 @@ class PtSpider:
hr = ''.join(details_html.xpath(site.my_hr_rule)).replace('H&R:', '').replace('有效\n:', '').strip()
my_hr = hr if hr else '0'
logger.info(f'h&r: {hr} ,解析后:{my_hr}')
logger.info(f'h&r: "{hr}" ,解析后:{my_hr}')
# logger.info(my_bonus)
# 更新我的站点数据
# invitation = converter.convert(invitation)
# x = invitation.split('/')
# invitation = re.sub('[\u4e00-\u9fa5]', '', invitation)
logger.info(f'当前获取邀请数:{invitation}')
if invitation == '没有邀请资格' or invitation == '沒有邀請資格':
my_site.invitation = 0
logger.info(f'当前获取邀请数:"{invitation}"')
if '没有邀请资格' in invitation or '沒有邀請資格' in invitation:
invitation = 0
elif '/' in invitation:
invitation_list = [int(n) for n in invitation.split('/')]
# my_site.invitation = int(invitation) if invitation else 0
my_site.invitation = sum(invitation_list)
invitation = sum(invitation_list)
elif '(' in invitation:
invitation_list = [int(get_decimals(n)) for n in invitation.split('(')]
# my_site.invitation = int(invitation) if invitation else 0
my_site.invitation = sum(invitation_list)
invitation = sum(invitation_list)
elif not invitation:
my_site.invitation = 0
invitation = 0
else:
my_site.invitation = int(re.sub('\D', '', invitation))
invitation = int(re.sub('\D', '', invitation))
my_site.latest_active = datetime.now()
my_site.my_level = my_level if my_level != '' else ' '
if my_hr:
my_site.my_hr = my_hr
my_site.seed = int(get_decimals(seed)) if seed else 0
seed = int(get_decimals(seed)) if seed else 0
logger.info(f'当前下载数:{leech}')
my_site.leech = int(get_decimals(leech)) if leech else 0
leech = int(get_decimals(leech)) if leech else 0
logger.info('站点:{}'.format(site))
logger.info('等级:{}'.format(my_level))
@@ -2312,7 +2319,7 @@ class PtSpider:
# logger.info(details_html)
# logger.info(res_sp_hour_list)
res_sp_hour = ''.join(res_sp_hour_list)
my_site.sp_hour = get_decimals(res_sp_hour)
sp_hour = get_decimals(res_sp_hour)
# 飞天邀请获取
logger.info(f'邀请页面:{site.url}Invites')
res_next_pt_invite = self.send_request(my_site, f'{site.url}Invites')
@@ -2324,13 +2331,13 @@ class PtSpider:
print(f'邀请字符串:{str_next_pt_invite}')
list_next_pt_invite = re.findall('\d+', str_next_pt_invite)
print(list_next_pt_invite)
my_site.invitation = int(list_next_pt_invite[0]) - int(list_next_pt_invite[1])
invitation = int(list_next_pt_invite[0]) - int(list_next_pt_invite[1])
else:
res_sp_hour = self.get_hour_sp(my_site=my_site)
if res_sp_hour.code != StatusCodeEnum.OK.code:
logger.error(my_site.site.name + res_sp_hour.msg)
else:
my_site.sp_hour = res_sp_hour.data
sp_hour = res_sp_hour.data
# 保存上传下载等信息
my_site.save()
# 外键反向查询
@@ -2347,6 +2354,11 @@ class PtSpider:
my_bonus) if my_bonus != '' else 0,
# 做种体积
'seed_vol': seed_vol_all,
'seed': seed,
'leech': leech,
'sp_hour': sp_hour,
'invitation': invitation,
'publish': 0,
})
# logger.info(result) # result 本身就是元祖
return CommonResponse.success(data=result)

View File

@@ -314,9 +314,11 @@ class MySiteAdmin(AjaxAdmin): # instead of ModelAdmin
'<font color="#6c4c49">魔力:{}</font><br>'
'<font color="#708090">满魔:{} </font>'
'<font color="#708090">积分/HP{}</font>',
round(float(obj.sp_hour), 3) if obj.sp_hour else 0,
round(float(obj.sitestatus_set.order_by('id').first().sp_hour), 3) if obj.sitestatus_set.order_by(
'id').first().sp_hour else 0,
status_today.my_sp if status_today else 0,
'{:.2%}'.format(float(obj.sp_hour) / obj.site.sp_full) if status_today and obj.site.sp_full != 0 else 0,
'{:.2%}'.format(float(obj.sitestatus_set.order_by(
'id').first().sp_hour) / obj.site.sp_full) if status_today and obj.site.sp_full != 0 else 0,
status_today.my_bonus if status_today else 0
)
@@ -328,7 +330,7 @@ class MySiteAdmin(AjaxAdmin): # instead of ModelAdmin
'<font color="#6c4c49">邀请:{}</font><br>'
'<font color="Tomato">H&R{}</font>',
obj.my_level,
obj.invitation,
obj.sitestatus_set.order_by('id').first().invitation,
obj.my_hr if obj.my_hr else 0
)
@@ -349,7 +351,8 @@ class MySiteAdmin(AjaxAdmin): # instead of ModelAdmin
return format_html(
'<font color="#2570a1">做种:{}</font> / <font color="#6c4c49">下载:{} </font><br>'
'<font color="#708090"> 做种体积:{}</font>',
obj.seed, obj.leech,
status_today.seed,
status_today.leech,
FileSizeConvert.parse_2_file_size(status_today.seed_vol) if status_today else 0,
)
@@ -667,7 +670,7 @@ class MySiteAdmin(AjaxAdmin): # instead of ModelAdmin
['用户信息', {
'classes': ('collapse',), # CSS
'fields': (
('site', 'sign_in', 'hr', 'search'),
('site', 'sign_in', 'get_info', 'hr', 'search'),
('user_id', 'passkey',),
('time_join',),
'user_agent',

View File

@@ -0,0 +1,50 @@
# Generated by Django 4.1.2 on 2023-01-19 09:36
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("pt_site", "0037_alter_site_nickname"),
]
operations = [
migrations.RemoveField(model_name="mysite", name="invitation",),
migrations.RemoveField(model_name="mysite", name="leech",),
migrations.RemoveField(model_name="mysite", name="publish",),
migrations.RemoveField(model_name="mysite", name="seed",),
migrations.RemoveField(model_name="mysite", name="sp_hour",),
migrations.AddField(
model_name="mysite",
name="get_info",
field=models.BooleanField(
default=True, help_text="是否抓取站点数据", verbose_name="抓取信息"
),
),
migrations.AddField(
model_name="sitestatus",
name="invitation",
field=models.IntegerField(default=0, verbose_name="邀请资格"),
),
migrations.AddField(
model_name="sitestatus",
name="leech",
field=models.IntegerField(default=0, verbose_name="当前下载"),
),
migrations.AddField(
model_name="sitestatus",
name="publish",
field=models.IntegerField(default=0, verbose_name="发布种子"),
),
migrations.AddField(
model_name="sitestatus",
name="seed",
field=models.IntegerField(default=0, verbose_name="当前做种"),
),
migrations.AddField(
model_name="sitestatus",
name="sp_hour",
field=models.FloatField(default=0, verbose_name="时魔"),
),
]

View File

@@ -0,0 +1,22 @@
# Generated by Django 4.1.2 on 2023-01-19 10:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("pt_site", "0038_remove_mysite_invitation_remove_mysite_leech_and_more"),
]
operations = [
migrations.AddField(
model_name="site",
name="publish_rule",
field=models.CharField(
default="//p/preceding-sibling::b/text()[1]",
max_length=128,
verbose_name="发种数量",
),
),
]

View File

@@ -218,6 +218,10 @@ class Site(BaseEntity):
default='//img[@class="arrowdown"]/following-sibling::text()[1]',
max_length=128)
publish_rule = models.CharField(verbose_name='发种数量',
default='//p/preceding-sibling::b/text()[1]',
max_length=128)
seed_rule = models.CharField(verbose_name='做种数量',
default='//img[@class="arrowup"]/following-sibling::text()[1]',
max_length=128)
@@ -270,7 +274,7 @@ class Site(BaseEntity):
class UserLevelRule(BaseEntity):
site = models.ForeignKey(verbose_name='站 点', to=Site, to_field='url',on_delete=models.CASCADE)
site = models.ForeignKey(verbose_name='站 点', to=Site, to_field='url', on_delete=models.CASCADE)
level_id = models.IntegerField(verbose_name='等级id', default=1)
level = models.CharField(verbose_name='等 级', default='User', max_length=24, help_text='请去除空格')
days = models.IntegerField(verbose_name='时 间', default=0, help_text='原样输入,单位:周')
@@ -309,18 +313,20 @@ class MySite(BaseEntity):
# 用户设置
hr = models.BooleanField(verbose_name='开启HR下载', default=False, help_text='是否下载HR种子')
sign_in = models.BooleanField(verbose_name='开启签到', default=True, help_text='是否开启签到')
get_info = models.BooleanField(verbose_name='抓取信息', default=True, help_text='是否抓取站点数据')
search = models.BooleanField(verbose_name='开启搜索', default=True, help_text='是否开启搜索')
# 用户数据 自动拉取
invitation = models.IntegerField(verbose_name='邀请资格', default=0)
# invitation = models.IntegerField(verbose_name='邀请资格', default=0)
time_join = models.DateTimeField(verbose_name='注册时间', blank=True, null=True, help_text='请务必填写此项!')
latest_active = models.DateTimeField(verbose_name='最近活动时间', blank=True, null=True)
sp_hour = models.FloatField(verbose_name='时魔', default=0)
# sp_hour = models.FloatField(verbose_name='时魔', default=0)
my_level = models.CharField(verbose_name='用户等级', max_length=16, default='')
my_hr = models.CharField(verbose_name='H&R', max_length=16, default='')
leech = models.IntegerField(verbose_name='当前下载', default=0)
seed = models.IntegerField(verbose_name='当前做种', default=0)
# leech = models.IntegerField(verbose_name='当前下载', default=0)
# seed = models.IntegerField(verbose_name='当前做种', default=0)
mail = models.IntegerField(verbose_name='新邮件', default=0)
publish = models.IntegerField(verbose_name='发布种子', default=0)
# publish = models.IntegerField(verbose_name='发布种子', default=0)
def __str__(self):
return self.site.name
@@ -341,6 +347,11 @@ class SiteStatus(BaseEntity):
my_sp = models.FloatField(verbose_name='魔力值', default=0)
my_bonus = models.FloatField(verbose_name='做种积分', default=0)
seed_vol = models.IntegerField(verbose_name='做种体积', default=0)
leech = models.IntegerField(verbose_name='当前下载', default=0)
seed = models.IntegerField(verbose_name='当前做种', default=0)
sp_hour = models.FloatField(verbose_name='时魔', default=0)
publish = models.IntegerField(verbose_name='发布种子', default=0)
invitation = models.IntegerField(verbose_name='邀请资格', default=0)
class Meta:
verbose_name = '我的数据'

View File

@@ -78,15 +78,15 @@ def auto_get_status():
my_site.site.name,
my_site.my_level,
status.my_sp,
my_site.sp_hour,
status.sp_hour,
status.my_bonus,
status.ratio,
FileSizeConvert.parse_2_file_size(status.seed_vol),
FileSizeConvert.parse_2_file_size(status.uploaded),
FileSizeConvert.parse_2_file_size(status.downloaded),
my_site.seed,
my_site.leech,
my_site.invitation,
status.seed,
status.leech,
status.invitation,
my_site.my_hr
)
logger.info('组装Message{}'.format(message))

View File

@@ -530,7 +530,7 @@
<div style="margin-bottom: 5px;">
<el-radio-group v-model="dataLength" size="mini"
{# @change="showAllIncrementHistory">#}
@change="showHistoryList(site.id)">
@change="showHistoryList(showAllHistory ? 0 : site.id)">
<el-radio-button label="-7">7天</el-radio-button>
<el-radio-button label="-15">15天</el-radio-button>
<el-radio-button label="-30">30天</el-radio-button>
@@ -592,9 +592,10 @@
</el-form-item>
<el-form-item label="功能开关" label-position="right" required>
<el-radio-group>
<el-checkbox-button v-model="addMySiteForm.sign_in" label="签 到" border></el-checkbox-button>
<el-checkbox-button v-model="addMySiteForm.get_info" label="站点信息" border></el-checkbox-button>
<el-checkbox-button v-model="addMySiteForm.sign_in" label="站点签到" border></el-checkbox-button>
<el-checkbox-button v-model="addMySiteForm.hr" label="HR下载" border></el-checkbox-button>
<el-checkbox-button v-model="addMySiteForm.search" label="搜 索" border></el-checkbox-button>
<el-checkbox-button v-model="addMySiteForm.search" label="聚合搜索" border></el-checkbox-button>
</el-radio-group>
</el-form-item>
@@ -640,7 +641,7 @@
</el-popconfirm>
<el-button @click="showAddMySite = false">取 消</el-button>
<el-button type="primary" @click="editMySite('addMySiteForm')">
<span v-text="addMySiteForm.id==0 ? '添加站点' : '编辑站点'"></span>
<span v-text="addMySiteForm.id==0 ? '添加站点' : '保存站点'"></span>
</el-button>
</div>
</el-dialog>
@@ -665,7 +666,7 @@
showSiteChart: false,
begInvite: false,
showLogo: true,
showLegend: true,
showLegend: window.innerWidth > 1200 ? true : false,
shuffle: false,
searchKey: '',
sortStep: 1,
@@ -680,6 +681,7 @@
addMySiteForm: {
id: 0,
site: null,
get_info: true,
sign_in: true,
hr: false,
search: true,
@@ -697,7 +699,7 @@
{required: true, message: '请输入数字UID指定站点请输入用户名', trigger: 'blur'},
],
passkey: [
{required: true, message: '请输入Passkey', trigger: 'blur'},
{required: false, message: '请输入Passkey', trigger: 'blur'},
{min: 16, max: 32, message: '长度为32位', trigger: 'blur'}
],
cookie: [