From 99571c56a4f51e27a25534f0abc0b5c0c5d0cb57 Mon Sep 17 00:00:00 2001 From: lyz05 <294068487@qq.com> Date: Tue, 30 Apr 2024 19:01:47 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=8D=87=E7=BA=A7axios=E7=89=88?= =?UTF-8?q?=E6=9C=AC=EF=BC=8C=E6=94=AF=E6=8C=81br=E8=A7=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=85=BE=E8=AE=AF=E8=A7=86=E9=A2=91=E4=BB=8Eworkers?= =?UTF-8?q?=E4=B8=AD=E8=8E=B7=E5=8F=96=E6=95=B4=E5=90=88=E5=A5=BD=E7=9A=84?= =?UTF-8?q?=E5=BC=B9=E5=B9=95=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fly.toml | 2 +- package-lock.json | 30 +++++++++++++++--------------- package.json | 2 +- routes/api/tencentvideo.js | 29 ++++++++++++++++++----------- routes/api/utils.js | 29 ++++++++++++++++++++++++++++- 5 files changed, 63 insertions(+), 29 deletions(-) diff --git a/fly.toml b/fly.toml index 5a4a3be..61c44e4 100644 --- a/fly.toml +++ b/fly.toml @@ -27,7 +27,7 @@ kill_timeout = "5s" [[vm]] cpu_kind = "shared" cpus = 1 - memory_mb = 512 + memory_mb = 256 [mounts] source="danmaku" diff --git a/package-lock.json b/package-lock.json index 1d1e2f8..dfc283c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "dependencies": { "ali-oss": "^6.17.1", - "axios": "^1.2.1", + "axios": "^1.6.8", "bootstrap": "^3.4.1", "chai": "^4.3.7", "chai-http": "^4.3.0", @@ -1484,11 +1484,11 @@ "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "node_modules/axios": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.1.tgz", - "integrity": "sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A==", + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", + "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", "dependencies": { - "follow-redirects": "^1.15.0", + "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -3542,9 +3542,9 @@ "peer": true }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -10157,11 +10157,11 @@ "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, "axios": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.1.tgz", - "integrity": "sha512-I88cFiGu9ryt/tfVEi4kX2SITsvDddTajXTOFmt2uK1ZVA8LytjtdeyefdQWEf5PU8w+4SSJDoYnggflB5tW4A==", + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", + "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", "requires": { - "follow-redirects": "^1.15.0", + "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -11736,9 +11736,9 @@ "peer": true }, "follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==" }, "foreground-child": { "version": "2.0.0", diff --git a/package.json b/package.json index 0e3c439..3b43523 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "ali-oss": "^6.17.1", - "axios": "^1.2.1", + "axios": "^1.6.8", "bootstrap": "^3.4.1", "chai": "^4.3.7", "chai-http": "^4.3.0", diff --git a/routes/api/tencentvideo.js b/routes/api/tencentvideo.js index 801ebc2..4a53361 100644 --- a/routes/api/tencentvideo.js +++ b/routes/api/tencentvideo.js @@ -1,7 +1,9 @@ const urlmodule = require("url"); const axios = require("axios"); const whacko = require("whacko"); -const {content_template} = require("./utils"); +const {content_template,fetchInBatches} = require("./utils"); +const fs = require("fs") +const memory = require("../../utils/memory"); function Tencentvideo() { this.name = "腾讯视频"; @@ -37,20 +39,24 @@ function Tencentvideo() { } else throw e; } - let promises = []; + // let promises = []; + let urls = []; let list = Object.values(res.data.segment_index); for (const item of list) { - promises.push(axios.get(`${api_danmaku_segment}${vid}/${item.segment_name}`)); + urls.push(`${api_danmaku_segment}${vid}/${item.segment_name}`) + // promises.push(axios.get(`${api_danmaku_segment}${vid}/${item.segment_name}`)); } - return promises; + return urls; }; - this.parse = async (promises) => { + this.parse = async (urls) => { let contents = []; - const results = await Promise.allSettled(promises); - let datas = results.filter(result => result.status === 'fulfilled') - .map(result => result.value.data); - + // const results = await Promise.allSettled(promises); + // let datas = results.filter(result => result.status === 'fulfilled') + // .map(result => result.value.data); + memory() + let datas = await fetchInBatches(urls) + memory() for (const data of datas) { for (const item of data.barrage_list) { const content = JSON.parse(JSON.stringify(content_template)); @@ -63,6 +69,7 @@ function Tencentvideo() { contents.push(content); } } + memory() // contents = make_response(contents); return contents; }; @@ -87,8 +94,8 @@ module.exports = Tencentvideo; if (!module.parent) { console.log("main"); const t = new Tencentvideo(); - t.work(t.example_urls[0]).then(() => { - console.log(t.content); + t.work(t.example_urls[2]).then(() => { + // console.log(t.content); console.log(t.title); }); } diff --git a/routes/api/utils.js b/routes/api/utils.js index d845fb0..acc5310 100644 --- a/routes/api/utils.js +++ b/routes/api/utils.js @@ -1,3 +1,4 @@ +const axios = require("axios"); const content_template = { timepoint: 0, // 弹幕发送时间（秒） ct: 1, // 弹幕类型，1-3 为滚动弹幕、4 为底部、5 为顶端、6 为逆向、7 为精确、8 为高级 @@ -19,4 +20,30 @@ function time_to_second(time) { return s; } -module.exports = {time_to_second, content_template}; +async function fetchInBatches(urls, batchSize = 32) { + let results = [],promises=[]; + console.log("=== Batch Process Start ==="); + for (let i = 0; i < urls.length; i += batchSize) { + // console.log(`Processing batch starting from index ${i}`); + const batchurls = urls.slice(i, i + batchSize); + promises.push(axios.post("https://url-combine.lyz05.workers.dev/", batchurls)) + // console.log(`Batch ending at index ${i + batchSize - 1} processed`); + } + // Promise.allSettled不会抛出错误，所以不需要try/catch + let batchResults = await Promise.allSettled(promises); + + // 处理每个承诺的结果 + batchResults.forEach(result => { + if (result.status === 'fulfilled') { + // console.log('Result:', result.value.data); + results.push(result.value.data); // 假设每个成功的承诺都有一个data属性 + } else { + console.error('Failed:', result.reason.message); // 显示失败的原因 + } + }); + console.log("=== Batch Process End ==="); + results = results.flat() + return results; +} + +module.exports = {time_to_second, content_template, fetchInBatches};