mirror of
https://github.com/EstrellaXD/Auto_Bangumi.git
synced 2026-11-03 06:04:20 +08:00
fix(plugins): 处理 Codex 审查的六个问题
- apprise:urls 含令牌,改为 secret_field;secret_field 支持 default_factory,可声明列表 / 字典秘密字段
- run-command:stderr 改为自建管道边读边丢,只保留最后 2000 字节;完成仍只看 proc.wait(),结束后给读取 0.5 秒再关闭管道
- run-command:{files} 按原始路径列表展开,含换行的路径不再拆成两个参数
- 通知:插件渠道 extra 里的已声明字段与内置渠道一样展开 $VAR
- slack:只在 HTTP 400 invalid_blocks 时退回纯文字重试,其它失败不重发
- onebot:先发文字,再单独发海报;结果只看文字,去掉重试
- strm-writer:media_root 为 "/" 时正确映射完整路径
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
|
||||
- `@subscribe("*", timeout=...)` 订阅全部事件,按配置过滤;处理超时放宽到命令超时之上
|
||||
- `asyncio.create_subprocess_exec` 直接执行参数列表,不经过 shell
|
||||
- 命令超时被杀掉,退出码非 0 时记录 stderr 末尾;失败只记日志,不让插件被熔断
|
||||
- 命令超时被杀掉,退出码非 0 时记录 stderr 的最后 2000 字节(边读边丢,输出再多也不占满内存);命令退出后插件即关闭 stderr,留在后台的进程需自行重定向输出。失败只记日志,不让插件被熔断
|
||||
|
||||
## 配置
|
||||
|
||||
@@ -32,7 +32,7 @@ ln -f "$AB_NEW_PATH" "/media/links/$(basename "$AB_NEW_PATH")"
|
||||
| `AB_BANGUMI_ID` | 全部 | 番剧 id;旧种子没有 `ab:<id>` 标签时为空 |
|
||||
| `AB_DOWNLOADER_ID` | 全部 | 下载器实例 id |
|
||||
| `AB_TORRENT_HASH` | `torrent.organized` | 种子 hash |
|
||||
| `AB_FILES` | `torrent.organized` | 正片与字幕的最终路径,每行一个 |
|
||||
| `AB_FILES` | `torrent.organized` | 正片与字幕的最终路径,每行一个。路径本身含换行时无法按行区分,需要准确的路径请用 `{files}` 参数 |
|
||||
| `AB_OLD_PATH` / `AB_NEW_PATH` | `file.renamed` | 重命名前后的路径 |
|
||||
| `AB_FILE_KIND` | `file.renamed` | `media` 或 `subtitle` |
|
||||
|
||||
|
||||
@@ -12,14 +12,13 @@
|
||||
- ``torrent.organized``:``AB_TORRENT_HASH``、``AB_FILES``(正片与字幕的路径,每行一个)
|
||||
- ``file.renamed``:``AB_OLD_PATH``、``AB_NEW_PATH``、``AB_FILE_KIND``
|
||||
|
||||
路径是下载器视角。命令超时被杀掉;退出码非 0 时记录 stderr 末尾。命令的失败只记日志,
|
||||
不抛给宿主:外部脚本出错不应让插件被熔断。
|
||||
路径是下载器视角。命令超时被杀掉;退出码非 0 时记录 stderr 的最后 2000 字节。
|
||||
命令的失败只记日志,不抛给宿主:外部脚本出错不应让插件被熔断。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from subprocess import DEVNULL
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -28,6 +27,7 @@ from ab_sdk import Event, Plugin, subscribe
|
||||
from ab_sdk.events import FileRenamed, TorrentOrganized
|
||||
|
||||
MAX_TIMEOUT = 3600
|
||||
STDERR_TAIL = 2000
|
||||
_PLACEHOLDER = re.compile(r"\{(\w+)\}")
|
||||
|
||||
|
||||
@@ -75,10 +75,13 @@ def event_fields(event: Event) -> dict[str, str] | None:
|
||||
}
|
||||
|
||||
|
||||
def build_argv(command: list[str], fields: dict[str, str]) -> list[str]:
|
||||
def build_argv(
|
||||
command: list[str], fields: dict[str, str], files: list[str]
|
||||
) -> list[str]:
|
||||
"""整项为 ``{字段}`` 的参数替换为字段值,``{files}`` 展开为每个文件一个参数。
|
||||
|
||||
部分包含占位符的参数与未知的占位符原样保留。
|
||||
``{files}`` 取原始路径列表,而不是按行拆 ``AB_FILES``:路径里的换行不会拆出
|
||||
多余的参数。部分包含占位符的参数与未知的占位符原样保留。
|
||||
"""
|
||||
argv: list[str] = []
|
||||
for arg in command:
|
||||
@@ -86,12 +89,19 @@ def build_argv(command: list[str], fields: dict[str, str]) -> list[str]:
|
||||
if m is None or m[1] not in fields:
|
||||
argv.append(arg)
|
||||
elif m[1] == "files":
|
||||
argv.extend(fields["files"].splitlines())
|
||||
argv.extend(files)
|
||||
else:
|
||||
argv.append(fields[m[1]])
|
||||
return argv
|
||||
|
||||
|
||||
async def drain_tail(stream: asyncio.StreamReader, tail: bytearray) -> None:
|
||||
"""持续读完 ``stream``,只保留最后 ``STDERR_TAIL`` 字节,内存有上限。"""
|
||||
while chunk := await stream.read(65536):
|
||||
tail += chunk
|
||||
del tail[:-STDERR_TAIL]
|
||||
|
||||
|
||||
def build_env(fields: dict[str, str]) -> dict[str, str]:
|
||||
"""继承宿主环境(否则子进程没有 PATH),再加上 ``AB_*``。
|
||||
|
||||
@@ -112,38 +122,57 @@ class RunCommand(Plugin[Options]):
|
||||
fields = event_fields(event)
|
||||
if fields is None:
|
||||
return
|
||||
argv = build_argv(cfg.command, fields)
|
||||
# stderr 写进临时文件而不是管道:命令留在后台的子进程继承 stderr 时,
|
||||
# 读管道要等到它们退出,命令本身早已结束也会被当成超时
|
||||
with tempfile.TemporaryFile() as err:
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
stdin=DEVNULL,
|
||||
stdout=DEVNULL,
|
||||
stderr=err,
|
||||
env=build_env(fields),
|
||||
)
|
||||
except OSError as e:
|
||||
self.ctx.log.warning("无法启动命令 %s:%s", argv[0], e)
|
||||
return
|
||||
try:
|
||||
async with asyncio.timeout(cfg.timeout):
|
||||
await proc.wait()
|
||||
except TimeoutError:
|
||||
self.ctx.log.warning("命令 %s 超过 %s 秒,已终止", argv[0], cfg.timeout)
|
||||
return
|
||||
finally:
|
||||
# 超时或宿主取消处理时,不留下孤儿进程
|
||||
# ponytail: 只杀直接子进程;命令自己再派生的进程需由脚本负责
|
||||
if proc.returncode is None:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
err.seek(max(0, err.seek(0, os.SEEK_END) - 2000))
|
||||
tail = err.read().decode(errors="replace").strip()
|
||||
files = (
|
||||
[f.path for f in event.files] if isinstance(event, TorrentOrganized) else []
|
||||
)
|
||||
argv = build_argv(cfg.command, fields, files)
|
||||
# 自建管道而不是 stderr=PIPE:读端由插件自己关闭。命令留在后台的子进程继承了
|
||||
# stderr 时,管道要等它们退出才会关闭,不能让宿主一直挂着它
|
||||
stream = asyncio.StreamReader()
|
||||
r, w = os.pipe()
|
||||
pipe, _ = await asyncio.get_running_loop().connect_read_pipe(
|
||||
lambda: asyncio.StreamReaderProtocol(stream), os.fdopen(r, "rb")
|
||||
)
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
stdin=DEVNULL,
|
||||
stdout=DEVNULL,
|
||||
stderr=w,
|
||||
env=build_env(fields),
|
||||
)
|
||||
except OSError as e:
|
||||
pipe.close()
|
||||
self.ctx.log.warning("无法启动命令 %s:%s", argv[0], e)
|
||||
return
|
||||
finally:
|
||||
os.close(w)
|
||||
# 边读边丢:命令狂写 stderr 时既不会写满管道卡住,也不会占满内存或磁盘
|
||||
tail = bytearray()
|
||||
reader = asyncio.create_task(drain_tail(stream, tail))
|
||||
try:
|
||||
# 是否完成只看 proc.wait(),不等 stderr 读到 EOF
|
||||
async with asyncio.timeout(cfg.timeout):
|
||||
await proc.wait()
|
||||
except TimeoutError:
|
||||
self.ctx.log.warning("命令 %s 超过 %s 秒,已终止", argv[0], cfg.timeout)
|
||||
return
|
||||
finally:
|
||||
# 超时或宿主取消处理时,不留下孤儿进程
|
||||
# ponytail: 只杀直接子进程;命令自己再派生的进程需由脚本负责
|
||||
if proc.returncode is None:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
# 给读取一点时间读完管道里剩下的内容,后台子进程不关管道时放弃
|
||||
await asyncio.wait({reader}, timeout=0.5)
|
||||
reader.cancel()
|
||||
pipe.close()
|
||||
if proc.returncode:
|
||||
self.ctx.log.warning(
|
||||
"命令 %s 退出码 %s:%s", argv[0], proc.returncode, tail
|
||||
"命令 %s 退出码 %s:%s",
|
||||
argv[0],
|
||||
proc.returncode,
|
||||
tail.decode(errors="replace").strip(),
|
||||
)
|
||||
else:
|
||||
self.ctx.log.info("命令 %s 已完成(%s)", argv[0], event.kind)
|
||||
|
||||
@@ -110,6 +110,33 @@ def test_on_event_background_child_does_not_delay_success(caplog):
|
||||
assert "已完成" in caplog.text and "已终止" not in caplog.text
|
||||
|
||||
|
||||
def test_on_event_files_placeholder_keeps_newline_in_path(tmp_path):
|
||||
# {files} 取原始路径列表:含换行的路径仍是一个参数(AB_FILES 只能按行拼接)
|
||||
out = tmp_path / "out.json"
|
||||
event = TorrentOrganized(
|
||||
torrent_hash="abc",
|
||||
bangumi_id=1,
|
||||
files=(OrganizedFile("/dl/a\nb.mkv", "media"),),
|
||||
)
|
||||
run({"command": [sys.executable, "-c", DUMP, str(out), "{files}"]}, event)
|
||||
assert json.loads(out.read_text())["argv"] == ["/dl/a\nb.mkv"]
|
||||
|
||||
|
||||
def test_on_event_huge_stderr_keeps_only_tail(caplog):
|
||||
# 输出远超管道缓冲:边读边丢,命令不会卡到超时,日志只带最后 2000 字节
|
||||
script = (
|
||||
"import sys; sys.stderr.write('x' * 5_000_000 + 'END'); sys.stderr.flush(); "
|
||||
"sys.exit(3)"
|
||||
)
|
||||
start = time.monotonic()
|
||||
with caplog.at_level(logging.WARNING):
|
||||
run({"command": [sys.executable, "-c", script], "timeout": 10})
|
||||
assert time.monotonic() - start < 5
|
||||
[record] = caplog.records
|
||||
assert "退出码 3" in record.getMessage()
|
||||
assert record.args[-1] == "x" * 1997 + "END"
|
||||
|
||||
|
||||
def test_on_event_nonzero_exit_logs_code_and_stderr(caplog):
|
||||
script = "import sys; sys.stderr.write('boom'); sys.exit(3)"
|
||||
with caplog.at_level(logging.WARNING):
|
||||
|
||||
Reference in New Issue
Block a user