Merge pull request #7301 from jxxghp/fix/issue-7300-event-isolation

fix(event): 隔离持久事件处理器,避免慢插件阻塞整批投递
This commit is contained in:
jxxghp
2026-11-05 02:58:01 +08:00
committed by GitHub
17 changed files with 703 additions and 257 deletions

View File

@@ -32,6 +32,10 @@ SUBTITLE_TRANSFER_COMPLETED_TOPIC = "transfer.subtitle.completed"
SUBTITLE_TRANSFER_FAILED_TOPIC = "transfer.subtitle.failed"
AUDIO_TRANSFER_COMPLETED_TOPIC = "transfer.audio.completed"
AUDIO_TRANSFER_FAILED_TOPIC = "transfer.audio.failed"
# 每个处理器拥有独立恢复主题,摘要保证主题长度不超过数据库限制。
EVENT_HANDLER_TOPIC_PREFIX = "event.handler."
# 父消息与处理器投递记录在同一事务内冻结,重放不得再次展开订阅快照。
EVENT_FANOUT_COMPLETE_FIELD = "__mp_event_fanout_complete"
OUTBOX_LEASE_SECONDS = 60
OUTBOX_MAX_INFLIGHT_TOPICS = 20
@@ -521,9 +525,10 @@ class OutboxDispatcher:
clock: Optional[Callable[[], datetime]] = None,
close: Optional[Callable[[], None]] = None,
failure_observer: Optional[Callable[[bool], None]] = None,
fallback_handler: Optional[Callable[[ClaimedOutboxMessage], None]] = None,
) -> None:
"""
注入持久端口、topic handler、有界重试策略与失败观测端口。
注入持久端口、主题处理器和有界重试策略;动态主题由可选回调解析。
"""
self._repository = repository
self._handlers = handlers
@@ -532,6 +537,7 @@ class OutboxDispatcher:
self._clock = clock or (lambda: datetime.now(timezone.utc))
self._close = close or (lambda: None)
self._failure_observer = failure_observer or (lambda _dead: None)
self._fallback_handler = fallback_handler
def claim_one(
self, *, topic: Optional[str] = None, exclude_topics: tuple[str, ...] = (),
@@ -569,7 +575,10 @@ class OutboxDispatcher:
执行已认领效果,完成与重试时间均取真实结算时刻。
"""
try:
self._handlers[message.topic](message)
handler = self._handlers.get(message.topic, self._fallback_handler)
if handler is None:
raise KeyError(message.topic)
handler(message)
except OutboxLeaseLostError:
raise
except Exception as error:

View File

@@ -13,7 +13,12 @@ from sqlalchemy.orm import Session
from sqlalchemy.sql import Select
from sqlalchemy.sql.dml import Update
from app.application.outbox import ClaimedOutboxMessage, OutboxIntent
from app.application.outbox import (
EVENT_FANOUT_COMPLETE_FIELD,
ClaimedOutboxMessage,
OutboxIntent,
OutboxLeaseLostError,
)
from app.db.base import execute_dml
from app.db.models.outbox import OutboxMessage
@@ -156,6 +161,41 @@ class SqlAlchemyOutboxDispatchStore:
"""
return self._claim(now, lease_until, topic=topic, exclude_topics=exclude_topics)
def stage_fanout(
self,
message: ClaimedOutboxMessage,
intents: tuple[OutboxIntent, ...],
now: datetime,
) -> None:
"""
在独立短事务内冻结处理器投递记录,父消息重放时复用已提交的快照。
先锁定并核对父消息的 attempt,阻止失效拥有者展开事件。投递记录和完成
标记一起提交;父消息随后结算失败也不会重复展开或增加新订阅者。
:param message: 当前持有租约的父事件消息
:param intents: 从当前启用订阅快照生成的处理器投递意图
:param now: 子消息的创建时间及首次重试时间
:raises OutboxLeaseLostError: 父消息已由其他 attempt 接管或结算
"""
with self._session_factory() as session:
locked = execute_dml(
session,
update(OutboxMessage).where(
OutboxMessage.id == message.message_id,
OutboxMessage.status == "processing",
OutboxMessage.attempt == message.attempt,
).values(lease_until=OutboxMessage.lease_until),
)
if not locked:
raise OutboxLeaseLostError("Outbox 事件展开凭证已失效")
parent = session.scalars(select(OutboxMessage).where(OutboxMessage.id == message.message_id)).one()
if parent.payload.get(EVENT_FANOUT_COMPLETE_FIELD):
return
session.add_all(_outbox_model(intent, now) for intent in intents)
parent.payload = {**parent.payload, EVENT_FANOUT_COMPLETE_FIELD: True}
session.commit()
def claim_by_event_key(
self,
event_key: str,

View File

@@ -8,7 +8,7 @@ import inspect
import time
from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from typing import Any
from typing import Any, Optional
from app.runtime.correlation import correlation_scope
from app.runtime.event.binding import EventBindingResolver
@@ -175,11 +175,20 @@ class EventDispatcher:
self,
event: Any,
async_runner: Callable[[Any], Any],
*,
handler_id: Optional[str] = None,
) -> None:
"""
串行执行广播处理器并等待完成,任一处理失败时向调用方抛出。
等待广播处理器完成;持久投递可限定单个处理器,失败继续向调用方传播。
:param handler_id: 指定稳定处理器标识;None 保留原有的全体串行派发
:raises RuntimeError: 指定处理器未注册,保留消息供重载后重试
"""
handlers = self._registry.broadcast_snapshot(event.event_type)
if handler_id is not None:
handlers = tuple(item for item in handlers if item[0] == handler_id)
if not handlers:
raise RuntimeError(f"事件处理器尚未注册:{handler_id}")
target_plugin_id = None
if event.event_type == EventType.MessageAction and isinstance(
event.event_data,

View File

@@ -612,14 +612,33 @@ class EventManager(metaclass=Singleton):
logger.error(f"Unknown event type: {etype}")
return None
def broadcast_handler_ids(self, etype: EventType) -> tuple[str, ...]:
"""
返回已启用广播处理器的稳定标识快照,供持久投递冻结接收方。
此处只读取注册表,不解析实例或执行第三方代码。
"""
with self.__lifecycle_lock:
if self.__lifecycle_state != "running":
raise RuntimeError(f"事件处理处于 {self.__lifecycle_state} 状态")
return tuple(
handler_id
for handler_id, handler in self.__registry.broadcast_snapshot(etype)
if self.__registry.is_handler_enabled(handler)
)
def send_event_strict(
self,
etype: EventType,
data: Optional[Union[dict[str, object], BaseEventData]] = None,
priority: Optional[int] = DEFAULT_EVENT_PRIORITY,
*,
handler_id: Optional[str] = None,
) -> Event:
"""
同步等待全部广播处理器完成,任一失败时阻止 durable 消息结算。
同步等待指定或全部广播处理器完成,失败阻止对应 durable 消息结算。
:param handler_id: 持久投递的目标处理器;None 保留全体派发的兼容行为
"""
event = Event(etype, data, priority)
with self.__lifecycle_lock:
@@ -628,6 +647,7 @@ class EventManager(metaclass=Singleton):
self.__dispatcher.dispatch_broadcast_strict(
event,
self.__wait_strict_async_handler,
handler_id=handler_id,
)
return event

View File

@@ -3,6 +3,9 @@
"""
from collections.abc import Callable
from datetime import datetime, timezone
from functools import partial
from hashlib import sha256
from app.adapters.external.server import MoviePilotServerHelper
from app.application.chain.events import (
@@ -14,8 +17,11 @@ from app.application.outbox import (
DOWNLOAD_MODULE_TOPIC,
DOWNLOAD_NOTIFICATION_TOPIC,
DOWNLOAD_SUBTITLE_TOPIC,
DURABLE_EVENT_TOPICS,
EVENT_HANDLER_TOPIC_PREFIX,
ClaimedOutboxMessage,
OutboxDispatcher,
OutboxIntent,
durable_event_topic,
reset_outbox_dispatcher,
validate_durable_event_handlers,
@@ -30,20 +36,36 @@ from app.schemas.message import Message
from app.schemas.types import EventType
def _stage_event_handlers(event_type: EventType, message: ClaimedOutboxMessage) -> None:
"""
冻结启用订阅并持久化逐处理器投递,避免第三方调用阻塞父主题的恢复。
"""
intents = []
for handler_id in EventManager().broadcast_handler_ids(event_type):
handler_digest = sha256(handler_id.encode()).hexdigest()
delivery_digest = sha256(f"{message.event_key}\0{handler_id}".encode()).hexdigest()
intents.append(OutboxIntent(
event_key=f"event.handler:{delivery_digest}",
topic=f"{EVENT_HANDLER_TOPIC_PREFIX}{handler_digest}",
payload={
"event_type": event_type.value,
"event_key": message.event_key,
"handler_id": handler_id,
"event_payload": message.payload,
},
))
SqlAlchemyOutboxDispatchStore(SessionFactory).stage_fanout(
message, tuple(intents), datetime.now(timezone.utc),
)
def build_outbox_handlers() -> dict[
str,
Callable[[ClaimedOutboxMessage], None],
]:
"""
构造等待真实执行边界的 at-least-once 通知、事件和统计 handler。
构造持久事件展开、通知和统计 handler,事件处理器由独立消息等待真实结算。
"""
event_manager_factory: Callable[[], EventManager] = EventManager
def discard_event_receipt(_event: object) -> None:
"""
丢弃普通事件 API 的回执,使 outbox handler 仅表达结算成功。
"""
def dispatch_subscribe_deleted_report(message: ClaimedOutboxMessage) -> None:
"""
重放订阅删除统计;未确认时抛错以进入有限重试。
@@ -132,85 +154,51 @@ def build_outbox_handlers() -> dict[
)
handlers: dict[str, Callable[[ClaimedOutboxMessage], None]] = {
durable_event_topic(EventType.SubscribeAdded): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.SubscribeAdded,
message.payload,
)
),
durable_event_topic(event_type): partial(_stage_event_handlers, event_type)
for event_type in DURABLE_EVENT_TOPICS
}
handlers.update({
"subscribe.added.report": dispatch_subscribe_added_report,
"subscribe.added.notification": dispatch_subscribe_added_notification,
durable_event_topic(EventType.SubscribeModified): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.SubscribeModified,
message.payload,
)
),
durable_event_topic(EventType.SubscribeDeleted): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.SubscribeDeleted,
message.payload,
)
),
"subscribe.deleted.report": dispatch_subscribe_deleted_report,
durable_event_topic(EventType.SubscribeComplete): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.SubscribeComplete,
message.payload,
)
),
"subscribe.complete.report": dispatch_subscribe_complete_report,
"subscribe.complete.notification": dispatch_subscribe_notification,
durable_event_topic(EventType.DownloadAdded): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.DownloadAdded,
restore_download_added(message.payload),
)
),
DOWNLOAD_NOTIFICATION_TOPIC: dispatch_download_notification,
DOWNLOAD_MODULE_TOPIC: dispatch_download_module,
DOWNLOAD_SUBTITLE_TOPIC: dispatch_download_subtitle,
durable_event_topic(EventType.TransferComplete): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.TransferComplete,
restore_transfer_result(message.payload),
)
),
durable_event_topic(EventType.TransferFailed): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.TransferFailed,
restore_transfer_result(message.payload),
)
),
durable_event_topic(EventType.SubtitleTransferComplete): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.SubtitleTransferComplete,
restore_transfer_result(message.payload),
)
),
durable_event_topic(EventType.SubtitleTransferFailed): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.SubtitleTransferFailed,
restore_transfer_result(message.payload),
)
),
durable_event_topic(EventType.AudioTransferComplete): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.AudioTransferComplete,
restore_transfer_result(message.payload),
)
),
durable_event_topic(EventType.AudioTransferFailed): lambda message: discard_event_receipt(
event_manager_factory().send_event_strict(
EventType.AudioTransferFailed,
restore_transfer_result(message.payload),
)
),
}
})
validate_durable_event_handlers(handlers)
return handlers
def dispatch_event_handler(message: ClaimedOutboxMessage) -> None:
"""
恢复一个处理器的事件对象并等待真实完成,失败只重试该处理器的投递记录。
保留父事件的幂等键与关联上下文,子消息身份仅用于隔离租约及结算。
不取消仍在执行的第三方代码,关闭与续租继续由 OutboxRecovery 持有。
"""
if not message.topic.startswith(EVENT_HANDLER_TOPIC_PREFIX):
raise ValueError(f"未登记的 Outbox 主题:{message.topic}")
event_type = EventType(message.payload["event_type"])
if event_type not in DURABLE_EVENT_TOPICS:
raise ValueError(f"未登记的持久事件:{event_type.value}")
payload = message.payload["event_payload"]
if event_type == EventType.DownloadAdded:
payload = restore_download_added(payload)
elif event_type in {
EventType.TransferComplete, EventType.TransferFailed,
EventType.SubtitleTransferComplete, EventType.SubtitleTransferFailed,
EventType.AudioTransferComplete, EventType.AudioTransferFailed,
}:
payload = restore_transfer_result(payload)
with correlation_scope(message.payload["event_key"]):
manager = EventManager()
manager.send_event_strict(
event_type, payload, handler_id=message.payload["handler_id"],
)
def build_outbox_dispatcher() -> OutboxDispatcher:
"""
创建使用独立短事务和 attempt fencing 的恢复 dispatcher。
@@ -218,6 +206,7 @@ def build_outbox_dispatcher() -> OutboxDispatcher:
return OutboxDispatcher(
repository=SqlAlchemyOutboxDispatchStore(SessionFactory),
handlers=build_outbox_handlers(),
fallback_handler=dispatch_event_handler,
failure_observer=lambda dead: record_metric(
"scheduler.job.dead_letter" if dead else "scheduler.job.retry",
owner="outbox",

View File

@@ -415,9 +415,10 @@ collector 只接受 canonical `eventmanager`、`EventManager()` 及其有限别
`register`/`add_event_listener`;当前宿主有 17 个静态注册点,另保留 1 个由工作流配置驱动的
真实动态注册。`app/plugins/**` 插件副本不进入宿主事实。
生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 86 个
生产调用,其中 85 个静态解析为 87 个事件引用,只有 `Command.send_plugin_event` 的插件事件类型
保持动态;18 个消费注册中 17 个静态、1 个动态。生成的
生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 76 个
生产调用,其中 74 个静态解析为 76 个事件引用,`Command.send_plugin_event` 的插件事件类型和
`dispatch_event_handler` 的持久事件恢复类型保持动态;后者只接受 `DURABLE_EVENT_TOPICS`
登记的事件。18 个消费注册中 17 个静态、1 个动态。生成的
`runtime-contract-baseline.json` 保存 line-free 事实、数量和枚举索引;人工维护的
`runtime-contract-policy.json` 只批准 consumer 的精确 fingerprint、owner 和理由,任何新增、替换、
重复或陈旧项都会失败,`--write-host` 不会改写该 policy。
@@ -488,7 +489,10 @@ flowchart LR
`sdk/_legacy/subscribe.py` + Compat 门面保留插件 ABI,不进入 canonical `__all__`。
- **Outbox 可靠副作用**:业务行与 durable intent 在同一 Session/UoW 中提交;提交后由
Outbox dispatcher 依据 topic、claim/lease、有限重试和 dead-letter 执行。完成通知、事件和统计
的 post-commit 逻辑必须保持幂等,不能用普通线程或 TaskRegistry 代替持久 intent。终态历史随统一
的 post-commit 逻辑必须保持幂等,不能用普通线程或 TaskRegistry 代替持久 intent。持久广播先
冻结接收方并写入逐处理器子消息;父消息完成表示展开已持久化,实际结果由 `event.handler.*`
子消息结算。每个处理器独立串行恢复,复用总计最多 20 个 worker、续租与关闭屏障,慢插件
只积压自身投递,失败不会重放已成功的其他处理器。终态历史随统一
数据维护任务分批清理,默认成功记录保留 30 天、dead letter 保留 90 天;总开关和两项保留期由
高级设置维护,待投递和 lease 中记录不参与清理。
- **统一历史保留期**:所有可安全按时间回收的追加型数据均受 `DATA_CLEANUP_ENABLE` 控制,包括消息、
@@ -815,7 +819,7 @@ flowchart LR
### 10.2 当前架构快照
以下是 2026-11-02 核对的 `tests/fixtures/architecture/` 数据;这是源码合同快照,
以下是 2026-11-05 核对的 `tests/fixtures/architecture/` 数据;这是源码合同快照,
不代表某个已发布镜像或正在运行的实例已经验证。
| 指标 | 当前值 |
@@ -827,7 +831,7 @@ flowchart LR
| Direct egress | 51(精确 policy 准入,Application / Chain 均为 0) |
| Module Contract V2 spec | 224(其中 220 个方法进入 `run_module` 观察面) |
| Event Contract | 53 |
| Event producer / consumer | 86(85 静态、1 动态)/ 18(17 静态、1 动态) |
| Event producer / consumer | 76(74 静态、2 动态)/ 18(17 静态、1 动态) |
| Model/Oper 自动事务与自建 Session | 0 |
| 组合根外 `SystemConfigOper()` | 0 |

View File

@@ -996,6 +996,19 @@ Durable post-commit side effects have a separate boundary:
before plugin, module or database teardown; a timeout preserves ownership and
fails shutdown rather than resetting bindings. Retry backoff starts when the
failed handler finishes, not when the message was claimed.
- Durable broadcast topics first freeze their enabled handler identifiers and
persist one delivery per handler in a fenced short transaction. The parent
records that expansion in the same transaction; replay after a settlement
failure reuses the committed recipients. Parent completion confirms durable
expansion, while `event.handler.<sha256>` records track actual consumer
completion. Each stable handler has its own serial recovery topic, so a slow
plugin cannot serialize the same batch for its peers. All deliveries share the
existing 20-worker bound, lease renewal and shutdown ownership; no third-party
thread is abandoned or treated as complete on timeout. Child envelopes retain
the original event payload, idempotency key and correlation context. The
strict event API can select one registered handler; missing handlers retry
through reload, disabled handlers retain the existing skip behavior. An
individual failure retries only that delivery, never successful peers.
- `PostCommitResult` distinguishes the committed business value from completed
and pending effects. This does not provide exactly-once delivery: a process may
stop after an external sink succeeds but before complete is persisted. Outbox

View File

@@ -62,6 +62,12 @@ focused 测试验证,不要求启动产品全套测试。
提交失败与取消后的重试,以及关闭超时保留资源。`tests/test_telegram.py` 覆盖海报失败后的文本
兜底、会话/话题/按钮保留、429 不追加请求,以及发送失败不能被 outbox 当作成功确认。
`tests/test_outbox_event_isolation.py` 使用临时 SQLite、真实恢复线程和可释放屏障验证逐处理器
持久投递:慢处理器尚未返回时,其他处理器已经收到同批十二集;失败只重试对应投递,父消息
重放不重复展开或增加新订阅者。用例同时覆盖事务回滚、陈旧 attempt、租约续期、异步处理器、
重载窗口和关闭超时保留 owner。父事件完成表示投递记录已持久化,实际执行结果以
`event.handler.*` 子记录为准;这些离线测试不代表真实插件或媒体服务器验收。
AppPush 的所有用户范围由 `tests/test_notification_routing_parity.py` 验证:下载、入库和订阅消息
经同步、异步与持久事件入口到达真实渠道;明确用户和管理员范围保持隔离,组合范围不重复发送。
`tests/test_user_repository.py` 使用临时数据库验证接收集合只含启用账号的不可变身份,

View File

@@ -1125,19 +1125,24 @@ class _EventFactCollector(ast.NodeVisitor):
node: ast.Call,
parameter_names: tuple[str, ...],
required_names: frozenset[str],
*,
keyword_only_names: frozenset[str] = frozenset(),
) -> dict[str, ast.AST] | None:
"""
按真实 Python 调用规则绑定有限签名,拒绝未知或重复参数。
:param keyword_only_names: 仅可通过关键字传递的可选参数名
"""
if len(node.args) > len(parameter_names) or any(
isinstance(argument, ast.Starred) for argument in node.args
):
return None
arguments = dict(zip(parameter_names, node.args))
allowed_names = frozenset(parameter_names) | keyword_only_names
for keyword in node.keywords:
if (
keyword.arg is None
or keyword.arg not in parameter_names
or keyword.arg not in allowed_names
or keyword.arg in arguments
):
return None
@@ -1205,6 +1210,9 @@ class _EventFactCollector(ast.NodeVisitor):
node,
("etype", "data", "priority"),
frozenset({"etype"}),
keyword_only_names=(
frozenset({"handler_id"}) if bound_method.method == "send_event_strict" else frozenset()
),
)
if arguments is None:
return None

View File

@@ -120,7 +120,6 @@
"app/runtime/tasks.py:TaskRegistry.submit_threadsafe": 25,
"app/scheduler/bridge.py:SchedulerBridgeOwner._submit_cross_thread": 16,
"app/scheduler/execution.py:SchedulerExecutionOwner.start": 21,
"app/startup/composition/outbox.py:build_outbox_handlers": 16,
"app/workflow/__init__.py:WorkflowManager._check_complex_condition": 16,
"app/workflow/actions/fetch_torrents.py:FetchTorrentsAction.execute": 16,
"app/workflow/actions/transfer_file.py:TransferFileAction.execute": 19

View File

@@ -1654,7 +1654,7 @@
}
],
"dynamic_consumer_count": 1,
"dynamic_producer_count": 1,
"dynamic_producer_count": 2,
"event_count": 53,
"event_index": {
"ChainEventType.AgentLLMProvider": {
@@ -1809,15 +1809,13 @@
"EventType.AudioTransferComplete": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"75aa6118952ed941e778b0347b281cf07cff05858b658c811fdbead76a23341f",
"93c221b3f8700f899d9613ba205f36d521c76ed5681048c0a754bcb73e6749bf"
]
},
"EventType.AudioTransferFailed": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"174dd87157d87d94e5656f4251b33005914e52e2c5fa1b53342188a4641b3225",
"57da97424ec6e2f81255d39105e0e4047226b47c05ecd5e9ab64b1164e79de53"
"174dd87157d87d94e5656f4251b33005914e52e2c5fa1b53342188a4641b3225"
]
},
"EventType.CommandExcute": {
@@ -1842,7 +1840,6 @@
"EventType.DownloadAdded": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"35375727017385323cceeeb4db950615a604b51c6a4783171cc506256f3002d6",
"6ab7645a6d0422f873242c8c9162c67904948890f429467a7f533e218aec1554",
"6ab7645a6d0422f873242c8c9162c67904948890f429467a7f533e218aec1554"
]
@@ -1945,14 +1942,12 @@
"consumer_fingerprints": [],
"producer_fingerprints": [
"019a094d2d0fd4fa84076bfab9263c2bf33ece199f4d5a9a5b57f68e07c4a9c0",
"97717bff58a8806d2696054f1a1c9865954d2f0218dab51fb27027f0d560fbea",
"cc7bd553745df756f0a69b1cf42852ab166c9fc6a58a546998b2956a0b5d7305"
"97717bff58a8806d2696054f1a1c9865954d2f0218dab51fb27027f0d560fbea"
]
},
"EventType.SubscribeComplete": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"c86fcb5d64d2367e007f01ecdccf01de80f3e0a0dae625c77246d698e1ca9d4b",
"eba9652f1bf1fab63b1ab459a1636e2167e03142099cfd2dd665dab6c88989f9"
]
},
@@ -1961,7 +1956,6 @@
"producer_fingerprints": [
"7ee0ba0bfdba520a389d45c33d361120a0d710c3ea37b1e03532dd7364a52d47",
"a08051c5da0861d7198f3f5aa1b4ce6ed054abfbeab35e06ecb64dea71ba8c5e",
"b0e58582f05faa7b8385a100589bcd724072a41cd3523615d7ac927e3272c7f8",
"ea7538a3f2ad3ddbf8de6264e0525f065eec338d2ee003a501b725e3194d490a"
]
},
@@ -1969,7 +1963,6 @@
"consumer_fingerprints": [],
"producer_fingerprints": [
"89165b03827d9801a260dd54dd03e77f903795b8872bd043a79e8283181c786f",
"b71e46714bd811b9c1703fecb112d0de83dd12e89ee70529702817958a9759a8",
"ca9bf55c3886cd02fa31feade91fe333377fa8bf7bc9cc2d1bbdfb45214acfa3",
"dbd5afd0aa1c2b449880ad3853e7e988dc29d799d40d2309780cf133a7e3c4a5"
]
@@ -1977,15 +1970,13 @@
"EventType.SubtitleTransferComplete": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"b2bacfe7cc27d289b4744dc2f7b904fb8f20368e79c60adab6988188cc011eea",
"f8ed3d63cdd6f007dd4f281c1e493b5fa5ffd7dc1591576a0224618315531fa7"
]
},
"EventType.SubtitleTransferFailed": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"45f2e225d176c9d7d49feedd0e9b3e7510760a286e89ec26b57d8173b6d4c002",
"7acfc1c2dd0ca50b8f9b72d55de769507e74851fb88381d66298f649078f5d9b"
"45f2e225d176c9d7d49feedd0e9b3e7510760a286e89ec26b57d8173b6d4c002"
]
},
"EventType.SystemError": {
@@ -2002,15 +1993,13 @@
"ff208671796f8e35b6d88733b5038a1799f33f67db8e62e9f61189d4e91a4d8e"
],
"producer_fingerprints": [
"6b3cb59b259d9ca82a52fe8376504ad848351c433d4427efd9b496874acdb1b2",
"b9c576056cd1bea0a768eea4e927144e393e194c44a0f0b6f3fe1a86fe91ddd5"
]
},
"EventType.TransferFailed": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"8ac0a3e2e96daa1e00ed16560f57bbe1ad55263753b46d1f8f196c69d80e978f",
"d0fb23c410c6fda68c9e676f107666e31e0828886f5d9aeb4f5d3dcd9fc2bda7"
"8ac0a3e2e96daa1e00ed16560f57bbe1ad55263753b46d1f8f196c69d80e978f"
]
},
"EventType.UserMessage": {
@@ -2034,11 +2023,11 @@
]
}
},
"fact_count": 104,
"fact_count": 94,
"invalid_consumer_count": 0,
"invalid_producer_count": 0,
"producer_call_count": 86,
"producer_event_reference_count": 87,
"producer_call_count": 76,
"producer_event_reference_count": 76,
"producers": [
{
"caller": "app.agent.orchestrator",
@@ -2834,134 +2823,12 @@
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.DownloadAdded"
],
"fingerprint": "35375727017385323cceeeb4db950615a604b51c6a4783171cc506256f3002d6",
"dynamic": true,
"events": [],
"fingerprint": "cfb3f40492fabb5d57078f54ba5b72e5e104ee22f619e8105598dfcba8662619",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.AudioTransferFailed"
],
"fingerprint": "57da97424ec6e2f81255d39105e0e4047226b47c05ecd5e9ab64b1164e79de53",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.TransferComplete"
],
"fingerprint": "6b3cb59b259d9ca82a52fe8376504ad848351c433d4427efd9b496874acdb1b2",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.AudioTransferComplete"
],
"fingerprint": "75aa6118952ed941e778b0347b281cf07cff05858b658c811fdbead76a23341f",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.SubtitleTransferFailed"
],
"fingerprint": "7acfc1c2dd0ca50b8f9b72d55de769507e74851fb88381d66298f649078f5d9b",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.SubscribeDeleted"
],
"fingerprint": "b0e58582f05faa7b8385a100589bcd724072a41cd3523615d7ac927e3272c7f8",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.SubtitleTransferComplete"
],
"fingerprint": "b2bacfe7cc27d289b4744dc2f7b904fb8f20368e79c60adab6988188cc011eea",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.SubscribeModified"
],
"fingerprint": "b71e46714bd811b9c1703fecb112d0de83dd12e89ee70529702817958a9759a8",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.SubscribeComplete"
],
"fingerprint": "c86fcb5d64d2367e007f01ecdccf01de80f3e0a0dae625c77246d698e1ca9d4b",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.SubscribeAdded"
],
"fingerprint": "cc7bd553745df756f0a69b1cf42852ab166c9fc6a58a546998b2956a0b5d7305",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"receiver_kind": "constructed_manager"
},
{
"caller": "app.startup.composition.outbox",
"dynamic": false,
"events": [
"EventType.TransferFailed"
],
"fingerprint": "d0fb23c410c6fda68c9e676f107666e31e0828886f5d9aeb4f5d3dcd9fc2bda7",
"invalid": false,
"method": "send_event_strict",
"qualname": "build_outbox_handlers",
"qualname": "dispatch_event_handler",
"receiver_kind": "constructed_manager"
},
{
@@ -3074,7 +2941,7 @@
}
],
"static_consumer_count": 17,
"static_producer_call_count": 85
"static_producer_call_count": 74
},
"event_specs": {
"ChainEventType.AgentLLMProvider": {

View File

@@ -452,17 +452,17 @@ def test_event_contract_baseline_covers_every_public_event_enum() -> None:
assert set(events["event_index"]) == expected
assert events["event_count"] == len(expected)
assert events["producer_call_count"] == 86
assert events["static_producer_call_count"] == 85
assert events["dynamic_producer_count"] == 1
assert events["producer_call_count"] == 76
assert events["static_producer_call_count"] == 74
assert events["dynamic_producer_count"] == 2
assert events["invalid_producer_count"] == 0
assert events["producer_event_reference_count"] == 87
assert events["producer_event_reference_count"] == 76
assert events["consumer_registration_count"] == 18
assert events["static_consumer_count"] == 17
assert events["dynamic_consumer_count"] == 1
assert events["invalid_consumer_count"] == 0
assert events["consumer_event_reference_count"] == 17
assert events["fact_count"] == 104
assert events["fact_count"] == 94
assert len({fact["fingerprint"] for fact in events["consumers"]}) == 18
assert all(
not fact["caller"].startswith("app.plugins")

View File

@@ -1194,19 +1194,44 @@ def handler(event):
assert facts["consumers"][0]["caller"] == "app.host"
def test_collect_event_facts_recognizes_targeted_strict_delivery(tmp_path: Path) -> None:
"""
处理器定向严格投递仍进入事件事实,关键字专用参数不能作为第四个位置参数。
"""
facts = _collect_facts(tmp_path, '''
from app.runtime.events import EventManager
from app.schemas.types import EventType
def publish(kind):
"""
展示定向投递与非法签名的事件事实解析边界。
"""
manager = EventManager()
manager.send_event_strict(EventType.Alpha, {}, handler_id="app.plugins.sample.handle")
manager.send_event_strict(kind, {}, handler_id="app.plugins.sample.handle")
manager.send_event_strict(EventType.Beta, {}, 10, "invalid")
manager.send_event(EventType.Gamma, handler_id="invalid")
manager.async_send_event_strict(EventType.Delta, handler_id="invalid")
''')
assert len(facts["producers"]) == 2
assert facts["producers"][0]["events"] == ["EventType.Alpha"]
assert facts["producers"][1]["dynamic"]
assert all(fact["receiver_kind"] == "constructed_manager" for fact in facts["producers"])
def test_collect_event_facts_matches_current_host_inventory() -> None:
"""
统一事实覆盖当前 86 个 producer 调用与 18 个 consumer 调用。
统一事实覆盖当前 76 个 producer 调用与 18 个 consumer 调用。
"""
facts = current_event_facts()
producers = facts["producers"]
consumers = facts["consumers"]
assert len(producers) == 86
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 85
assert sum(fact["dynamic"] for fact in producers) == 1
assert len(producers) == 76
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 74
assert sum(fact["dynamic"] for fact in producers) == 2
assert sum(fact["invalid"] for fact in producers) == 0
assert sum(len(fact["events"]) for fact in producers) == 87
assert sum(len(fact["events"]) for fact in producers) == 76
assert len(consumers) == 18
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in consumers) == 17
assert sum(fact["dynamic"] for fact in consumers) == 1

View File

@@ -37,7 +37,7 @@ from app.application.outbox import (
DOWNLOAD_MODULE_TOPIC,
DOWNLOAD_NOTIFICATION_TOPIC,
DOWNLOAD_SUBTITLE_TOPIC,
TRANSFER_COMPLETED_TOPIC,
EVENT_HANDLER_TOPIC_PREFIX,
ClaimedOutboxMessage,
OutboxDispatcher,
PostCommitEffectError,
@@ -357,7 +357,7 @@ def test_durable_music_snapshots_keep_explicit_music_type(media_type, title):
def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch):
"""
真实整理 handler 恢复 AAC 尾缀影视快照后应派发事件并完成消息结算。
真实处理器子投递恢复 AAC 尾缀影视快照后应派发事件并完成自身结算。
"""
from app.startup.composition import outbox as outbox_composition
@@ -374,13 +374,17 @@ def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch):
"idempotency_key": "transfer.completed:7023:v1",
})))
message = ClaimedOutboxMessage(
7023, "transfer.completed:7023:v1", TRANSFER_COMPLETED_TOPIC, payload, 1, 1,
7023, "event.handler:7023:v1", f"{EVENT_HANDLER_TOPIC_PREFIX}test",
{"event_type": EventType.TransferComplete.value,
"event_key": payload["idempotency_key"], "handler_id": "test.video_handler",
"event_payload": payload}, 1, 1,
)
repository = Mock()
repository.claim.return_value = message
event_manager = Mock()
monkeypatch.setattr(outbox_composition, "EventManager", lambda: event_manager)
dispatcher = OutboxDispatcher(repository, outbox_composition.build_outbox_handlers())
dispatcher = OutboxDispatcher(repository, outbox_composition.build_outbox_handlers(),
fallback_handler=outbox_composition.dispatch_event_handler)
assert dispatcher.dispatch_one() is True
@@ -390,7 +394,8 @@ def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch):
assert restored["meta"].type == MediaType.TV
assert restored["meta"].to_dict() == meta.to_dict()
assert restored["fileitem"] == fileitem
assert restored["idempotency_key"] == message.event_key
assert restored["idempotency_key"] == payload["idempotency_key"]
assert event_manager.send_event_strict.call_args.kwargs == {"handler_id": "test.video_handler"}
repository.complete.assert_called_once()
assert repository.complete.call_args.args[:2] == (message.message_id, message.attempt)
repository.retry.assert_not_called()

View File

@@ -1089,8 +1089,9 @@ def test_outbox_retries_strict_failure_without_repeating_system_alert(
"""
from unittest.mock import Mock
from app.application.outbox import ClaimedOutboxMessage, OutboxDispatcher
from app.application.outbox import EVENT_HANDLER_TOPIC_PREFIX, ClaimedOutboxMessage, OutboxDispatcher
from app.runtime.event.errors import EventErrorPolicy
from app.runtime.event.registry import EventRegistry
from app.startup.composition import outbox
isolated_eventmanager._EventManager__lifecycle_state = "running"
@@ -1111,12 +1112,16 @@ def test_outbox_retries_strict_failure_without_repeating_system_alert(
store = Mock()
event_key = f"{event_type.value}:task-1:1"
store.claim.side_effect = [
ClaimedOutboxMessage(1, event_key, event_type.value,
{"idempotency_key": event_key,
"transferinfo": {"success": False, "message": "failed"}}, 1, attempt)
ClaimedOutboxMessage(1, "child-key", f"{EVENT_HANDLER_TOPIC_PREFIX}test",
{"event_type": event_type.value,
"event_key": event_key,
"handler_id": EventRegistry.handler_identifier(failed_handler),
"event_payload": {"idempotency_key": event_key,
"transferinfo": {"success": False, "message": "failed"}}}, 1, attempt)
for attempt in range(1, 6)
] + [None]
dispatcher = OutboxDispatcher(store, outbox.build_outbox_handlers())
dispatcher = OutboxDispatcher(store, outbox.build_outbox_handlers(),
fallback_handler=outbox.dispatch_event_handler)
for _ in range(5):
assert dispatcher.dispatch_one()
assert not dispatcher.dispatch_one()

View File

@@ -385,8 +385,9 @@ def test_startup_handler_replays_strict_boundary_with_stable_key(
monkeypatch.setattr(
EventManager,
"send_event_strict",
lambda _self, _etype, data: calls.append(data["idempotency_key"]),
lambda _self, _etype, data, *, handler_id: calls.append((data["idempotency_key"], handler_id)),
)
monkeypatch.setattr(EventManager, "broadcast_handler_ids", lambda _self, _etype: ("test.handler",))
elif handler_kind in {"notification", "download_notification"}:
topic = (
"subscribe.complete.notification"
@@ -420,6 +421,10 @@ def test_startup_handler_replays_strict_boundary_with_stable_key(
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
factory = sessionmaker(bind=engine)
if handler_kind == "event":
from app.startup.composition import outbox as composition
monkeypatch.setattr(composition, "SessionFactory", factory)
now = datetime(2026, 8, 28, tzinfo=timezone.utc)
event_key = f"startup:{handler_kind}:v1"
with factory() as session:
@@ -439,7 +444,18 @@ def test_startup_handler_replays_strict_boundary_with_stable_key(
clock=lambda: now + timedelta(seconds=2),
)
assert dispatcher.dispatch_one() is True
assert calls == [event_key, event_key]
if handler_kind == "event":
from app.startup.composition.outbox import dispatch_event_handler
current = datetime.now(timezone.utc)
child = store.claim(current, current + timedelta(seconds=60))
assert child is not None
dispatch_event_handler(child)
assert calls == [(event_key, "test.handler")]
assert store.complete(child.message_id, child.attempt, current)
assert store.claim(current, current + timedelta(seconds=60)) is None
else:
assert calls == [event_key, event_key]
def test_download_writer_contract_has_no_anonymous_after_commit_path() -> None:

View File

@@ -0,0 +1,431 @@
"""
持久事件逐处理器投递的批次隔离、事务快照、重试和生命周期回归。
"""
import asyncio
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from threading import Event
from unittest.mock import Mock
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from app.application.outbox import (
DURABLE_EVENT_TOPICS,
EVENT_FANOUT_COMPLETE_FIELD,
EVENT_HANDLER_TOPIC_PREFIX,
OutboxDispatcher,
OutboxIntent,
OutboxLeaseLostError,
OutboxRecovery,
durable_event_topic,
)
from app.db.adapters.outbox import SqlAlchemyOutboxDispatchStore, SqlAlchemyOutboxStager
from app.db.models.outbox import OutboxMessage
from app.runtime.correlation import get_correlation_id
from app.runtime.event.errors import EventErrorPolicy
from app.runtime.event.registry import EventRegistry
from app.runtime.events import eventmanager
from app.runtime.loop import main_loop_registry
from app.schemas.transfer import TransferInfo
from app.schemas.types import EventType
from app.startup.composition import outbox as composition
@pytest.fixture
def runtime(tmp_path, monkeypatch):
"""
隔离事件订阅、错误通知和文件数据库,真实执行宿主投递及租约边界。
"""
engine = create_engine(f"sqlite:///{tmp_path / 'events.db'}")
OutboxMessage.__table__.create(engine)
factory = sessionmaker(engine, expire_on_commit=False)
for attribute, value in {
"__broadcast_subscribers": {},
"__disabled_handlers": set(),
"__disabled_classes": set(),
"__handler_instance_resolvers": {},
"__lifecycle_state": "running",
"__error_policy": EventErrorPolicy(notifier=lambda: None, emit_system_error=lambda _data: None),
}.items():
monkeypatch.setattr(eventmanager, f"_EventManager{attribute}", value)
monkeypatch.setattr(composition, "SessionFactory", factory)
monkeypatch.setattr(composition, "EventManager", lambda: eventmanager)
yield factory, SqlAlchemyOutboxDispatchStore(factory)
engine.dispose()
def _stage(factory, count=1, event_type=EventType.TransferComplete):
"""
暂存保持旧插件载荷格式的整理或订阅事件,返回稳定业务事件键。
"""
keys = []
with factory() as session:
for index in range(count):
key = f"{durable_event_topic(event_type)}:task-{index}:v1"
keys.append(key)
SqlAlchemyOutboxStager(session).stage(
OutboxIntent(key, durable_event_topic(event_type), {
"sequence": index,
"subscribe_id": index,
"transferinfo": {"success": True},
}),
datetime.now(timezone.utc),
)
session.commit()
return keys
def _rows(factory):
"""
返回脱离会话的消息快照,便于核对跨线程结算。
"""
with factory() as session:
return session.scalars(select(OutboxMessage).order_by(OutboxMessage.id)).all()
def _expand(factory, count=1):
"""
只处理先写入的父事件,确认此阶段不执行任何插件代码。
"""
dispatcher = composition.build_outbox_dispatcher()
for _ in range(count):
assert dispatcher.dispatch_one()
return [row for row in _rows(factory) if row.topic.startswith(EVENT_HANDLER_TOPIC_PREFIX)]
def test_blocked_handler_does_not_delay_the_other_handlers_entire_batch(runtime):
"""
首个插件未返回时另一个插件已收到全部十二集,慢插件仍持有租约和关闭屏障。
"""
factory, store = runtime
started, release, received_all = Event(), Event(), Event()
fast_calls, slow_calls = [], []
def slow(event):
"""
用可释放屏障代替插件的六十秒等待,不遗留后台执行。
"""
slow_calls.append(event.event_data["sequence"])
started.set()
assert release.wait(10)
def fast(event):
"""
验证对象恢复与关联标识,并记录整批接收顺序。
"""
assert isinstance(event.event_data["transferinfo"], TransferInfo)
assert event.correlation_id == get_correlation_id() == event.event_data["idempotency_key"]
fast_calls.append(event.event_data["sequence"])
if len(fast_calls) == 12:
received_all.set()
eventmanager.add_event_listener(EventType.TransferComplete, slow)
eventmanager.add_event_listener(EventType.TransferComplete, fast)
keys = _stage(factory, 12)
children = _expand(factory, 12)
assert len(children) == 24
assert slow_calls == fast_calls == []
assert all(row.status == "completed" for row in _rows(factory)[:12])
assert {row.payload["event_key"] for row in children} == set(keys)
with ThreadPoolExecutor(max_workers=2) as pool:
recovery = OutboxRecovery(composition.build_outbox_dispatcher, pool.submit, max_inflight=2)
try:
assert recovery.dispatch(20) == 2
assert started.wait(2)
assert received_all.wait(5)
assert fast_calls == list(range(12))
assert slow_calls == [0]
# 模拟租约过期后再次扫描,活跃处理器仍不允许启动第二个拥有者。
slow_row = next(row for row in _rows(factory) if row.status == "processing")
with factory() as session:
session.get(OutboxMessage, slow_row.id).lease_until = "2000-01-01T00:00:00+00:00"
session.commit()
recovery.dispatch(20)
renewed = next(row for row in _rows(factory) if row.id == slow_row.id)
assert renewed.attempt == 1
assert renewed.lease_until > datetime.now(timezone.utc).isoformat()
assert not recovery.stop(0)
assert slow_calls == [0]
finally:
release.set()
assert recovery.stop(5)
assert len(slow_calls) == 1
assert sum(row.status == "pending" for row in _rows(factory)) == 11
# 新恢复器重放尚未认领的记录,已经成功的接收方不会再收到整批事件。
dispatcher = composition.build_outbox_dispatcher()
while dispatcher.dispatch_one():
pass
assert slow_calls == list(range(12))
assert fast_calls == list(range(12))
assert all(row.status == "completed" for row in _rows(factory))
assert store.claim(datetime.now(timezone.utc), datetime.now(timezone.utc)) is None
def test_parent_replay_reuses_committed_snapshot_even_after_subscriptions_change(runtime):
"""
展开成功但父消息结算前崩溃时,重放不会再次投递或把新订阅加入旧事件。
"""
factory, store = runtime
original, added_later = Mock(), Mock()
def first(event):
"""
记录原订阅处理器收到的稳定事件键。
"""
original(event.event_data["idempotency_key"])
def later(event):
"""
后加入的订阅不应进入已冻结的父事件快照。
"""
added_later(event)
eventmanager.add_event_listener(EventType.TransferComplete, first)
keys = _stage(factory)
now = datetime.now(timezone.utc)
parent = store.claim(now, now + timedelta(seconds=1))
composition.build_outbox_handlers()[parent.topic](parent)
assert _rows(factory)[0].payload[EVENT_FANOUT_COMPLETE_FIELD]
eventmanager.add_event_listener(EventType.TransferComplete, later)
replay = store.claim(now + timedelta(seconds=2), now + timedelta(seconds=62))
assert replay.message_id == parent.message_id and replay.attempt == 2
composition.build_outbox_handlers()[replay.topic](replay)
assert store.complete(replay.message_id, replay.attempt, now)
# 模拟进程恢复,使用新的派发器与短会话完成已持久化的子消息。
assert composition.build_outbox_dispatcher().dispatch_one()
assert not composition.build_outbox_dispatcher().dispatch_one()
original.assert_called_once_with(keys[0])
added_later.assert_not_called()
assert len(_rows(factory)) == 2
def test_fanout_transaction_rolls_back_all_children_and_marker(runtime):
"""
子记录写入冲突时全事务回滚,父消息仍可安全重试展开。
"""
factory, store = runtime
_stage(factory)
now = datetime.now(timezone.utc)
parent = store.claim(now, now + timedelta(seconds=60))
duplicate = OutboxIntent("same-child", "event.handler.test", {})
with pytest.raises(IntegrityError):
store.stage_fanout(parent, (duplicate, duplicate), now)
rows = _rows(factory)
assert len(rows) == 1
assert not rows[0].payload.get(EVENT_FANOUT_COMPLETE_FIELD)
store.stage_fanout(parent, (duplicate,), now)
assert len(_rows(factory)) == 2
def test_stale_parent_cannot_expand_event(runtime):
"""
旧 attempt 不得写入子消息或覆盖新拥有者的展开状态。
"""
factory, store = runtime
_stage(factory)
now = datetime.now(timezone.utc)
stale = store.claim(now, now + timedelta(seconds=1))
current = store.claim(now + timedelta(seconds=2), now + timedelta(seconds=62))
assert current.attempt == 2
with pytest.raises(OutboxLeaseLostError):
store.stage_fanout(stale, (OutboxIntent("stale-child", "event.handler.test", {}),), now)
assert len(_rows(factory)) == 1
def test_handler_failure_only_retries_its_own_delivery(runtime):
"""
失败处理器重试至死信,已完成的同事件处理器只执行一次。
"""
factory, store = runtime
attempts, successful = [], []
def failing(event):
"""
模拟每次投递均失败的第三方处理器。
"""
attempts.append(event.event_data["idempotency_key"])
raise RuntimeError("插件事件失败")
def peer(event):
"""
记录不应跟随其他处理器重试的成功接收方。
"""
successful.append(event.event_data["idempotency_key"])
eventmanager.add_event_listener(EventType.TransferComplete, failing)
eventmanager.add_event_listener(EventType.TransferComplete, peer)
keys = _stage(factory)
_expand(factory)
now = [datetime.now(timezone.utc)]
dispatcher = OutboxDispatcher(store, composition.build_outbox_handlers(),
fallback_handler=composition.dispatch_event_handler, clock=lambda: now[0])
assert dispatcher.dispatch_one() and dispatcher.dispatch_one()
for _ in range(4):
now[0] += timedelta(hours=1)
assert dispatcher.dispatch_one()
assert not dispatcher.dispatch_one()
assert attempts == keys * 5
assert successful == keys
rows = _rows(factory)
assert [row.status for row in rows] == ["completed", "dead", "completed"]
assert rows[1].payload["handler_id"] == EventRegistry.handler_identifier(failing)
@pytest.mark.parametrize("event_type", list(DURABLE_EVENT_TOPICS))
def test_all_durable_topics_keep_payload_and_disabled_handler_semantics(runtime, event_type):
"""
所有持久事件保留业务载荷与幂等键,仅为启用处理器创建可恢复投递。
"""
factory, _store = runtime
received = []
def enabled(event):
"""
记录实际收到的事件类型、数据和关联上下文。
"""
received.append((event.event_type, event.event_data, event.correlation_id))
def disabled(_event):
"""
禁用处理器不能执行,也不能生成子投递记录。
"""
raise AssertionError("禁用处理器不应执行")
eventmanager.add_event_listener(event_type, enabled)
eventmanager.add_event_listener(event_type, disabled)
eventmanager.disable_event_handler(disabled)
keys = _stage(factory, event_type=event_type)
children = _expand(factory)
assert len(children) == 1
assert len(children[0].topic) <= 100 and len(children[0].event_key) <= 255
assert children[0].payload["handler_id"] == EventRegistry.handler_identifier(enabled)
assert composition.build_outbox_dispatcher().dispatch_one()
assert len(received) == 1
kind, payload, correlation_id = received[0]
assert kind == event_type
assert payload["idempotency_key"] == keys[0] == correlation_id
assert payload["sequence"] == 0
def test_unregistered_target_is_retryable_and_does_not_call_peers(runtime):
"""
插件重载窗口内目标缺失时保留待重试消息,恢复同一标识后可继续投递。
"""
factory, store = runtime
received = []
def handler(event):
"""
收集重载后接续投递的事件键。
"""
received.append(event.event_data["idempotency_key"])
eventmanager.add_event_listener(EventType.TransferComplete, handler)
keys = _stage(factory)
_expand(factory)
eventmanager.remove_event_listener(EventType.TransferComplete, handler)
dispatcher = composition.build_outbox_dispatcher()
assert dispatcher.dispatch_one()
assert _rows(factory)[1].status == "pending"
assert "尚未注册" in _rows(factory)[1].last_error
eventmanager.add_event_listener(EventType.TransferComplete, handler)
later = datetime.now(timezone.utc) + timedelta(seconds=2)
replay = store.claim(later, later + timedelta(seconds=60))
dispatcher.dispatch_claimed(replay)
assert received == keys
@pytest.mark.anyio
async def test_async_handler_waits_for_real_completion_without_blocking_peer(runtime):
"""
协程插件继续在主循环执行,独立线程拥有租约直到异步处理实际返回。
"""
factory, _store = runtime
started, release, peer_done = asyncio.Event(), asyncio.Event(), Event()
async def slow(_event):
"""
用协程屏障保持处理器在途,验证没有提前结算。
"""
started.set()
await release.wait()
def peer(_event):
"""
同事件的同步接收方应在协程屏障释放前完成。
"""
peer_done.set()
eventmanager.add_event_listener(EventType.TransferComplete, slow)
eventmanager.add_event_listener(EventType.TransferComplete, peer)
_stage(factory)
_expand(factory)
loop_owner = main_loop_registry.register(asyncio.get_running_loop())
try:
with ThreadPoolExecutor(max_workers=2) as pool:
recovery = OutboxRecovery(composition.build_outbox_dispatcher, pool.submit)
try:
assert recovery.dispatch(20) == 2
await asyncio.wait_for(started.wait(), 2)
assert await asyncio.to_thread(peer_done.wait, 2)
assert any(row.status == "processing" for row in _rows(factory))
assert not recovery.stop(0)
finally:
release.set()
assert await asyncio.to_thread(recovery.stop, 5)
finally:
main_loop_registry.release(loop_owner)
assert all(row.status == "completed" for row in _rows(factory))
@pytest.mark.parametrize("topic,event_type,with_fallback", [
("unknown", EventType.TransferComplete, True),
("unknown", EventType.TransferComplete, False),
(f"{EVENT_HANDLER_TOPIC_PREFIX}invalid", EventType.ConfigChanged, True),
])
def test_unknown_topic_or_event_remains_a_failed_delivery(runtime, topic, event_type, with_fallback):
"""
动态主题回调不接受未登记主题,不能把未知效果误记为成功。
"""
factory, store = runtime
with factory() as session:
SqlAlchemyOutboxStager(session).stage(
OutboxIntent("unknown", topic, {"event_type": event_type.value}), datetime.now(timezone.utc),
)
session.commit()
dispatcher = composition.build_outbox_dispatcher() if with_fallback else OutboxDispatcher(store, {})
assert dispatcher.dispatch_one()
row = _rows(factory)[0]
assert row.status == "pending"
assert ("未登记" if with_fallback else "unknown") in row.last_error
def test_stopped_bus_cannot_settle_parent_or_child_delivery(runtime, monkeypatch):
"""
总线封口后不能把尚未展开的父消息或未执行的子投递误记为完成。
"""
factory, store = runtime
def handler(_event):
"""
封口后的任何调用都应先被生命周期边界拒绝。
"""
raise AssertionError("停止状态不应执行处理器")
eventmanager.add_event_listener(EventType.TransferComplete, handler)
_stage(factory, 2)
_expand(factory)
monkeypatch.setattr(eventmanager, "_EventManager__lifecycle_state", "stopping")
dispatcher = composition.build_outbox_dispatcher()
assert dispatcher.dispatch_one() and dispatcher.dispatch_one()
assert not dispatcher.dispatch_one()
rows = _rows(factory)
assert [row.status for row in rows] == ["completed", "pending", "pending"]
assert all("stopping" in row.last_error for row in rows[1:])
assert not rows[1].payload.get(EVENT_FANOUT_COMPLETE_FIELD)
assert store.claim(datetime.now(timezone.utc), datetime.now(timezone.utc)) is None