From 054119e885e71e3b240986e35892d6b974cfce68 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 5 Nov 2026 02:51:43 +0800 Subject: [PATCH] fix(event): isolate durable broadcast handler delivery --- app/application/outbox.py | 13 +- app/db/adapters/outbox.py | 42 +- app/runtime/event/dispatch.py | 13 +- app/runtime/events.py | 22 +- app/startup/composition/outbox.py | 139 +++--- docs/architecture-overview.md | 16 +- docs/rules/05-architecture.md | 13 + docs/testing.md | 6 + scripts/architecture/event_facts.py | 10 +- .../architecture/complexity-baseline.json | 1 - .../runtime-contract-baseline.json | 159 +------ tests/test_architecture_contract_baseline.py | 10 +- tests/test_architecture_event_facts.py | 35 +- tests/test_chain_durable_events.py | 15 +- tests/test_event_dispatch_snapshot.py | 15 +- tests/test_outbox.py | 20 +- tests/test_outbox_event_isolation.py | 431 ++++++++++++++++++ 17 files changed, 703 insertions(+), 257 deletions(-) create mode 100644 tests/test_outbox_event_isolation.py diff --git a/app/application/outbox.py b/app/application/outbox.py index 434302963..fbdda58f4 100644 --- a/app/application/outbox.py +++ b/app/application/outbox.py @@ -32,6 +32,10 @@ SUBTITLE_TRANSFER_COMPLETED_TOPIC = "transfer.subtitle.completed" SUBTITLE_TRANSFER_FAILED_TOPIC = "transfer.subtitle.failed" AUDIO_TRANSFER_COMPLETED_TOPIC = "transfer.audio.completed" AUDIO_TRANSFER_FAILED_TOPIC = "transfer.audio.failed" +# 每个处理器拥有独立恢复主题,摘要保证主题长度不超过数据库限制。 +EVENT_HANDLER_TOPIC_PREFIX = "event.handler." +# 父消息与处理器投递记录在同一事务内冻结,重放不得再次展开订阅快照。 +EVENT_FANOUT_COMPLETE_FIELD = "__mp_event_fanout_complete" OUTBOX_LEASE_SECONDS = 60 OUTBOX_MAX_INFLIGHT_TOPICS = 20 @@ -521,9 +525,10 @@ class OutboxDispatcher: clock: Optional[Callable[[], datetime]] = None, close: Optional[Callable[[], None]] = None, failure_observer: Optional[Callable[[bool], None]] = None, + fallback_handler: Optional[Callable[[ClaimedOutboxMessage], None]] = None, ) -> None: """ - 注入持久端口、topic handler、有界重试策略与失败观测端口。 + 注入持久端口、主题处理器和有界重试策略;动态主题由可选回调解析。 """ self._repository = repository self._handlers = handlers @@ -532,6 +537,7 @@ class OutboxDispatcher: self._clock = clock or (lambda: datetime.now(timezone.utc)) self._close = close or (lambda: None) self._failure_observer = failure_observer or (lambda _dead: None) + self._fallback_handler = fallback_handler def claim_one( self, *, topic: Optional[str] = None, exclude_topics: tuple[str, ...] = (), @@ -569,7 +575,10 @@ class OutboxDispatcher: 执行已认领效果,完成与重试时间均取真实结算时刻。 """ try: - self._handlers[message.topic](message) + handler = self._handlers.get(message.topic, self._fallback_handler) + if handler is None: + raise KeyError(message.topic) + handler(message) except OutboxLeaseLostError: raise except Exception as error: diff --git a/app/db/adapters/outbox.py b/app/db/adapters/outbox.py index db2ff7c12..e652a033a 100644 --- a/app/db/adapters/outbox.py +++ b/app/db/adapters/outbox.py @@ -13,7 +13,12 @@ from sqlalchemy.orm import Session from sqlalchemy.sql import Select from sqlalchemy.sql.dml import Update -from app.application.outbox import ClaimedOutboxMessage, OutboxIntent +from app.application.outbox import ( + EVENT_FANOUT_COMPLETE_FIELD, + ClaimedOutboxMessage, + OutboxIntent, + OutboxLeaseLostError, +) from app.db.base import execute_dml from app.db.models.outbox import OutboxMessage @@ -156,6 +161,41 @@ class SqlAlchemyOutboxDispatchStore: """ return self._claim(now, lease_until, topic=topic, exclude_topics=exclude_topics) + def stage_fanout( + self, + message: ClaimedOutboxMessage, + intents: tuple[OutboxIntent, ...], + now: datetime, + ) -> None: + """ + 在独立短事务内冻结处理器投递记录,父消息重放时复用已提交的快照。 + + 先锁定并核对父消息的 attempt,阻止失效拥有者展开事件。投递记录和完成 + 标记一起提交;父消息随后结算失败也不会重复展开或增加新订阅者。 + + :param message: 当前持有租约的父事件消息 + :param intents: 从当前启用订阅快照生成的处理器投递意图 + :param now: 子消息的创建时间及首次重试时间 + :raises OutboxLeaseLostError: 父消息已由其他 attempt 接管或结算 + """ + with self._session_factory() as session: + locked = execute_dml( + session, + update(OutboxMessage).where( + OutboxMessage.id == message.message_id, + OutboxMessage.status == "processing", + OutboxMessage.attempt == message.attempt, + ).values(lease_until=OutboxMessage.lease_until), + ) + if not locked: + raise OutboxLeaseLostError("Outbox 事件展开凭证已失效") + parent = session.scalars(select(OutboxMessage).where(OutboxMessage.id == message.message_id)).one() + if parent.payload.get(EVENT_FANOUT_COMPLETE_FIELD): + return + session.add_all(_outbox_model(intent, now) for intent in intents) + parent.payload = {**parent.payload, EVENT_FANOUT_COMPLETE_FIELD: True} + session.commit() + def claim_by_event_key( self, event_key: str, diff --git a/app/runtime/event/dispatch.py b/app/runtime/event/dispatch.py index 4fa1b239f..d666b2507 100644 --- a/app/runtime/event/dispatch.py +++ b/app/runtime/event/dispatch.py @@ -8,7 +8,7 @@ import inspect import time from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext -from typing import Any +from typing import Any, Optional from app.runtime.correlation import correlation_scope from app.runtime.event.binding import EventBindingResolver @@ -175,11 +175,20 @@ class EventDispatcher: self, event: Any, async_runner: Callable[[Any], Any], + *, + handler_id: Optional[str] = None, ) -> None: """ - 串行执行广播处理器并等待完成,任一处理失败时向调用方抛出。 + 等待广播处理器完成;持久投递可限定单个处理器,失败继续向调用方传播。 + + :param handler_id: 指定稳定处理器标识;None 保留原有的全体串行派发 + :raises RuntimeError: 指定处理器未注册,保留消息供重载后重试 """ handlers = self._registry.broadcast_snapshot(event.event_type) + if handler_id is not None: + handlers = tuple(item for item in handlers if item[0] == handler_id) + if not handlers: + raise RuntimeError(f"事件处理器尚未注册:{handler_id}") target_plugin_id = None if event.event_type == EventType.MessageAction and isinstance( event.event_data, diff --git a/app/runtime/events.py b/app/runtime/events.py index 998f57684..b1d6ac969 100644 --- a/app/runtime/events.py +++ b/app/runtime/events.py @@ -612,14 +612,33 @@ class EventManager(metaclass=Singleton): logger.error(f"Unknown event type: {etype}") return None + def broadcast_handler_ids(self, etype: EventType) -> tuple[str, ...]: + """ + 返回已启用广播处理器的稳定标识快照,供持久投递冻结接收方。 + + 此处只读取注册表,不解析实例或执行第三方代码。 + """ + with self.__lifecycle_lock: + if self.__lifecycle_state != "running": + raise RuntimeError(f"事件处理处于 {self.__lifecycle_state} 状态") + return tuple( + handler_id + for handler_id, handler in self.__registry.broadcast_snapshot(etype) + if self.__registry.is_handler_enabled(handler) + ) + def send_event_strict( self, etype: EventType, data: Optional[Union[dict[str, object], BaseEventData]] = None, priority: Optional[int] = DEFAULT_EVENT_PRIORITY, + *, + handler_id: Optional[str] = None, ) -> Event: """ - 同步等待全部广播处理器完成,任一失败时阻止 durable 消息结算。 + 同步等待指定或全部广播处理器完成,失败阻止对应 durable 消息结算。 + + :param handler_id: 持久投递的目标处理器;None 保留全体派发的兼容行为 """ event = Event(etype, data, priority) with self.__lifecycle_lock: @@ -628,6 +647,7 @@ class EventManager(metaclass=Singleton): self.__dispatcher.dispatch_broadcast_strict( event, self.__wait_strict_async_handler, + handler_id=handler_id, ) return event diff --git a/app/startup/composition/outbox.py b/app/startup/composition/outbox.py index 935a5394b..86d2c8dde 100644 --- a/app/startup/composition/outbox.py +++ b/app/startup/composition/outbox.py @@ -3,6 +3,9 @@ """ from collections.abc import Callable +from datetime import datetime, timezone +from functools import partial +from hashlib import sha256 from app.adapters.external.server import MoviePilotServerHelper from app.application.chain.events import ( @@ -14,8 +17,11 @@ from app.application.outbox import ( DOWNLOAD_MODULE_TOPIC, DOWNLOAD_NOTIFICATION_TOPIC, DOWNLOAD_SUBTITLE_TOPIC, + DURABLE_EVENT_TOPICS, + EVENT_HANDLER_TOPIC_PREFIX, ClaimedOutboxMessage, OutboxDispatcher, + OutboxIntent, durable_event_topic, reset_outbox_dispatcher, validate_durable_event_handlers, @@ -30,20 +36,36 @@ from app.schemas.message import Message from app.schemas.types import EventType +def _stage_event_handlers(event_type: EventType, message: ClaimedOutboxMessage) -> None: + """ + 冻结启用订阅并持久化逐处理器投递,避免第三方调用阻塞父主题的恢复。 + """ + intents = [] + for handler_id in EventManager().broadcast_handler_ids(event_type): + handler_digest = sha256(handler_id.encode()).hexdigest() + delivery_digest = sha256(f"{message.event_key}\0{handler_id}".encode()).hexdigest() + intents.append(OutboxIntent( + event_key=f"event.handler:{delivery_digest}", + topic=f"{EVENT_HANDLER_TOPIC_PREFIX}{handler_digest}", + payload={ + "event_type": event_type.value, + "event_key": message.event_key, + "handler_id": handler_id, + "event_payload": message.payload, + }, + )) + SqlAlchemyOutboxDispatchStore(SessionFactory).stage_fanout( + message, tuple(intents), datetime.now(timezone.utc), + ) + + def build_outbox_handlers() -> dict[ str, Callable[[ClaimedOutboxMessage], None], ]: """ - 构造等待真实执行边界的 at-least-once 通知、事件和统计 handler。 + 构造持久事件展开、通知和统计 handler,事件处理器由独立消息等待真实结算。 """ - event_manager_factory: Callable[[], EventManager] = EventManager - - def discard_event_receipt(_event: object) -> None: - """ - 丢弃普通事件 API 的回执,使 outbox handler 仅表达结算成功。 - """ - def dispatch_subscribe_deleted_report(message: ClaimedOutboxMessage) -> None: """ 重放订阅删除统计;未确认时抛错以进入有限重试。 @@ -132,85 +154,51 @@ def build_outbox_handlers() -> dict[ ) handlers: dict[str, Callable[[ClaimedOutboxMessage], None]] = { - durable_event_topic(EventType.SubscribeAdded): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.SubscribeAdded, - message.payload, - ) - ), + durable_event_topic(event_type): partial(_stage_event_handlers, event_type) + for event_type in DURABLE_EVENT_TOPICS + } + handlers.update({ "subscribe.added.report": dispatch_subscribe_added_report, "subscribe.added.notification": dispatch_subscribe_added_notification, - durable_event_topic(EventType.SubscribeModified): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.SubscribeModified, - message.payload, - ) - ), - durable_event_topic(EventType.SubscribeDeleted): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.SubscribeDeleted, - message.payload, - ) - ), "subscribe.deleted.report": dispatch_subscribe_deleted_report, - durable_event_topic(EventType.SubscribeComplete): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.SubscribeComplete, - message.payload, - ) - ), "subscribe.complete.report": dispatch_subscribe_complete_report, "subscribe.complete.notification": dispatch_subscribe_notification, - durable_event_topic(EventType.DownloadAdded): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.DownloadAdded, - restore_download_added(message.payload), - ) - ), DOWNLOAD_NOTIFICATION_TOPIC: dispatch_download_notification, DOWNLOAD_MODULE_TOPIC: dispatch_download_module, DOWNLOAD_SUBTITLE_TOPIC: dispatch_download_subtitle, - durable_event_topic(EventType.TransferComplete): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.TransferComplete, - restore_transfer_result(message.payload), - ) - ), - durable_event_topic(EventType.TransferFailed): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.TransferFailed, - restore_transfer_result(message.payload), - ) - ), - durable_event_topic(EventType.SubtitleTransferComplete): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.SubtitleTransferComplete, - restore_transfer_result(message.payload), - ) - ), - durable_event_topic(EventType.SubtitleTransferFailed): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.SubtitleTransferFailed, - restore_transfer_result(message.payload), - ) - ), - durable_event_topic(EventType.AudioTransferComplete): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.AudioTransferComplete, - restore_transfer_result(message.payload), - ) - ), - durable_event_topic(EventType.AudioTransferFailed): lambda message: discard_event_receipt( - event_manager_factory().send_event_strict( - EventType.AudioTransferFailed, - restore_transfer_result(message.payload), - ) - ), - } + }) validate_durable_event_handlers(handlers) return handlers +def dispatch_event_handler(message: ClaimedOutboxMessage) -> None: + """ + 恢复一个处理器的事件对象并等待真实完成,失败只重试该处理器的投递记录。 + + 保留父事件的幂等键与关联上下文,子消息身份仅用于隔离租约及结算。 + 不取消仍在执行的第三方代码,关闭与续租继续由 OutboxRecovery 持有。 + """ + if not message.topic.startswith(EVENT_HANDLER_TOPIC_PREFIX): + raise ValueError(f"未登记的 Outbox 主题:{message.topic}") + event_type = EventType(message.payload["event_type"]) + if event_type not in DURABLE_EVENT_TOPICS: + raise ValueError(f"未登记的持久事件:{event_type.value}") + payload = message.payload["event_payload"] + if event_type == EventType.DownloadAdded: + payload = restore_download_added(payload) + elif event_type in { + EventType.TransferComplete, EventType.TransferFailed, + EventType.SubtitleTransferComplete, EventType.SubtitleTransferFailed, + EventType.AudioTransferComplete, EventType.AudioTransferFailed, + }: + payload = restore_transfer_result(payload) + with correlation_scope(message.payload["event_key"]): + manager = EventManager() + manager.send_event_strict( + event_type, payload, handler_id=message.payload["handler_id"], + ) + + def build_outbox_dispatcher() -> OutboxDispatcher: """ 创建使用独立短事务和 attempt fencing 的恢复 dispatcher。 @@ -218,6 +206,7 @@ def build_outbox_dispatcher() -> OutboxDispatcher: return OutboxDispatcher( repository=SqlAlchemyOutboxDispatchStore(SessionFactory), handlers=build_outbox_handlers(), + fallback_handler=dispatch_event_handler, failure_observer=lambda dead: record_metric( "scheduler.job.dead_letter" if dead else "scheduler.job.retry", owner="outbox", diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index 803c8eceb..e39253c4c 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -415,9 +415,10 @@ collector 只接受 canonical `eventmanager`、`EventManager()` 及其有限别 `register`/`add_event_listener`;当前宿主有 17 个静态注册点,另保留 1 个由工作流配置驱动的 真实动态注册。`app/plugins/**` 插件副本不进入宿主事实。 -生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 86 个 -生产调用,其中 85 个静态解析为 87 个事件引用,只有 `Command.send_plugin_event` 的插件事件类型 -保持动态;18 个消费注册中 17 个静态、1 个动态。生成的 +生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 76 个 +生产调用,其中 74 个静态解析为 76 个事件引用,`Command.send_plugin_event` 的插件事件类型和 +`dispatch_event_handler` 的持久事件恢复类型保持动态;后者只接受 `DURABLE_EVENT_TOPICS` +登记的事件。18 个消费注册中 17 个静态、1 个动态。生成的 `runtime-contract-baseline.json` 保存 line-free 事实、数量和枚举索引;人工维护的 `runtime-contract-policy.json` 只批准 consumer 的精确 fingerprint、owner 和理由,任何新增、替换、 重复或陈旧项都会失败,`--write-host` 不会改写该 policy。 @@ -488,7 +489,10 @@ flowchart LR `sdk/_legacy/subscribe.py` + Compat 门面保留插件 ABI,不进入 canonical `__all__`。 - **Outbox 可靠副作用**:业务行与 durable intent 在同一 Session/UoW 中提交;提交后由 Outbox dispatcher 依据 topic、claim/lease、有限重试和 dead-letter 执行。完成通知、事件和统计 - 的 post-commit 逻辑必须保持幂等,不能用普通线程或 TaskRegistry 代替持久 intent。终态历史随统一 + 的 post-commit 逻辑必须保持幂等,不能用普通线程或 TaskRegistry 代替持久 intent。持久广播先 + 冻结接收方并写入逐处理器子消息;父消息完成表示展开已持久化,实际结果由 `event.handler.*` + 子消息结算。每个处理器独立串行恢复,复用总计最多 20 个 worker、续租与关闭屏障,慢插件 + 只积压自身投递,失败不会重放已成功的其他处理器。终态历史随统一 数据维护任务分批清理,默认成功记录保留 30 天、dead letter 保留 90 天;总开关和两项保留期由 高级设置维护,待投递和 lease 中记录不参与清理。 - **统一历史保留期**:所有可安全按时间回收的追加型数据均受 `DATA_CLEANUP_ENABLE` 控制,包括消息、 @@ -815,7 +819,7 @@ flowchart LR ### 10.2 当前架构快照 -以下是 2026-11-02 核对的 `tests/fixtures/architecture/` 数据;这是源码合同快照, +以下是 2026-11-05 核对的 `tests/fixtures/architecture/` 数据;这是源码合同快照, 不代表某个已发布镜像或正在运行的实例已经验证。 | 指标 | 当前值 | @@ -827,7 +831,7 @@ flowchart LR | Direct egress | 51(精确 policy 准入,Application / Chain 均为 0) | | Module Contract V2 spec | 224(其中 220 个方法进入 `run_module` 观察面) | | Event Contract | 53 | -| Event producer / consumer | 86(85 静态、1 动态)/ 18(17 静态、1 动态) | +| Event producer / consumer | 76(74 静态、2 动态)/ 18(17 静态、1 动态) | | Model/Oper 自动事务与自建 Session | 0 | | 组合根外 `SystemConfigOper()` | 0 | diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index 6a6deff63..a1b6bc2d2 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -996,6 +996,19 @@ Durable post-commit side effects have a separate boundary: before plugin, module or database teardown; a timeout preserves ownership and fails shutdown rather than resetting bindings. Retry backoff starts when the failed handler finishes, not when the message was claimed. +- Durable broadcast topics first freeze their enabled handler identifiers and + persist one delivery per handler in a fenced short transaction. The parent + records that expansion in the same transaction; replay after a settlement + failure reuses the committed recipients. Parent completion confirms durable + expansion, while `event.handler.` records track actual consumer + completion. Each stable handler has its own serial recovery topic, so a slow + plugin cannot serialize the same batch for its peers. All deliveries share the + existing 20-worker bound, lease renewal and shutdown ownership; no third-party + thread is abandoned or treated as complete on timeout. Child envelopes retain + the original event payload, idempotency key and correlation context. The + strict event API can select one registered handler; missing handlers retry + through reload, disabled handlers retain the existing skip behavior. An + individual failure retries only that delivery, never successful peers. - `PostCommitResult` distinguishes the committed business value from completed and pending effects. This does not provide exactly-once delivery: a process may stop after an external sink succeeds but before complete is persisted. Outbox diff --git a/docs/testing.md b/docs/testing.md index d8fea4f5c..137da7195 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -62,6 +62,12 @@ focused 测试验证,不要求启动产品全套测试。 提交失败与取消后的重试,以及关闭超时保留资源。`tests/test_telegram.py` 覆盖海报失败后的文本 兜底、会话/话题/按钮保留、429 不追加请求,以及发送失败不能被 outbox 当作成功确认。 +`tests/test_outbox_event_isolation.py` 使用临时 SQLite、真实恢复线程和可释放屏障验证逐处理器 +持久投递:慢处理器尚未返回时,其他处理器已经收到同批十二集;失败只重试对应投递,父消息 +重放不重复展开或增加新订阅者。用例同时覆盖事务回滚、陈旧 attempt、租约续期、异步处理器、 +重载窗口和关闭超时保留 owner。父事件完成表示投递记录已持久化,实际执行结果以 +`event.handler.*` 子记录为准;这些离线测试不代表真实插件或媒体服务器验收。 + AppPush 的所有用户范围由 `tests/test_notification_routing_parity.py` 验证:下载、入库和订阅消息 经同步、异步与持久事件入口到达真实渠道;明确用户和管理员范围保持隔离,组合范围不重复发送。 `tests/test_user_repository.py` 使用临时数据库验证接收集合只含启用账号的不可变身份, diff --git a/scripts/architecture/event_facts.py b/scripts/architecture/event_facts.py index 8fd038a32..d939004a5 100644 --- a/scripts/architecture/event_facts.py +++ b/scripts/architecture/event_facts.py @@ -1125,19 +1125,24 @@ class _EventFactCollector(ast.NodeVisitor): node: ast.Call, parameter_names: tuple[str, ...], required_names: frozenset[str], + *, + keyword_only_names: frozenset[str] = frozenset(), ) -> dict[str, ast.AST] | None: """ 按真实 Python 调用规则绑定有限签名,拒绝未知或重复参数。 + + :param keyword_only_names: 仅可通过关键字传递的可选参数名 """ if len(node.args) > len(parameter_names) or any( isinstance(argument, ast.Starred) for argument in node.args ): return None arguments = dict(zip(parameter_names, node.args)) + allowed_names = frozenset(parameter_names) | keyword_only_names for keyword in node.keywords: if ( keyword.arg is None - or keyword.arg not in parameter_names + or keyword.arg not in allowed_names or keyword.arg in arguments ): return None @@ -1205,6 +1210,9 @@ class _EventFactCollector(ast.NodeVisitor): node, ("etype", "data", "priority"), frozenset({"etype"}), + keyword_only_names=( + frozenset({"handler_id"}) if bound_method.method == "send_event_strict" else frozenset() + ), ) if arguments is None: return None diff --git a/tests/fixtures/architecture/complexity-baseline.json b/tests/fixtures/architecture/complexity-baseline.json index c51c8257c..6efeeb494 100644 --- a/tests/fixtures/architecture/complexity-baseline.json +++ b/tests/fixtures/architecture/complexity-baseline.json @@ -120,7 +120,6 @@ "app/runtime/tasks.py:TaskRegistry.submit_threadsafe": 25, "app/scheduler/bridge.py:SchedulerBridgeOwner._submit_cross_thread": 16, "app/scheduler/execution.py:SchedulerExecutionOwner.start": 21, - "app/startup/composition/outbox.py:build_outbox_handlers": 16, "app/workflow/__init__.py:WorkflowManager._check_complex_condition": 16, "app/workflow/actions/fetch_torrents.py:FetchTorrentsAction.execute": 16, "app/workflow/actions/transfer_file.py:TransferFileAction.execute": 19 diff --git a/tests/fixtures/architecture/runtime-contract-baseline.json b/tests/fixtures/architecture/runtime-contract-baseline.json index 6f3624b7f..3403f01f3 100644 --- a/tests/fixtures/architecture/runtime-contract-baseline.json +++ b/tests/fixtures/architecture/runtime-contract-baseline.json @@ -1654,7 +1654,7 @@ } ], "dynamic_consumer_count": 1, - "dynamic_producer_count": 1, + "dynamic_producer_count": 2, "event_count": 53, "event_index": { "ChainEventType.AgentLLMProvider": { @@ -1809,15 +1809,13 @@ "EventType.AudioTransferComplete": { "consumer_fingerprints": [], "producer_fingerprints": [ - "75aa6118952ed941e778b0347b281cf07cff05858b658c811fdbead76a23341f", "93c221b3f8700f899d9613ba205f36d521c76ed5681048c0a754bcb73e6749bf" ] }, "EventType.AudioTransferFailed": { "consumer_fingerprints": [], "producer_fingerprints": [ - "174dd87157d87d94e5656f4251b33005914e52e2c5fa1b53342188a4641b3225", - "57da97424ec6e2f81255d39105e0e4047226b47c05ecd5e9ab64b1164e79de53" + "174dd87157d87d94e5656f4251b33005914e52e2c5fa1b53342188a4641b3225" ] }, "EventType.CommandExcute": { @@ -1842,7 +1840,6 @@ "EventType.DownloadAdded": { "consumer_fingerprints": [], "producer_fingerprints": [ - "35375727017385323cceeeb4db950615a604b51c6a4783171cc506256f3002d6", "6ab7645a6d0422f873242c8c9162c67904948890f429467a7f533e218aec1554", "6ab7645a6d0422f873242c8c9162c67904948890f429467a7f533e218aec1554" ] @@ -1945,14 +1942,12 @@ "consumer_fingerprints": [], "producer_fingerprints": [ "019a094d2d0fd4fa84076bfab9263c2bf33ece199f4d5a9a5b57f68e07c4a9c0", - "97717bff58a8806d2696054f1a1c9865954d2f0218dab51fb27027f0d560fbea", - "cc7bd553745df756f0a69b1cf42852ab166c9fc6a58a546998b2956a0b5d7305" + "97717bff58a8806d2696054f1a1c9865954d2f0218dab51fb27027f0d560fbea" ] }, "EventType.SubscribeComplete": { "consumer_fingerprints": [], "producer_fingerprints": [ - "c86fcb5d64d2367e007f01ecdccf01de80f3e0a0dae625c77246d698e1ca9d4b", "eba9652f1bf1fab63b1ab459a1636e2167e03142099cfd2dd665dab6c88989f9" ] }, @@ -1961,7 +1956,6 @@ "producer_fingerprints": [ "7ee0ba0bfdba520a389d45c33d361120a0d710c3ea37b1e03532dd7364a52d47", "a08051c5da0861d7198f3f5aa1b4ce6ed054abfbeab35e06ecb64dea71ba8c5e", - "b0e58582f05faa7b8385a100589bcd724072a41cd3523615d7ac927e3272c7f8", "ea7538a3f2ad3ddbf8de6264e0525f065eec338d2ee003a501b725e3194d490a" ] }, @@ -1969,7 +1963,6 @@ "consumer_fingerprints": [], "producer_fingerprints": [ "89165b03827d9801a260dd54dd03e77f903795b8872bd043a79e8283181c786f", - "b71e46714bd811b9c1703fecb112d0de83dd12e89ee70529702817958a9759a8", "ca9bf55c3886cd02fa31feade91fe333377fa8bf7bc9cc2d1bbdfb45214acfa3", "dbd5afd0aa1c2b449880ad3853e7e988dc29d799d40d2309780cf133a7e3c4a5" ] @@ -1977,15 +1970,13 @@ "EventType.SubtitleTransferComplete": { "consumer_fingerprints": [], "producer_fingerprints": [ - "b2bacfe7cc27d289b4744dc2f7b904fb8f20368e79c60adab6988188cc011eea", "f8ed3d63cdd6f007dd4f281c1e493b5fa5ffd7dc1591576a0224618315531fa7" ] }, "EventType.SubtitleTransferFailed": { "consumer_fingerprints": [], "producer_fingerprints": [ - "45f2e225d176c9d7d49feedd0e9b3e7510760a286e89ec26b57d8173b6d4c002", - "7acfc1c2dd0ca50b8f9b72d55de769507e74851fb88381d66298f649078f5d9b" + "45f2e225d176c9d7d49feedd0e9b3e7510760a286e89ec26b57d8173b6d4c002" ] }, "EventType.SystemError": { @@ -2002,15 +1993,13 @@ "ff208671796f8e35b6d88733b5038a1799f33f67db8e62e9f61189d4e91a4d8e" ], "producer_fingerprints": [ - "6b3cb59b259d9ca82a52fe8376504ad848351c433d4427efd9b496874acdb1b2", "b9c576056cd1bea0a768eea4e927144e393e194c44a0f0b6f3fe1a86fe91ddd5" ] }, "EventType.TransferFailed": { "consumer_fingerprints": [], "producer_fingerprints": [ - "8ac0a3e2e96daa1e00ed16560f57bbe1ad55263753b46d1f8f196c69d80e978f", - "d0fb23c410c6fda68c9e676f107666e31e0828886f5d9aeb4f5d3dcd9fc2bda7" + "8ac0a3e2e96daa1e00ed16560f57bbe1ad55263753b46d1f8f196c69d80e978f" ] }, "EventType.UserMessage": { @@ -2034,11 +2023,11 @@ ] } }, - "fact_count": 104, + "fact_count": 94, "invalid_consumer_count": 0, "invalid_producer_count": 0, - "producer_call_count": 86, - "producer_event_reference_count": 87, + "producer_call_count": 76, + "producer_event_reference_count": 76, "producers": [ { "caller": "app.agent.orchestrator", @@ -2834,134 +2823,12 @@ }, { "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.DownloadAdded" - ], - "fingerprint": "35375727017385323cceeeb4db950615a604b51c6a4783171cc506256f3002d6", + "dynamic": true, + "events": [], + "fingerprint": "cfb3f40492fabb5d57078f54ba5b72e5e104ee22f619e8105598dfcba8662619", "invalid": false, "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.AudioTransferFailed" - ], - "fingerprint": "57da97424ec6e2f81255d39105e0e4047226b47c05ecd5e9ab64b1164e79de53", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.TransferComplete" - ], - "fingerprint": "6b3cb59b259d9ca82a52fe8376504ad848351c433d4427efd9b496874acdb1b2", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.AudioTransferComplete" - ], - "fingerprint": "75aa6118952ed941e778b0347b281cf07cff05858b658c811fdbead76a23341f", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.SubtitleTransferFailed" - ], - "fingerprint": "7acfc1c2dd0ca50b8f9b72d55de769507e74851fb88381d66298f649078f5d9b", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.SubscribeDeleted" - ], - "fingerprint": "b0e58582f05faa7b8385a100589bcd724072a41cd3523615d7ac927e3272c7f8", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.SubtitleTransferComplete" - ], - "fingerprint": "b2bacfe7cc27d289b4744dc2f7b904fb8f20368e79c60adab6988188cc011eea", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.SubscribeModified" - ], - "fingerprint": "b71e46714bd811b9c1703fecb112d0de83dd12e89ee70529702817958a9759a8", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.SubscribeComplete" - ], - "fingerprint": "c86fcb5d64d2367e007f01ecdccf01de80f3e0a0dae625c77246d698e1ca9d4b", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.SubscribeAdded" - ], - "fingerprint": "cc7bd553745df756f0a69b1cf42852ab166c9fc6a58a546998b2956a0b5d7305", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", - "receiver_kind": "constructed_manager" - }, - { - "caller": "app.startup.composition.outbox", - "dynamic": false, - "events": [ - "EventType.TransferFailed" - ], - "fingerprint": "d0fb23c410c6fda68c9e676f107666e31e0828886f5d9aeb4f5d3dcd9fc2bda7", - "invalid": false, - "method": "send_event_strict", - "qualname": "build_outbox_handlers", + "qualname": "dispatch_event_handler", "receiver_kind": "constructed_manager" }, { @@ -3074,7 +2941,7 @@ } ], "static_consumer_count": 17, - "static_producer_call_count": 85 + "static_producer_call_count": 74 }, "event_specs": { "ChainEventType.AgentLLMProvider": { diff --git a/tests/test_architecture_contract_baseline.py b/tests/test_architecture_contract_baseline.py index ecec8fc9f..9ce3437c1 100644 --- a/tests/test_architecture_contract_baseline.py +++ b/tests/test_architecture_contract_baseline.py @@ -452,17 +452,17 @@ def test_event_contract_baseline_covers_every_public_event_enum() -> None: assert set(events["event_index"]) == expected assert events["event_count"] == len(expected) - assert events["producer_call_count"] == 86 - assert events["static_producer_call_count"] == 85 - assert events["dynamic_producer_count"] == 1 + assert events["producer_call_count"] == 76 + assert events["static_producer_call_count"] == 74 + assert events["dynamic_producer_count"] == 2 assert events["invalid_producer_count"] == 0 - assert events["producer_event_reference_count"] == 87 + assert events["producer_event_reference_count"] == 76 assert events["consumer_registration_count"] == 18 assert events["static_consumer_count"] == 17 assert events["dynamic_consumer_count"] == 1 assert events["invalid_consumer_count"] == 0 assert events["consumer_event_reference_count"] == 17 - assert events["fact_count"] == 104 + assert events["fact_count"] == 94 assert len({fact["fingerprint"] for fact in events["consumers"]}) == 18 assert all( not fact["caller"].startswith("app.plugins") diff --git a/tests/test_architecture_event_facts.py b/tests/test_architecture_event_facts.py index 01cc70657..4c7aa74f6 100644 --- a/tests/test_architecture_event_facts.py +++ b/tests/test_architecture_event_facts.py @@ -1194,19 +1194,44 @@ def handler(event): assert facts["consumers"][0]["caller"] == "app.host" +def test_collect_event_facts_recognizes_targeted_strict_delivery(tmp_path: Path) -> None: + """ + 处理器定向严格投递仍进入事件事实,关键字专用参数不能作为第四个位置参数。 + """ + facts = _collect_facts(tmp_path, ''' +from app.runtime.events import EventManager +from app.schemas.types import EventType + +def publish(kind): + """ + 展示定向投递与非法签名的事件事实解析边界。 + """ + manager = EventManager() + manager.send_event_strict(EventType.Alpha, {}, handler_id="app.plugins.sample.handle") + manager.send_event_strict(kind, {}, handler_id="app.plugins.sample.handle") + manager.send_event_strict(EventType.Beta, {}, 10, "invalid") + manager.send_event(EventType.Gamma, handler_id="invalid") + manager.async_send_event_strict(EventType.Delta, handler_id="invalid") +''') + assert len(facts["producers"]) == 2 + assert facts["producers"][0]["events"] == ["EventType.Alpha"] + assert facts["producers"][1]["dynamic"] + assert all(fact["receiver_kind"] == "constructed_manager" for fact in facts["producers"]) + + def test_collect_event_facts_matches_current_host_inventory() -> None: """ - 统一事实覆盖当前 86 个 producer 调用与 18 个 consumer 调用。 + 统一事实覆盖当前 76 个 producer 调用与 18 个 consumer 调用。 """ facts = current_event_facts() producers = facts["producers"] consumers = facts["consumers"] - assert len(producers) == 86 - assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 85 - assert sum(fact["dynamic"] for fact in producers) == 1 + assert len(producers) == 76 + assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 74 + assert sum(fact["dynamic"] for fact in producers) == 2 assert sum(fact["invalid"] for fact in producers) == 0 - assert sum(len(fact["events"]) for fact in producers) == 87 + assert sum(len(fact["events"]) for fact in producers) == 76 assert len(consumers) == 18 assert sum(not fact["dynamic"] and not fact["invalid"] for fact in consumers) == 17 assert sum(fact["dynamic"] for fact in consumers) == 1 diff --git a/tests/test_chain_durable_events.py b/tests/test_chain_durable_events.py index cd224d661..dedb19a8b 100644 --- a/tests/test_chain_durable_events.py +++ b/tests/test_chain_durable_events.py @@ -37,7 +37,7 @@ from app.application.outbox import ( DOWNLOAD_MODULE_TOPIC, DOWNLOAD_NOTIFICATION_TOPIC, DOWNLOAD_SUBTITLE_TOPIC, - TRANSFER_COMPLETED_TOPIC, + EVENT_HANDLER_TOPIC_PREFIX, ClaimedOutboxMessage, OutboxDispatcher, PostCommitEffectError, @@ -357,7 +357,7 @@ def test_durable_music_snapshots_keep_explicit_music_type(media_type, title): def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch): """ - 真实整理 handler 恢复 AAC 尾缀影视快照后应派发事件并完成消息结算。 + 真实处理器子投递恢复 AAC 尾缀影视快照后应派发事件并完成自身结算。 """ from app.startup.composition import outbox as outbox_composition @@ -374,13 +374,17 @@ def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch): "idempotency_key": "transfer.completed:7023:v1", }))) message = ClaimedOutboxMessage( - 7023, "transfer.completed:7023:v1", TRANSFER_COMPLETED_TOPIC, payload, 1, 1, + 7023, "event.handler:7023:v1", f"{EVENT_HANDLER_TOPIC_PREFIX}test", + {"event_type": EventType.TransferComplete.value, + "event_key": payload["idempotency_key"], "handler_id": "test.video_handler", + "event_payload": payload}, 1, 1, ) repository = Mock() repository.claim.return_value = message event_manager = Mock() monkeypatch.setattr(outbox_composition, "EventManager", lambda: event_manager) - dispatcher = OutboxDispatcher(repository, outbox_composition.build_outbox_handlers()) + dispatcher = OutboxDispatcher(repository, outbox_composition.build_outbox_handlers(), + fallback_handler=outbox_composition.dispatch_event_handler) assert dispatcher.dispatch_one() is True @@ -390,7 +394,8 @@ def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch): assert restored["meta"].type == MediaType.TV assert restored["meta"].to_dict() == meta.to_dict() assert restored["fileitem"] == fileitem - assert restored["idempotency_key"] == message.event_key + assert restored["idempotency_key"] == payload["idempotency_key"] + assert event_manager.send_event_strict.call_args.kwargs == {"handler_id": "test.video_handler"} repository.complete.assert_called_once() assert repository.complete.call_args.args[:2] == (message.message_id, message.attempt) repository.retry.assert_not_called() diff --git a/tests/test_event_dispatch_snapshot.py b/tests/test_event_dispatch_snapshot.py index f72edc60e..521815a2b 100644 --- a/tests/test_event_dispatch_snapshot.py +++ b/tests/test_event_dispatch_snapshot.py @@ -1089,8 +1089,9 @@ def test_outbox_retries_strict_failure_without_repeating_system_alert( """ from unittest.mock import Mock - from app.application.outbox import ClaimedOutboxMessage, OutboxDispatcher + from app.application.outbox import EVENT_HANDLER_TOPIC_PREFIX, ClaimedOutboxMessage, OutboxDispatcher from app.runtime.event.errors import EventErrorPolicy + from app.runtime.event.registry import EventRegistry from app.startup.composition import outbox isolated_eventmanager._EventManager__lifecycle_state = "running" @@ -1111,12 +1112,16 @@ def test_outbox_retries_strict_failure_without_repeating_system_alert( store = Mock() event_key = f"{event_type.value}:task-1:1" store.claim.side_effect = [ - ClaimedOutboxMessage(1, event_key, event_type.value, - {"idempotency_key": event_key, - "transferinfo": {"success": False, "message": "failed"}}, 1, attempt) + ClaimedOutboxMessage(1, "child-key", f"{EVENT_HANDLER_TOPIC_PREFIX}test", + {"event_type": event_type.value, + "event_key": event_key, + "handler_id": EventRegistry.handler_identifier(failed_handler), + "event_payload": {"idempotency_key": event_key, + "transferinfo": {"success": False, "message": "failed"}}}, 1, attempt) for attempt in range(1, 6) ] + [None] - dispatcher = OutboxDispatcher(store, outbox.build_outbox_handlers()) + dispatcher = OutboxDispatcher(store, outbox.build_outbox_handlers(), + fallback_handler=outbox.dispatch_event_handler) for _ in range(5): assert dispatcher.dispatch_one() assert not dispatcher.dispatch_one() diff --git a/tests/test_outbox.py b/tests/test_outbox.py index f5b34a718..3a1b843b6 100644 --- a/tests/test_outbox.py +++ b/tests/test_outbox.py @@ -385,8 +385,9 @@ def test_startup_handler_replays_strict_boundary_with_stable_key( monkeypatch.setattr( EventManager, "send_event_strict", - lambda _self, _etype, data: calls.append(data["idempotency_key"]), + lambda _self, _etype, data, *, handler_id: calls.append((data["idempotency_key"], handler_id)), ) + monkeypatch.setattr(EventManager, "broadcast_handler_ids", lambda _self, _etype: ("test.handler",)) elif handler_kind in {"notification", "download_notification"}: topic = ( "subscribe.complete.notification" @@ -420,6 +421,10 @@ def test_startup_handler_replays_strict_boundary_with_stable_key( engine = create_engine("sqlite+pysqlite:///:memory:") Base.metadata.create_all(engine) factory = sessionmaker(bind=engine) + if handler_kind == "event": + from app.startup.composition import outbox as composition + + monkeypatch.setattr(composition, "SessionFactory", factory) now = datetime(2026, 8, 28, tzinfo=timezone.utc) event_key = f"startup:{handler_kind}:v1" with factory() as session: @@ -439,7 +444,18 @@ def test_startup_handler_replays_strict_boundary_with_stable_key( clock=lambda: now + timedelta(seconds=2), ) assert dispatcher.dispatch_one() is True - assert calls == [event_key, event_key] + if handler_kind == "event": + from app.startup.composition.outbox import dispatch_event_handler + + current = datetime.now(timezone.utc) + child = store.claim(current, current + timedelta(seconds=60)) + assert child is not None + dispatch_event_handler(child) + assert calls == [(event_key, "test.handler")] + assert store.complete(child.message_id, child.attempt, current) + assert store.claim(current, current + timedelta(seconds=60)) is None + else: + assert calls == [event_key, event_key] def test_download_writer_contract_has_no_anonymous_after_commit_path() -> None: diff --git a/tests/test_outbox_event_isolation.py b/tests/test_outbox_event_isolation.py new file mode 100644 index 000000000..b9ef66324 --- /dev/null +++ b/tests/test_outbox_event_isolation.py @@ -0,0 +1,431 @@ +""" +持久事件逐处理器投递的批次隔离、事务快照、重试和生命周期回归。 +""" + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from threading import Event +from unittest.mock import Mock + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import sessionmaker + +from app.application.outbox import ( + DURABLE_EVENT_TOPICS, + EVENT_FANOUT_COMPLETE_FIELD, + EVENT_HANDLER_TOPIC_PREFIX, + OutboxDispatcher, + OutboxIntent, + OutboxLeaseLostError, + OutboxRecovery, + durable_event_topic, +) +from app.db.adapters.outbox import SqlAlchemyOutboxDispatchStore, SqlAlchemyOutboxStager +from app.db.models.outbox import OutboxMessage +from app.runtime.correlation import get_correlation_id +from app.runtime.event.errors import EventErrorPolicy +from app.runtime.event.registry import EventRegistry +from app.runtime.events import eventmanager +from app.runtime.loop import main_loop_registry +from app.schemas.transfer import TransferInfo +from app.schemas.types import EventType +from app.startup.composition import outbox as composition + + +@pytest.fixture +def runtime(tmp_path, monkeypatch): + """ + 隔离事件订阅、错误通知和文件数据库,真实执行宿主投递及租约边界。 + """ + engine = create_engine(f"sqlite:///{tmp_path / 'events.db'}") + OutboxMessage.__table__.create(engine) + factory = sessionmaker(engine, expire_on_commit=False) + for attribute, value in { + "__broadcast_subscribers": {}, + "__disabled_handlers": set(), + "__disabled_classes": set(), + "__handler_instance_resolvers": {}, + "__lifecycle_state": "running", + "__error_policy": EventErrorPolicy(notifier=lambda: None, emit_system_error=lambda _data: None), + }.items(): + monkeypatch.setattr(eventmanager, f"_EventManager{attribute}", value) + monkeypatch.setattr(composition, "SessionFactory", factory) + monkeypatch.setattr(composition, "EventManager", lambda: eventmanager) + yield factory, SqlAlchemyOutboxDispatchStore(factory) + engine.dispose() + + +def _stage(factory, count=1, event_type=EventType.TransferComplete): + """ + 暂存保持旧插件载荷格式的整理或订阅事件,返回稳定业务事件键。 + """ + keys = [] + with factory() as session: + for index in range(count): + key = f"{durable_event_topic(event_type)}:task-{index}:v1" + keys.append(key) + SqlAlchemyOutboxStager(session).stage( + OutboxIntent(key, durable_event_topic(event_type), { + "sequence": index, + "subscribe_id": index, + "transferinfo": {"success": True}, + }), + datetime.now(timezone.utc), + ) + session.commit() + return keys + + +def _rows(factory): + """ + 返回脱离会话的消息快照,便于核对跨线程结算。 + """ + with factory() as session: + return session.scalars(select(OutboxMessage).order_by(OutboxMessage.id)).all() + + +def _expand(factory, count=1): + """ + 只处理先写入的父事件,确认此阶段不执行任何插件代码。 + """ + dispatcher = composition.build_outbox_dispatcher() + for _ in range(count): + assert dispatcher.dispatch_one() + return [row for row in _rows(factory) if row.topic.startswith(EVENT_HANDLER_TOPIC_PREFIX)] + + +def test_blocked_handler_does_not_delay_the_other_handlers_entire_batch(runtime): + """ + 首个插件未返回时另一个插件已收到全部十二集,慢插件仍持有租约和关闭屏障。 + """ + factory, store = runtime + started, release, received_all = Event(), Event(), Event() + fast_calls, slow_calls = [], [] + + def slow(event): + """ + 用可释放屏障代替插件的六十秒等待,不遗留后台执行。 + """ + slow_calls.append(event.event_data["sequence"]) + started.set() + assert release.wait(10) + + def fast(event): + """ + 验证对象恢复与关联标识,并记录整批接收顺序。 + """ + assert isinstance(event.event_data["transferinfo"], TransferInfo) + assert event.correlation_id == get_correlation_id() == event.event_data["idempotency_key"] + fast_calls.append(event.event_data["sequence"]) + if len(fast_calls) == 12: + received_all.set() + + eventmanager.add_event_listener(EventType.TransferComplete, slow) + eventmanager.add_event_listener(EventType.TransferComplete, fast) + keys = _stage(factory, 12) + children = _expand(factory, 12) + assert len(children) == 24 + assert slow_calls == fast_calls == [] + assert all(row.status == "completed" for row in _rows(factory)[:12]) + assert {row.payload["event_key"] for row in children} == set(keys) + + with ThreadPoolExecutor(max_workers=2) as pool: + recovery = OutboxRecovery(composition.build_outbox_dispatcher, pool.submit, max_inflight=2) + try: + assert recovery.dispatch(20) == 2 + assert started.wait(2) + assert received_all.wait(5) + assert fast_calls == list(range(12)) + assert slow_calls == [0] + # 模拟租约过期后再次扫描,活跃处理器仍不允许启动第二个拥有者。 + slow_row = next(row for row in _rows(factory) if row.status == "processing") + with factory() as session: + session.get(OutboxMessage, slow_row.id).lease_until = "2000-01-01T00:00:00+00:00" + session.commit() + recovery.dispatch(20) + renewed = next(row for row in _rows(factory) if row.id == slow_row.id) + assert renewed.attempt == 1 + assert renewed.lease_until > datetime.now(timezone.utc).isoformat() + assert not recovery.stop(0) + assert slow_calls == [0] + finally: + release.set() + assert recovery.stop(5) + assert len(slow_calls) == 1 + assert sum(row.status == "pending" for row in _rows(factory)) == 11 + # 新恢复器重放尚未认领的记录,已经成功的接收方不会再收到整批事件。 + dispatcher = composition.build_outbox_dispatcher() + while dispatcher.dispatch_one(): + pass + assert slow_calls == list(range(12)) + assert fast_calls == list(range(12)) + assert all(row.status == "completed" for row in _rows(factory)) + assert store.claim(datetime.now(timezone.utc), datetime.now(timezone.utc)) is None + + +def test_parent_replay_reuses_committed_snapshot_even_after_subscriptions_change(runtime): + """ + 展开成功但父消息结算前崩溃时,重放不会再次投递或把新订阅加入旧事件。 + """ + factory, store = runtime + original, added_later = Mock(), Mock() + + def first(event): + """ + 记录原订阅处理器收到的稳定事件键。 + """ + original(event.event_data["idempotency_key"]) + + def later(event): + """ + 后加入的订阅不应进入已冻结的父事件快照。 + """ + added_later(event) + + eventmanager.add_event_listener(EventType.TransferComplete, first) + keys = _stage(factory) + now = datetime.now(timezone.utc) + parent = store.claim(now, now + timedelta(seconds=1)) + composition.build_outbox_handlers()[parent.topic](parent) + assert _rows(factory)[0].payload[EVENT_FANOUT_COMPLETE_FIELD] + eventmanager.add_event_listener(EventType.TransferComplete, later) + replay = store.claim(now + timedelta(seconds=2), now + timedelta(seconds=62)) + assert replay.message_id == parent.message_id and replay.attempt == 2 + composition.build_outbox_handlers()[replay.topic](replay) + assert store.complete(replay.message_id, replay.attempt, now) + # 模拟进程恢复,使用新的派发器与短会话完成已持久化的子消息。 + assert composition.build_outbox_dispatcher().dispatch_one() + assert not composition.build_outbox_dispatcher().dispatch_one() + original.assert_called_once_with(keys[0]) + added_later.assert_not_called() + assert len(_rows(factory)) == 2 + + +def test_fanout_transaction_rolls_back_all_children_and_marker(runtime): + """ + 子记录写入冲突时全事务回滚,父消息仍可安全重试展开。 + """ + factory, store = runtime + _stage(factory) + now = datetime.now(timezone.utc) + parent = store.claim(now, now + timedelta(seconds=60)) + duplicate = OutboxIntent("same-child", "event.handler.test", {}) + with pytest.raises(IntegrityError): + store.stage_fanout(parent, (duplicate, duplicate), now) + rows = _rows(factory) + assert len(rows) == 1 + assert not rows[0].payload.get(EVENT_FANOUT_COMPLETE_FIELD) + store.stage_fanout(parent, (duplicate,), now) + assert len(_rows(factory)) == 2 + + +def test_stale_parent_cannot_expand_event(runtime): + """ + 旧 attempt 不得写入子消息或覆盖新拥有者的展开状态。 + """ + factory, store = runtime + _stage(factory) + now = datetime.now(timezone.utc) + stale = store.claim(now, now + timedelta(seconds=1)) + current = store.claim(now + timedelta(seconds=2), now + timedelta(seconds=62)) + assert current.attempt == 2 + with pytest.raises(OutboxLeaseLostError): + store.stage_fanout(stale, (OutboxIntent("stale-child", "event.handler.test", {}),), now) + assert len(_rows(factory)) == 1 + + +def test_handler_failure_only_retries_its_own_delivery(runtime): + """ + 失败处理器重试至死信,已完成的同事件处理器只执行一次。 + """ + factory, store = runtime + attempts, successful = [], [] + + def failing(event): + """ + 模拟每次投递均失败的第三方处理器。 + """ + attempts.append(event.event_data["idempotency_key"]) + raise RuntimeError("插件事件失败") + + def peer(event): + """ + 记录不应跟随其他处理器重试的成功接收方。 + """ + successful.append(event.event_data["idempotency_key"]) + + eventmanager.add_event_listener(EventType.TransferComplete, failing) + eventmanager.add_event_listener(EventType.TransferComplete, peer) + keys = _stage(factory) + _expand(factory) + now = [datetime.now(timezone.utc)] + dispatcher = OutboxDispatcher(store, composition.build_outbox_handlers(), + fallback_handler=composition.dispatch_event_handler, clock=lambda: now[0]) + assert dispatcher.dispatch_one() and dispatcher.dispatch_one() + for _ in range(4): + now[0] += timedelta(hours=1) + assert dispatcher.dispatch_one() + assert not dispatcher.dispatch_one() + assert attempts == keys * 5 + assert successful == keys + rows = _rows(factory) + assert [row.status for row in rows] == ["completed", "dead", "completed"] + assert rows[1].payload["handler_id"] == EventRegistry.handler_identifier(failing) + + +@pytest.mark.parametrize("event_type", list(DURABLE_EVENT_TOPICS)) +def test_all_durable_topics_keep_payload_and_disabled_handler_semantics(runtime, event_type): + """ + 所有持久事件保留业务载荷与幂等键,仅为启用处理器创建可恢复投递。 + """ + factory, _store = runtime + received = [] + + def enabled(event): + """ + 记录实际收到的事件类型、数据和关联上下文。 + """ + received.append((event.event_type, event.event_data, event.correlation_id)) + + def disabled(_event): + """ + 禁用处理器不能执行,也不能生成子投递记录。 + """ + raise AssertionError("禁用处理器不应执行") + + eventmanager.add_event_listener(event_type, enabled) + eventmanager.add_event_listener(event_type, disabled) + eventmanager.disable_event_handler(disabled) + keys = _stage(factory, event_type=event_type) + children = _expand(factory) + assert len(children) == 1 + assert len(children[0].topic) <= 100 and len(children[0].event_key) <= 255 + assert children[0].payload["handler_id"] == EventRegistry.handler_identifier(enabled) + assert composition.build_outbox_dispatcher().dispatch_one() + assert len(received) == 1 + kind, payload, correlation_id = received[0] + assert kind == event_type + assert payload["idempotency_key"] == keys[0] == correlation_id + assert payload["sequence"] == 0 + + +def test_unregistered_target_is_retryable_and_does_not_call_peers(runtime): + """ + 插件重载窗口内目标缺失时保留待重试消息,恢复同一标识后可继续投递。 + """ + factory, store = runtime + received = [] + + def handler(event): + """ + 收集重载后接续投递的事件键。 + """ + received.append(event.event_data["idempotency_key"]) + + eventmanager.add_event_listener(EventType.TransferComplete, handler) + keys = _stage(factory) + _expand(factory) + eventmanager.remove_event_listener(EventType.TransferComplete, handler) + dispatcher = composition.build_outbox_dispatcher() + assert dispatcher.dispatch_one() + assert _rows(factory)[1].status == "pending" + assert "尚未注册" in _rows(factory)[1].last_error + eventmanager.add_event_listener(EventType.TransferComplete, handler) + later = datetime.now(timezone.utc) + timedelta(seconds=2) + replay = store.claim(later, later + timedelta(seconds=60)) + dispatcher.dispatch_claimed(replay) + assert received == keys + + +@pytest.mark.anyio +async def test_async_handler_waits_for_real_completion_without_blocking_peer(runtime): + """ + 协程插件继续在主循环执行,独立线程拥有租约直到异步处理实际返回。 + """ + factory, _store = runtime + started, release, peer_done = asyncio.Event(), asyncio.Event(), Event() + + async def slow(_event): + """ + 用协程屏障保持处理器在途,验证没有提前结算。 + """ + started.set() + await release.wait() + + def peer(_event): + """ + 同事件的同步接收方应在协程屏障释放前完成。 + """ + peer_done.set() + + eventmanager.add_event_listener(EventType.TransferComplete, slow) + eventmanager.add_event_listener(EventType.TransferComplete, peer) + _stage(factory) + _expand(factory) + loop_owner = main_loop_registry.register(asyncio.get_running_loop()) + try: + with ThreadPoolExecutor(max_workers=2) as pool: + recovery = OutboxRecovery(composition.build_outbox_dispatcher, pool.submit) + try: + assert recovery.dispatch(20) == 2 + await asyncio.wait_for(started.wait(), 2) + assert await asyncio.to_thread(peer_done.wait, 2) + assert any(row.status == "processing" for row in _rows(factory)) + assert not recovery.stop(0) + finally: + release.set() + assert await asyncio.to_thread(recovery.stop, 5) + finally: + main_loop_registry.release(loop_owner) + assert all(row.status == "completed" for row in _rows(factory)) + + +@pytest.mark.parametrize("topic,event_type,with_fallback", [ + ("unknown", EventType.TransferComplete, True), + ("unknown", EventType.TransferComplete, False), + (f"{EVENT_HANDLER_TOPIC_PREFIX}invalid", EventType.ConfigChanged, True), +]) +def test_unknown_topic_or_event_remains_a_failed_delivery(runtime, topic, event_type, with_fallback): + """ + 动态主题回调不接受未登记主题,不能把未知效果误记为成功。 + """ + factory, store = runtime + with factory() as session: + SqlAlchemyOutboxStager(session).stage( + OutboxIntent("unknown", topic, {"event_type": event_type.value}), datetime.now(timezone.utc), + ) + session.commit() + dispatcher = composition.build_outbox_dispatcher() if with_fallback else OutboxDispatcher(store, {}) + assert dispatcher.dispatch_one() + row = _rows(factory)[0] + assert row.status == "pending" + assert ("未登记" if with_fallback else "unknown") in row.last_error + + +def test_stopped_bus_cannot_settle_parent_or_child_delivery(runtime, monkeypatch): + """ + 总线封口后不能把尚未展开的父消息或未执行的子投递误记为完成。 + """ + factory, store = runtime + + def handler(_event): + """ + 封口后的任何调用都应先被生命周期边界拒绝。 + """ + raise AssertionError("停止状态不应执行处理器") + + eventmanager.add_event_listener(EventType.TransferComplete, handler) + _stage(factory, 2) + _expand(factory) + monkeypatch.setattr(eventmanager, "_EventManager__lifecycle_state", "stopping") + dispatcher = composition.build_outbox_dispatcher() + assert dispatcher.dispatch_one() and dispatcher.dispatch_one() + assert not dispatcher.dispatch_one() + rows = _rows(factory) + assert [row.status for row in rows] == ["completed", "pending", "pending"] + assert all("stopping" in row.last_error for row in rows[1:]) + assert not rows[1].payload.get(EVENT_FANOUT_COMPLETE_FIELD) + assert store.claim(datetime.now(timezone.utc), datetime.now(timezone.utc)) is None