mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-11-23 04:13:01 +08:00
Merge pull request #7301 from jxxghp/fix/issue-7300-event-isolation
fix(event): 隔离持久事件处理器,避免慢插件阻塞整批投递
This commit is contained in:
@@ -32,6 +32,10 @@ SUBTITLE_TRANSFER_COMPLETED_TOPIC = "transfer.subtitle.completed"
|
||||
SUBTITLE_TRANSFER_FAILED_TOPIC = "transfer.subtitle.failed"
|
||||
AUDIO_TRANSFER_COMPLETED_TOPIC = "transfer.audio.completed"
|
||||
AUDIO_TRANSFER_FAILED_TOPIC = "transfer.audio.failed"
|
||||
# 每个处理器拥有独立恢复主题,摘要保证主题长度不超过数据库限制。
|
||||
EVENT_HANDLER_TOPIC_PREFIX = "event.handler."
|
||||
# 父消息与处理器投递记录在同一事务内冻结,重放不得再次展开订阅快照。
|
||||
EVENT_FANOUT_COMPLETE_FIELD = "__mp_event_fanout_complete"
|
||||
OUTBOX_LEASE_SECONDS = 60
|
||||
OUTBOX_MAX_INFLIGHT_TOPICS = 20
|
||||
|
||||
@@ -521,9 +525,10 @@ class OutboxDispatcher:
|
||||
clock: Optional[Callable[[], datetime]] = None,
|
||||
close: Optional[Callable[[], None]] = None,
|
||||
failure_observer: Optional[Callable[[bool], None]] = None,
|
||||
fallback_handler: Optional[Callable[[ClaimedOutboxMessage], None]] = None,
|
||||
) -> None:
|
||||
"""
|
||||
注入持久端口、topic handler、有界重试策略与失败观测端口。
|
||||
注入持久端口、主题处理器和有界重试策略;动态主题由可选回调解析。
|
||||
"""
|
||||
self._repository = repository
|
||||
self._handlers = handlers
|
||||
@@ -532,6 +537,7 @@ class OutboxDispatcher:
|
||||
self._clock = clock or (lambda: datetime.now(timezone.utc))
|
||||
self._close = close or (lambda: None)
|
||||
self._failure_observer = failure_observer or (lambda _dead: None)
|
||||
self._fallback_handler = fallback_handler
|
||||
|
||||
def claim_one(
|
||||
self, *, topic: Optional[str] = None, exclude_topics: tuple[str, ...] = (),
|
||||
@@ -569,7 +575,10 @@ class OutboxDispatcher:
|
||||
执行已认领效果,完成与重试时间均取真实结算时刻。
|
||||
"""
|
||||
try:
|
||||
self._handlers[message.topic](message)
|
||||
handler = self._handlers.get(message.topic, self._fallback_handler)
|
||||
if handler is None:
|
||||
raise KeyError(message.topic)
|
||||
handler(message)
|
||||
except OutboxLeaseLostError:
|
||||
raise
|
||||
except Exception as error:
|
||||
|
||||
@@ -13,7 +13,12 @@ from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql import Select
|
||||
from sqlalchemy.sql.dml import Update
|
||||
|
||||
from app.application.outbox import ClaimedOutboxMessage, OutboxIntent
|
||||
from app.application.outbox import (
|
||||
EVENT_FANOUT_COMPLETE_FIELD,
|
||||
ClaimedOutboxMessage,
|
||||
OutboxIntent,
|
||||
OutboxLeaseLostError,
|
||||
)
|
||||
from app.db.base import execute_dml
|
||||
from app.db.models.outbox import OutboxMessage
|
||||
|
||||
@@ -156,6 +161,41 @@ class SqlAlchemyOutboxDispatchStore:
|
||||
"""
|
||||
return self._claim(now, lease_until, topic=topic, exclude_topics=exclude_topics)
|
||||
|
||||
def stage_fanout(
|
||||
self,
|
||||
message: ClaimedOutboxMessage,
|
||||
intents: tuple[OutboxIntent, ...],
|
||||
now: datetime,
|
||||
) -> None:
|
||||
"""
|
||||
在独立短事务内冻结处理器投递记录,父消息重放时复用已提交的快照。
|
||||
|
||||
先锁定并核对父消息的 attempt,阻止失效拥有者展开事件。投递记录和完成
|
||||
标记一起提交;父消息随后结算失败也不会重复展开或增加新订阅者。
|
||||
|
||||
:param message: 当前持有租约的父事件消息
|
||||
:param intents: 从当前启用订阅快照生成的处理器投递意图
|
||||
:param now: 子消息的创建时间及首次重试时间
|
||||
:raises OutboxLeaseLostError: 父消息已由其他 attempt 接管或结算
|
||||
"""
|
||||
with self._session_factory() as session:
|
||||
locked = execute_dml(
|
||||
session,
|
||||
update(OutboxMessage).where(
|
||||
OutboxMessage.id == message.message_id,
|
||||
OutboxMessage.status == "processing",
|
||||
OutboxMessage.attempt == message.attempt,
|
||||
).values(lease_until=OutboxMessage.lease_until),
|
||||
)
|
||||
if not locked:
|
||||
raise OutboxLeaseLostError("Outbox 事件展开凭证已失效")
|
||||
parent = session.scalars(select(OutboxMessage).where(OutboxMessage.id == message.message_id)).one()
|
||||
if parent.payload.get(EVENT_FANOUT_COMPLETE_FIELD):
|
||||
return
|
||||
session.add_all(_outbox_model(intent, now) for intent in intents)
|
||||
parent.payload = {**parent.payload, EVENT_FANOUT_COMPLETE_FIELD: True}
|
||||
session.commit()
|
||||
|
||||
def claim_by_event_key(
|
||||
self,
|
||||
event_key: str,
|
||||
|
||||
@@ -8,7 +8,7 @@ import inspect
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from typing import Any
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.runtime.correlation import correlation_scope
|
||||
from app.runtime.event.binding import EventBindingResolver
|
||||
@@ -175,11 +175,20 @@ class EventDispatcher:
|
||||
self,
|
||||
event: Any,
|
||||
async_runner: Callable[[Any], Any],
|
||||
*,
|
||||
handler_id: Optional[str] = None,
|
||||
) -> None:
|
||||
"""
|
||||
串行执行广播处理器并等待完成,任一处理失败时向调用方抛出。
|
||||
等待广播处理器完成;持久投递可限定单个处理器,失败继续向调用方传播。
|
||||
|
||||
:param handler_id: 指定稳定处理器标识;None 保留原有的全体串行派发
|
||||
:raises RuntimeError: 指定处理器未注册,保留消息供重载后重试
|
||||
"""
|
||||
handlers = self._registry.broadcast_snapshot(event.event_type)
|
||||
if handler_id is not None:
|
||||
handlers = tuple(item for item in handlers if item[0] == handler_id)
|
||||
if not handlers:
|
||||
raise RuntimeError(f"事件处理器尚未注册:{handler_id}")
|
||||
target_plugin_id = None
|
||||
if event.event_type == EventType.MessageAction and isinstance(
|
||||
event.event_data,
|
||||
|
||||
@@ -612,14 +612,33 @@ class EventManager(metaclass=Singleton):
|
||||
logger.error(f"Unknown event type: {etype}")
|
||||
return None
|
||||
|
||||
def broadcast_handler_ids(self, etype: EventType) -> tuple[str, ...]:
|
||||
"""
|
||||
返回已启用广播处理器的稳定标识快照,供持久投递冻结接收方。
|
||||
|
||||
此处只读取注册表,不解析实例或执行第三方代码。
|
||||
"""
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state != "running":
|
||||
raise RuntimeError(f"事件处理处于 {self.__lifecycle_state} 状态")
|
||||
return tuple(
|
||||
handler_id
|
||||
for handler_id, handler in self.__registry.broadcast_snapshot(etype)
|
||||
if self.__registry.is_handler_enabled(handler)
|
||||
)
|
||||
|
||||
def send_event_strict(
|
||||
self,
|
||||
etype: EventType,
|
||||
data: Optional[Union[dict[str, object], BaseEventData]] = None,
|
||||
priority: Optional[int] = DEFAULT_EVENT_PRIORITY,
|
||||
*,
|
||||
handler_id: Optional[str] = None,
|
||||
) -> Event:
|
||||
"""
|
||||
同步等待全部广播处理器完成,任一失败时阻止 durable 消息结算。
|
||||
同步等待指定或全部广播处理器完成,失败阻止对应 durable 消息结算。
|
||||
|
||||
:param handler_id: 持久投递的目标处理器;None 保留全体派发的兼容行为
|
||||
"""
|
||||
event = Event(etype, data, priority)
|
||||
with self.__lifecycle_lock:
|
||||
@@ -628,6 +647,7 @@ class EventManager(metaclass=Singleton):
|
||||
self.__dispatcher.dispatch_broadcast_strict(
|
||||
event,
|
||||
self.__wait_strict_async_handler,
|
||||
handler_id=handler_id,
|
||||
)
|
||||
return event
|
||||
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from functools import partial
|
||||
from hashlib import sha256
|
||||
|
||||
from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.application.chain.events import (
|
||||
@@ -14,8 +17,11 @@ from app.application.outbox import (
|
||||
DOWNLOAD_MODULE_TOPIC,
|
||||
DOWNLOAD_NOTIFICATION_TOPIC,
|
||||
DOWNLOAD_SUBTITLE_TOPIC,
|
||||
DURABLE_EVENT_TOPICS,
|
||||
EVENT_HANDLER_TOPIC_PREFIX,
|
||||
ClaimedOutboxMessage,
|
||||
OutboxDispatcher,
|
||||
OutboxIntent,
|
||||
durable_event_topic,
|
||||
reset_outbox_dispatcher,
|
||||
validate_durable_event_handlers,
|
||||
@@ -30,20 +36,36 @@ from app.schemas.message import Message
|
||||
from app.schemas.types import EventType
|
||||
|
||||
|
||||
def _stage_event_handlers(event_type: EventType, message: ClaimedOutboxMessage) -> None:
|
||||
"""
|
||||
冻结启用订阅并持久化逐处理器投递,避免第三方调用阻塞父主题的恢复。
|
||||
"""
|
||||
intents = []
|
||||
for handler_id in EventManager().broadcast_handler_ids(event_type):
|
||||
handler_digest = sha256(handler_id.encode()).hexdigest()
|
||||
delivery_digest = sha256(f"{message.event_key}\0{handler_id}".encode()).hexdigest()
|
||||
intents.append(OutboxIntent(
|
||||
event_key=f"event.handler:{delivery_digest}",
|
||||
topic=f"{EVENT_HANDLER_TOPIC_PREFIX}{handler_digest}",
|
||||
payload={
|
||||
"event_type": event_type.value,
|
||||
"event_key": message.event_key,
|
||||
"handler_id": handler_id,
|
||||
"event_payload": message.payload,
|
||||
},
|
||||
))
|
||||
SqlAlchemyOutboxDispatchStore(SessionFactory).stage_fanout(
|
||||
message, tuple(intents), datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
def build_outbox_handlers() -> dict[
|
||||
str,
|
||||
Callable[[ClaimedOutboxMessage], None],
|
||||
]:
|
||||
"""
|
||||
构造等待真实执行边界的 at-least-once 通知、事件和统计 handler。
|
||||
构造持久事件展开、通知和统计 handler,事件处理器由独立消息等待真实结算。
|
||||
"""
|
||||
event_manager_factory: Callable[[], EventManager] = EventManager
|
||||
|
||||
def discard_event_receipt(_event: object) -> None:
|
||||
"""
|
||||
丢弃普通事件 API 的回执,使 outbox handler 仅表达结算成功。
|
||||
"""
|
||||
|
||||
def dispatch_subscribe_deleted_report(message: ClaimedOutboxMessage) -> None:
|
||||
"""
|
||||
重放订阅删除统计;未确认时抛错以进入有限重试。
|
||||
@@ -132,85 +154,51 @@ def build_outbox_handlers() -> dict[
|
||||
)
|
||||
|
||||
handlers: dict[str, Callable[[ClaimedOutboxMessage], None]] = {
|
||||
durable_event_topic(EventType.SubscribeAdded): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.SubscribeAdded,
|
||||
message.payload,
|
||||
)
|
||||
),
|
||||
durable_event_topic(event_type): partial(_stage_event_handlers, event_type)
|
||||
for event_type in DURABLE_EVENT_TOPICS
|
||||
}
|
||||
handlers.update({
|
||||
"subscribe.added.report": dispatch_subscribe_added_report,
|
||||
"subscribe.added.notification": dispatch_subscribe_added_notification,
|
||||
durable_event_topic(EventType.SubscribeModified): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.SubscribeModified,
|
||||
message.payload,
|
||||
)
|
||||
),
|
||||
durable_event_topic(EventType.SubscribeDeleted): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.SubscribeDeleted,
|
||||
message.payload,
|
||||
)
|
||||
),
|
||||
"subscribe.deleted.report": dispatch_subscribe_deleted_report,
|
||||
durable_event_topic(EventType.SubscribeComplete): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.SubscribeComplete,
|
||||
message.payload,
|
||||
)
|
||||
),
|
||||
"subscribe.complete.report": dispatch_subscribe_complete_report,
|
||||
"subscribe.complete.notification": dispatch_subscribe_notification,
|
||||
durable_event_topic(EventType.DownloadAdded): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.DownloadAdded,
|
||||
restore_download_added(message.payload),
|
||||
)
|
||||
),
|
||||
DOWNLOAD_NOTIFICATION_TOPIC: dispatch_download_notification,
|
||||
DOWNLOAD_MODULE_TOPIC: dispatch_download_module,
|
||||
DOWNLOAD_SUBTITLE_TOPIC: dispatch_download_subtitle,
|
||||
durable_event_topic(EventType.TransferComplete): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.TransferComplete,
|
||||
restore_transfer_result(message.payload),
|
||||
)
|
||||
),
|
||||
durable_event_topic(EventType.TransferFailed): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.TransferFailed,
|
||||
restore_transfer_result(message.payload),
|
||||
)
|
||||
),
|
||||
durable_event_topic(EventType.SubtitleTransferComplete): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.SubtitleTransferComplete,
|
||||
restore_transfer_result(message.payload),
|
||||
)
|
||||
),
|
||||
durable_event_topic(EventType.SubtitleTransferFailed): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.SubtitleTransferFailed,
|
||||
restore_transfer_result(message.payload),
|
||||
)
|
||||
),
|
||||
durable_event_topic(EventType.AudioTransferComplete): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.AudioTransferComplete,
|
||||
restore_transfer_result(message.payload),
|
||||
)
|
||||
),
|
||||
durable_event_topic(EventType.AudioTransferFailed): lambda message: discard_event_receipt(
|
||||
event_manager_factory().send_event_strict(
|
||||
EventType.AudioTransferFailed,
|
||||
restore_transfer_result(message.payload),
|
||||
)
|
||||
),
|
||||
}
|
||||
})
|
||||
validate_durable_event_handlers(handlers)
|
||||
return handlers
|
||||
|
||||
|
||||
def dispatch_event_handler(message: ClaimedOutboxMessage) -> None:
|
||||
"""
|
||||
恢复一个处理器的事件对象并等待真实完成,失败只重试该处理器的投递记录。
|
||||
|
||||
保留父事件的幂等键与关联上下文,子消息身份仅用于隔离租约及结算。
|
||||
不取消仍在执行的第三方代码,关闭与续租继续由 OutboxRecovery 持有。
|
||||
"""
|
||||
if not message.topic.startswith(EVENT_HANDLER_TOPIC_PREFIX):
|
||||
raise ValueError(f"未登记的 Outbox 主题:{message.topic}")
|
||||
event_type = EventType(message.payload["event_type"])
|
||||
if event_type not in DURABLE_EVENT_TOPICS:
|
||||
raise ValueError(f"未登记的持久事件:{event_type.value}")
|
||||
payload = message.payload["event_payload"]
|
||||
if event_type == EventType.DownloadAdded:
|
||||
payload = restore_download_added(payload)
|
||||
elif event_type in {
|
||||
EventType.TransferComplete, EventType.TransferFailed,
|
||||
EventType.SubtitleTransferComplete, EventType.SubtitleTransferFailed,
|
||||
EventType.AudioTransferComplete, EventType.AudioTransferFailed,
|
||||
}:
|
||||
payload = restore_transfer_result(payload)
|
||||
with correlation_scope(message.payload["event_key"]):
|
||||
manager = EventManager()
|
||||
manager.send_event_strict(
|
||||
event_type, payload, handler_id=message.payload["handler_id"],
|
||||
)
|
||||
|
||||
|
||||
def build_outbox_dispatcher() -> OutboxDispatcher:
|
||||
"""
|
||||
创建使用独立短事务和 attempt fencing 的恢复 dispatcher。
|
||||
@@ -218,6 +206,7 @@ def build_outbox_dispatcher() -> OutboxDispatcher:
|
||||
return OutboxDispatcher(
|
||||
repository=SqlAlchemyOutboxDispatchStore(SessionFactory),
|
||||
handlers=build_outbox_handlers(),
|
||||
fallback_handler=dispatch_event_handler,
|
||||
failure_observer=lambda dead: record_metric(
|
||||
"scheduler.job.dead_letter" if dead else "scheduler.job.retry",
|
||||
owner="outbox",
|
||||
|
||||
@@ -415,9 +415,10 @@ collector 只接受 canonical `eventmanager`、`EventManager()` 及其有限别
|
||||
`register`/`add_event_listener`;当前宿主有 17 个静态注册点,另保留 1 个由工作流配置驱动的
|
||||
真实动态注册。`app/plugins/**` 插件副本不进入宿主事实。
|
||||
|
||||
生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 86 个
|
||||
生产调用,其中 85 个静态解析为 87 个事件引用,只有 `Command.send_plugin_event` 的插件事件类型
|
||||
保持动态;18 个消费注册中 17 个静态、1 个动态。生成的
|
||||
生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 76 个
|
||||
生产调用,其中 74 个静态解析为 76 个事件引用,`Command.send_plugin_event` 的插件事件类型和
|
||||
`dispatch_event_handler` 的持久事件恢复类型保持动态;后者只接受 `DURABLE_EVENT_TOPICS`
|
||||
登记的事件。18 个消费注册中 17 个静态、1 个动态。生成的
|
||||
`runtime-contract-baseline.json` 保存 line-free 事实、数量和枚举索引;人工维护的
|
||||
`runtime-contract-policy.json` 只批准 consumer 的精确 fingerprint、owner 和理由,任何新增、替换、
|
||||
重复或陈旧项都会失败,`--write-host` 不会改写该 policy。
|
||||
@@ -488,7 +489,10 @@ flowchart LR
|
||||
`sdk/_legacy/subscribe.py` + Compat 门面保留插件 ABI,不进入 canonical `__all__`。
|
||||
- **Outbox 可靠副作用**:业务行与 durable intent 在同一 Session/UoW 中提交;提交后由
|
||||
Outbox dispatcher 依据 topic、claim/lease、有限重试和 dead-letter 执行。完成通知、事件和统计
|
||||
的 post-commit 逻辑必须保持幂等,不能用普通线程或 TaskRegistry 代替持久 intent。终态历史随统一
|
||||
的 post-commit 逻辑必须保持幂等,不能用普通线程或 TaskRegistry 代替持久 intent。持久广播先
|
||||
冻结接收方并写入逐处理器子消息;父消息完成表示展开已持久化,实际结果由 `event.handler.*`
|
||||
子消息结算。每个处理器独立串行恢复,复用总计最多 20 个 worker、续租与关闭屏障,慢插件
|
||||
只积压自身投递,失败不会重放已成功的其他处理器。终态历史随统一
|
||||
数据维护任务分批清理,默认成功记录保留 30 天、dead letter 保留 90 天;总开关和两项保留期由
|
||||
高级设置维护,待投递和 lease 中记录不参与清理。
|
||||
- **统一历史保留期**:所有可安全按时间回收的追加型数据均受 `DATA_CLEANUP_ENABLE` 控制,包括消息、
|
||||
@@ -815,7 +819,7 @@ flowchart LR
|
||||
|
||||
### 10.2 当前架构快照
|
||||
|
||||
以下是 2026-11-02 核对的 `tests/fixtures/architecture/` 数据;这是源码合同快照,
|
||||
以下是 2026-11-05 核对的 `tests/fixtures/architecture/` 数据;这是源码合同快照,
|
||||
不代表某个已发布镜像或正在运行的实例已经验证。
|
||||
|
||||
| 指标 | 当前值 |
|
||||
@@ -827,7 +831,7 @@ flowchart LR
|
||||
| Direct egress | 51(精确 policy 准入,Application / Chain 均为 0) |
|
||||
| Module Contract V2 spec | 224(其中 220 个方法进入 `run_module` 观察面) |
|
||||
| Event Contract | 53 |
|
||||
| Event producer / consumer | 86(85 静态、1 动态)/ 18(17 静态、1 动态) |
|
||||
| Event producer / consumer | 76(74 静态、2 动态)/ 18(17 静态、1 动态) |
|
||||
| Model/Oper 自动事务与自建 Session | 0 |
|
||||
| 组合根外 `SystemConfigOper()` | 0 |
|
||||
|
||||
|
||||
@@ -996,6 +996,19 @@ Durable post-commit side effects have a separate boundary:
|
||||
before plugin, module or database teardown; a timeout preserves ownership and
|
||||
fails shutdown rather than resetting bindings. Retry backoff starts when the
|
||||
failed handler finishes, not when the message was claimed.
|
||||
- Durable broadcast topics first freeze their enabled handler identifiers and
|
||||
persist one delivery per handler in a fenced short transaction. The parent
|
||||
records that expansion in the same transaction; replay after a settlement
|
||||
failure reuses the committed recipients. Parent completion confirms durable
|
||||
expansion, while `event.handler.<sha256>` records track actual consumer
|
||||
completion. Each stable handler has its own serial recovery topic, so a slow
|
||||
plugin cannot serialize the same batch for its peers. All deliveries share the
|
||||
existing 20-worker bound, lease renewal and shutdown ownership; no third-party
|
||||
thread is abandoned or treated as complete on timeout. Child envelopes retain
|
||||
the original event payload, idempotency key and correlation context. The
|
||||
strict event API can select one registered handler; missing handlers retry
|
||||
through reload, disabled handlers retain the existing skip behavior. An
|
||||
individual failure retries only that delivery, never successful peers.
|
||||
- `PostCommitResult` distinguishes the committed business value from completed
|
||||
and pending effects. This does not provide exactly-once delivery: a process may
|
||||
stop after an external sink succeeds but before complete is persisted. Outbox
|
||||
|
||||
@@ -62,6 +62,12 @@ focused 测试验证,不要求启动产品全套测试。
|
||||
提交失败与取消后的重试,以及关闭超时保留资源。`tests/test_telegram.py` 覆盖海报失败后的文本
|
||||
兜底、会话/话题/按钮保留、429 不追加请求,以及发送失败不能被 outbox 当作成功确认。
|
||||
|
||||
`tests/test_outbox_event_isolation.py` 使用临时 SQLite、真实恢复线程和可释放屏障验证逐处理器
|
||||
持久投递:慢处理器尚未返回时,其他处理器已经收到同批十二集;失败只重试对应投递,父消息
|
||||
重放不重复展开或增加新订阅者。用例同时覆盖事务回滚、陈旧 attempt、租约续期、异步处理器、
|
||||
重载窗口和关闭超时保留 owner。父事件完成表示投递记录已持久化,实际执行结果以
|
||||
`event.handler.*` 子记录为准;这些离线测试不代表真实插件或媒体服务器验收。
|
||||
|
||||
AppPush 的所有用户范围由 `tests/test_notification_routing_parity.py` 验证:下载、入库和订阅消息
|
||||
经同步、异步与持久事件入口到达真实渠道;明确用户和管理员范围保持隔离,组合范围不重复发送。
|
||||
`tests/test_user_repository.py` 使用临时数据库验证接收集合只含启用账号的不可变身份,
|
||||
|
||||
@@ -1125,19 +1125,24 @@ class _EventFactCollector(ast.NodeVisitor):
|
||||
node: ast.Call,
|
||||
parameter_names: tuple[str, ...],
|
||||
required_names: frozenset[str],
|
||||
*,
|
||||
keyword_only_names: frozenset[str] = frozenset(),
|
||||
) -> dict[str, ast.AST] | None:
|
||||
"""
|
||||
按真实 Python 调用规则绑定有限签名,拒绝未知或重复参数。
|
||||
|
||||
:param keyword_only_names: 仅可通过关键字传递的可选参数名
|
||||
"""
|
||||
if len(node.args) > len(parameter_names) or any(
|
||||
isinstance(argument, ast.Starred) for argument in node.args
|
||||
):
|
||||
return None
|
||||
arguments = dict(zip(parameter_names, node.args))
|
||||
allowed_names = frozenset(parameter_names) | keyword_only_names
|
||||
for keyword in node.keywords:
|
||||
if (
|
||||
keyword.arg is None
|
||||
or keyword.arg not in parameter_names
|
||||
or keyword.arg not in allowed_names
|
||||
or keyword.arg in arguments
|
||||
):
|
||||
return None
|
||||
@@ -1205,6 +1210,9 @@ class _EventFactCollector(ast.NodeVisitor):
|
||||
node,
|
||||
("etype", "data", "priority"),
|
||||
frozenset({"etype"}),
|
||||
keyword_only_names=(
|
||||
frozenset({"handler_id"}) if bound_method.method == "send_event_strict" else frozenset()
|
||||
),
|
||||
)
|
||||
if arguments is None:
|
||||
return None
|
||||
|
||||
@@ -120,7 +120,6 @@
|
||||
"app/runtime/tasks.py:TaskRegistry.submit_threadsafe": 25,
|
||||
"app/scheduler/bridge.py:SchedulerBridgeOwner._submit_cross_thread": 16,
|
||||
"app/scheduler/execution.py:SchedulerExecutionOwner.start": 21,
|
||||
"app/startup/composition/outbox.py:build_outbox_handlers": 16,
|
||||
"app/workflow/__init__.py:WorkflowManager._check_complex_condition": 16,
|
||||
"app/workflow/actions/fetch_torrents.py:FetchTorrentsAction.execute": 16,
|
||||
"app/workflow/actions/transfer_file.py:TransferFileAction.execute": 19
|
||||
|
||||
@@ -1654,7 +1654,7 @@
|
||||
}
|
||||
],
|
||||
"dynamic_consumer_count": 1,
|
||||
"dynamic_producer_count": 1,
|
||||
"dynamic_producer_count": 2,
|
||||
"event_count": 53,
|
||||
"event_index": {
|
||||
"ChainEventType.AgentLLMProvider": {
|
||||
@@ -1809,15 +1809,13 @@
|
||||
"EventType.AudioTransferComplete": {
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"75aa6118952ed941e778b0347b281cf07cff05858b658c811fdbead76a23341f",
|
||||
"93c221b3f8700f899d9613ba205f36d521c76ed5681048c0a754bcb73e6749bf"
|
||||
]
|
||||
},
|
||||
"EventType.AudioTransferFailed": {
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"174dd87157d87d94e5656f4251b33005914e52e2c5fa1b53342188a4641b3225",
|
||||
"57da97424ec6e2f81255d39105e0e4047226b47c05ecd5e9ab64b1164e79de53"
|
||||
"174dd87157d87d94e5656f4251b33005914e52e2c5fa1b53342188a4641b3225"
|
||||
]
|
||||
},
|
||||
"EventType.CommandExcute": {
|
||||
@@ -1842,7 +1840,6 @@
|
||||
"EventType.DownloadAdded": {
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"35375727017385323cceeeb4db950615a604b51c6a4783171cc506256f3002d6",
|
||||
"6ab7645a6d0422f873242c8c9162c67904948890f429467a7f533e218aec1554",
|
||||
"6ab7645a6d0422f873242c8c9162c67904948890f429467a7f533e218aec1554"
|
||||
]
|
||||
@@ -1945,14 +1942,12 @@
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"019a094d2d0fd4fa84076bfab9263c2bf33ece199f4d5a9a5b57f68e07c4a9c0",
|
||||
"97717bff58a8806d2696054f1a1c9865954d2f0218dab51fb27027f0d560fbea",
|
||||
"cc7bd553745df756f0a69b1cf42852ab166c9fc6a58a546998b2956a0b5d7305"
|
||||
"97717bff58a8806d2696054f1a1c9865954d2f0218dab51fb27027f0d560fbea"
|
||||
]
|
||||
},
|
||||
"EventType.SubscribeComplete": {
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"c86fcb5d64d2367e007f01ecdccf01de80f3e0a0dae625c77246d698e1ca9d4b",
|
||||
"eba9652f1bf1fab63b1ab459a1636e2167e03142099cfd2dd665dab6c88989f9"
|
||||
]
|
||||
},
|
||||
@@ -1961,7 +1956,6 @@
|
||||
"producer_fingerprints": [
|
||||
"7ee0ba0bfdba520a389d45c33d361120a0d710c3ea37b1e03532dd7364a52d47",
|
||||
"a08051c5da0861d7198f3f5aa1b4ce6ed054abfbeab35e06ecb64dea71ba8c5e",
|
||||
"b0e58582f05faa7b8385a100589bcd724072a41cd3523615d7ac927e3272c7f8",
|
||||
"ea7538a3f2ad3ddbf8de6264e0525f065eec338d2ee003a501b725e3194d490a"
|
||||
]
|
||||
},
|
||||
@@ -1969,7 +1963,6 @@
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"89165b03827d9801a260dd54dd03e77f903795b8872bd043a79e8283181c786f",
|
||||
"b71e46714bd811b9c1703fecb112d0de83dd12e89ee70529702817958a9759a8",
|
||||
"ca9bf55c3886cd02fa31feade91fe333377fa8bf7bc9cc2d1bbdfb45214acfa3",
|
||||
"dbd5afd0aa1c2b449880ad3853e7e988dc29d799d40d2309780cf133a7e3c4a5"
|
||||
]
|
||||
@@ -1977,15 +1970,13 @@
|
||||
"EventType.SubtitleTransferComplete": {
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"b2bacfe7cc27d289b4744dc2f7b904fb8f20368e79c60adab6988188cc011eea",
|
||||
"f8ed3d63cdd6f007dd4f281c1e493b5fa5ffd7dc1591576a0224618315531fa7"
|
||||
]
|
||||
},
|
||||
"EventType.SubtitleTransferFailed": {
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"45f2e225d176c9d7d49feedd0e9b3e7510760a286e89ec26b57d8173b6d4c002",
|
||||
"7acfc1c2dd0ca50b8f9b72d55de769507e74851fb88381d66298f649078f5d9b"
|
||||
"45f2e225d176c9d7d49feedd0e9b3e7510760a286e89ec26b57d8173b6d4c002"
|
||||
]
|
||||
},
|
||||
"EventType.SystemError": {
|
||||
@@ -2002,15 +1993,13 @@
|
||||
"ff208671796f8e35b6d88733b5038a1799f33f67db8e62e9f61189d4e91a4d8e"
|
||||
],
|
||||
"producer_fingerprints": [
|
||||
"6b3cb59b259d9ca82a52fe8376504ad848351c433d4427efd9b496874acdb1b2",
|
||||
"b9c576056cd1bea0a768eea4e927144e393e194c44a0f0b6f3fe1a86fe91ddd5"
|
||||
]
|
||||
},
|
||||
"EventType.TransferFailed": {
|
||||
"consumer_fingerprints": [],
|
||||
"producer_fingerprints": [
|
||||
"8ac0a3e2e96daa1e00ed16560f57bbe1ad55263753b46d1f8f196c69d80e978f",
|
||||
"d0fb23c410c6fda68c9e676f107666e31e0828886f5d9aeb4f5d3dcd9fc2bda7"
|
||||
"8ac0a3e2e96daa1e00ed16560f57bbe1ad55263753b46d1f8f196c69d80e978f"
|
||||
]
|
||||
},
|
||||
"EventType.UserMessage": {
|
||||
@@ -2034,11 +2023,11 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"fact_count": 104,
|
||||
"fact_count": 94,
|
||||
"invalid_consumer_count": 0,
|
||||
"invalid_producer_count": 0,
|
||||
"producer_call_count": 86,
|
||||
"producer_event_reference_count": 87,
|
||||
"producer_call_count": 76,
|
||||
"producer_event_reference_count": 76,
|
||||
"producers": [
|
||||
{
|
||||
"caller": "app.agent.orchestrator",
|
||||
@@ -2834,134 +2823,12 @@
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.DownloadAdded"
|
||||
],
|
||||
"fingerprint": "35375727017385323cceeeb4db950615a604b51c6a4783171cc506256f3002d6",
|
||||
"dynamic": true,
|
||||
"events": [],
|
||||
"fingerprint": "cfb3f40492fabb5d57078f54ba5b72e5e104ee22f619e8105598dfcba8662619",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.AudioTransferFailed"
|
||||
],
|
||||
"fingerprint": "57da97424ec6e2f81255d39105e0e4047226b47c05ecd5e9ab64b1164e79de53",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.TransferComplete"
|
||||
],
|
||||
"fingerprint": "6b3cb59b259d9ca82a52fe8376504ad848351c433d4427efd9b496874acdb1b2",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.AudioTransferComplete"
|
||||
],
|
||||
"fingerprint": "75aa6118952ed941e778b0347b281cf07cff05858b658c811fdbead76a23341f",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.SubtitleTransferFailed"
|
||||
],
|
||||
"fingerprint": "7acfc1c2dd0ca50b8f9b72d55de769507e74851fb88381d66298f649078f5d9b",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.SubscribeDeleted"
|
||||
],
|
||||
"fingerprint": "b0e58582f05faa7b8385a100589bcd724072a41cd3523615d7ac927e3272c7f8",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.SubtitleTransferComplete"
|
||||
],
|
||||
"fingerprint": "b2bacfe7cc27d289b4744dc2f7b904fb8f20368e79c60adab6988188cc011eea",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.SubscribeModified"
|
||||
],
|
||||
"fingerprint": "b71e46714bd811b9c1703fecb112d0de83dd12e89ee70529702817958a9759a8",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.SubscribeComplete"
|
||||
],
|
||||
"fingerprint": "c86fcb5d64d2367e007f01ecdccf01de80f3e0a0dae625c77246d698e1ca9d4b",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.SubscribeAdded"
|
||||
],
|
||||
"fingerprint": "cc7bd553745df756f0a69b1cf42852ab166c9fc6a58a546998b2956a0b5d7305",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
"caller": "app.startup.composition.outbox",
|
||||
"dynamic": false,
|
||||
"events": [
|
||||
"EventType.TransferFailed"
|
||||
],
|
||||
"fingerprint": "d0fb23c410c6fda68c9e676f107666e31e0828886f5d9aeb4f5d3dcd9fc2bda7",
|
||||
"invalid": false,
|
||||
"method": "send_event_strict",
|
||||
"qualname": "build_outbox_handlers",
|
||||
"qualname": "dispatch_event_handler",
|
||||
"receiver_kind": "constructed_manager"
|
||||
},
|
||||
{
|
||||
@@ -3074,7 +2941,7 @@
|
||||
}
|
||||
],
|
||||
"static_consumer_count": 17,
|
||||
"static_producer_call_count": 85
|
||||
"static_producer_call_count": 74
|
||||
},
|
||||
"event_specs": {
|
||||
"ChainEventType.AgentLLMProvider": {
|
||||
|
||||
@@ -452,17 +452,17 @@ def test_event_contract_baseline_covers_every_public_event_enum() -> None:
|
||||
|
||||
assert set(events["event_index"]) == expected
|
||||
assert events["event_count"] == len(expected)
|
||||
assert events["producer_call_count"] == 86
|
||||
assert events["static_producer_call_count"] == 85
|
||||
assert events["dynamic_producer_count"] == 1
|
||||
assert events["producer_call_count"] == 76
|
||||
assert events["static_producer_call_count"] == 74
|
||||
assert events["dynamic_producer_count"] == 2
|
||||
assert events["invalid_producer_count"] == 0
|
||||
assert events["producer_event_reference_count"] == 87
|
||||
assert events["producer_event_reference_count"] == 76
|
||||
assert events["consumer_registration_count"] == 18
|
||||
assert events["static_consumer_count"] == 17
|
||||
assert events["dynamic_consumer_count"] == 1
|
||||
assert events["invalid_consumer_count"] == 0
|
||||
assert events["consumer_event_reference_count"] == 17
|
||||
assert events["fact_count"] == 104
|
||||
assert events["fact_count"] == 94
|
||||
assert len({fact["fingerprint"] for fact in events["consumers"]}) == 18
|
||||
assert all(
|
||||
not fact["caller"].startswith("app.plugins")
|
||||
|
||||
@@ -1194,19 +1194,44 @@ def handler(event):
|
||||
assert facts["consumers"][0]["caller"] == "app.host"
|
||||
|
||||
|
||||
def test_collect_event_facts_recognizes_targeted_strict_delivery(tmp_path: Path) -> None:
|
||||
"""
|
||||
处理器定向严格投递仍进入事件事实,关键字专用参数不能作为第四个位置参数。
|
||||
"""
|
||||
facts = _collect_facts(tmp_path, '''
|
||||
from app.runtime.events import EventManager
|
||||
from app.schemas.types import EventType
|
||||
|
||||
def publish(kind):
|
||||
"""
|
||||
展示定向投递与非法签名的事件事实解析边界。
|
||||
"""
|
||||
manager = EventManager()
|
||||
manager.send_event_strict(EventType.Alpha, {}, handler_id="app.plugins.sample.handle")
|
||||
manager.send_event_strict(kind, {}, handler_id="app.plugins.sample.handle")
|
||||
manager.send_event_strict(EventType.Beta, {}, 10, "invalid")
|
||||
manager.send_event(EventType.Gamma, handler_id="invalid")
|
||||
manager.async_send_event_strict(EventType.Delta, handler_id="invalid")
|
||||
''')
|
||||
assert len(facts["producers"]) == 2
|
||||
assert facts["producers"][0]["events"] == ["EventType.Alpha"]
|
||||
assert facts["producers"][1]["dynamic"]
|
||||
assert all(fact["receiver_kind"] == "constructed_manager" for fact in facts["producers"])
|
||||
|
||||
|
||||
def test_collect_event_facts_matches_current_host_inventory() -> None:
|
||||
"""
|
||||
统一事实覆盖当前 86 个 producer 调用与 18 个 consumer 调用。
|
||||
统一事实覆盖当前 76 个 producer 调用与 18 个 consumer 调用。
|
||||
"""
|
||||
facts = current_event_facts()
|
||||
producers = facts["producers"]
|
||||
consumers = facts["consumers"]
|
||||
|
||||
assert len(producers) == 86
|
||||
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 85
|
||||
assert sum(fact["dynamic"] for fact in producers) == 1
|
||||
assert len(producers) == 76
|
||||
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 74
|
||||
assert sum(fact["dynamic"] for fact in producers) == 2
|
||||
assert sum(fact["invalid"] for fact in producers) == 0
|
||||
assert sum(len(fact["events"]) for fact in producers) == 87
|
||||
assert sum(len(fact["events"]) for fact in producers) == 76
|
||||
assert len(consumers) == 18
|
||||
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in consumers) == 17
|
||||
assert sum(fact["dynamic"] for fact in consumers) == 1
|
||||
|
||||
@@ -37,7 +37,7 @@ from app.application.outbox import (
|
||||
DOWNLOAD_MODULE_TOPIC,
|
||||
DOWNLOAD_NOTIFICATION_TOPIC,
|
||||
DOWNLOAD_SUBTITLE_TOPIC,
|
||||
TRANSFER_COMPLETED_TOPIC,
|
||||
EVENT_HANDLER_TOPIC_PREFIX,
|
||||
ClaimedOutboxMessage,
|
||||
OutboxDispatcher,
|
||||
PostCommitEffectError,
|
||||
@@ -357,7 +357,7 @@ def test_durable_music_snapshots_keep_explicit_music_type(media_type, title):
|
||||
|
||||
def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch):
|
||||
"""
|
||||
真实整理 handler 恢复 AAC 尾缀影视快照后应派发事件并完成消息结算。
|
||||
真实处理器子投递恢复 AAC 尾缀影视快照后应派发事件并完成自身结算。
|
||||
"""
|
||||
from app.startup.composition import outbox as outbox_composition
|
||||
|
||||
@@ -374,13 +374,17 @@ def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch):
|
||||
"idempotency_key": "transfer.completed:7023:v1",
|
||||
})))
|
||||
message = ClaimedOutboxMessage(
|
||||
7023, "transfer.completed:7023:v1", TRANSFER_COMPLETED_TOPIC, payload, 1, 1,
|
||||
7023, "event.handler:7023:v1", f"{EVENT_HANDLER_TOPIC_PREFIX}test",
|
||||
{"event_type": EventType.TransferComplete.value,
|
||||
"event_key": payload["idempotency_key"], "handler_id": "test.video_handler",
|
||||
"event_payload": payload}, 1, 1,
|
||||
)
|
||||
repository = Mock()
|
||||
repository.claim.return_value = message
|
||||
event_manager = Mock()
|
||||
monkeypatch.setattr(outbox_composition, "EventManager", lambda: event_manager)
|
||||
dispatcher = OutboxDispatcher(repository, outbox_composition.build_outbox_handlers())
|
||||
dispatcher = OutboxDispatcher(repository, outbox_composition.build_outbox_handlers(),
|
||||
fallback_handler=outbox_composition.dispatch_event_handler)
|
||||
|
||||
assert dispatcher.dispatch_one() is True
|
||||
|
||||
@@ -390,7 +394,8 @@ def test_transfer_outbox_dispatches_video_release_ending_in_aac(monkeypatch):
|
||||
assert restored["meta"].type == MediaType.TV
|
||||
assert restored["meta"].to_dict() == meta.to_dict()
|
||||
assert restored["fileitem"] == fileitem
|
||||
assert restored["idempotency_key"] == message.event_key
|
||||
assert restored["idempotency_key"] == payload["idempotency_key"]
|
||||
assert event_manager.send_event_strict.call_args.kwargs == {"handler_id": "test.video_handler"}
|
||||
repository.complete.assert_called_once()
|
||||
assert repository.complete.call_args.args[:2] == (message.message_id, message.attempt)
|
||||
repository.retry.assert_not_called()
|
||||
|
||||
@@ -1089,8 +1089,9 @@ def test_outbox_retries_strict_failure_without_repeating_system_alert(
|
||||
"""
|
||||
from unittest.mock import Mock
|
||||
|
||||
from app.application.outbox import ClaimedOutboxMessage, OutboxDispatcher
|
||||
from app.application.outbox import EVENT_HANDLER_TOPIC_PREFIX, ClaimedOutboxMessage, OutboxDispatcher
|
||||
from app.runtime.event.errors import EventErrorPolicy
|
||||
from app.runtime.event.registry import EventRegistry
|
||||
from app.startup.composition import outbox
|
||||
|
||||
isolated_eventmanager._EventManager__lifecycle_state = "running"
|
||||
@@ -1111,12 +1112,16 @@ def test_outbox_retries_strict_failure_without_repeating_system_alert(
|
||||
store = Mock()
|
||||
event_key = f"{event_type.value}:task-1:1"
|
||||
store.claim.side_effect = [
|
||||
ClaimedOutboxMessage(1, event_key, event_type.value,
|
||||
{"idempotency_key": event_key,
|
||||
"transferinfo": {"success": False, "message": "failed"}}, 1, attempt)
|
||||
ClaimedOutboxMessage(1, "child-key", f"{EVENT_HANDLER_TOPIC_PREFIX}test",
|
||||
{"event_type": event_type.value,
|
||||
"event_key": event_key,
|
||||
"handler_id": EventRegistry.handler_identifier(failed_handler),
|
||||
"event_payload": {"idempotency_key": event_key,
|
||||
"transferinfo": {"success": False, "message": "failed"}}}, 1, attempt)
|
||||
for attempt in range(1, 6)
|
||||
] + [None]
|
||||
dispatcher = OutboxDispatcher(store, outbox.build_outbox_handlers())
|
||||
dispatcher = OutboxDispatcher(store, outbox.build_outbox_handlers(),
|
||||
fallback_handler=outbox.dispatch_event_handler)
|
||||
for _ in range(5):
|
||||
assert dispatcher.dispatch_one()
|
||||
assert not dispatcher.dispatch_one()
|
||||
|
||||
@@ -385,8 +385,9 @@ def test_startup_handler_replays_strict_boundary_with_stable_key(
|
||||
monkeypatch.setattr(
|
||||
EventManager,
|
||||
"send_event_strict",
|
||||
lambda _self, _etype, data: calls.append(data["idempotency_key"]),
|
||||
lambda _self, _etype, data, *, handler_id: calls.append((data["idempotency_key"], handler_id)),
|
||||
)
|
||||
monkeypatch.setattr(EventManager, "broadcast_handler_ids", lambda _self, _etype: ("test.handler",))
|
||||
elif handler_kind in {"notification", "download_notification"}:
|
||||
topic = (
|
||||
"subscribe.complete.notification"
|
||||
@@ -420,6 +421,10 @@ def test_startup_handler_replays_strict_boundary_with_stable_key(
|
||||
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(engine)
|
||||
factory = sessionmaker(bind=engine)
|
||||
if handler_kind == "event":
|
||||
from app.startup.composition import outbox as composition
|
||||
|
||||
monkeypatch.setattr(composition, "SessionFactory", factory)
|
||||
now = datetime(2026, 8, 28, tzinfo=timezone.utc)
|
||||
event_key = f"startup:{handler_kind}:v1"
|
||||
with factory() as session:
|
||||
@@ -439,7 +444,18 @@ def test_startup_handler_replays_strict_boundary_with_stable_key(
|
||||
clock=lambda: now + timedelta(seconds=2),
|
||||
)
|
||||
assert dispatcher.dispatch_one() is True
|
||||
assert calls == [event_key, event_key]
|
||||
if handler_kind == "event":
|
||||
from app.startup.composition.outbox import dispatch_event_handler
|
||||
|
||||
current = datetime.now(timezone.utc)
|
||||
child = store.claim(current, current + timedelta(seconds=60))
|
||||
assert child is not None
|
||||
dispatch_event_handler(child)
|
||||
assert calls == [(event_key, "test.handler")]
|
||||
assert store.complete(child.message_id, child.attempt, current)
|
||||
assert store.claim(current, current + timedelta(seconds=60)) is None
|
||||
else:
|
||||
assert calls == [event_key, event_key]
|
||||
|
||||
|
||||
def test_download_writer_contract_has_no_anonymous_after_commit_path() -> None:
|
||||
|
||||
431
tests/test_outbox_event_isolation.py
Normal file
431
tests/test_outbox_event_isolation.py
Normal file
@@ -0,0 +1,431 @@
|
||||
"""
|
||||
持久事件逐处理器投递的批次隔离、事务快照、重试和生命周期回归。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from threading import Event
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.application.outbox import (
|
||||
DURABLE_EVENT_TOPICS,
|
||||
EVENT_FANOUT_COMPLETE_FIELD,
|
||||
EVENT_HANDLER_TOPIC_PREFIX,
|
||||
OutboxDispatcher,
|
||||
OutboxIntent,
|
||||
OutboxLeaseLostError,
|
||||
OutboxRecovery,
|
||||
durable_event_topic,
|
||||
)
|
||||
from app.db.adapters.outbox import SqlAlchemyOutboxDispatchStore, SqlAlchemyOutboxStager
|
||||
from app.db.models.outbox import OutboxMessage
|
||||
from app.runtime.correlation import get_correlation_id
|
||||
from app.runtime.event.errors import EventErrorPolicy
|
||||
from app.runtime.event.registry import EventRegistry
|
||||
from app.runtime.events import eventmanager
|
||||
from app.runtime.loop import main_loop_registry
|
||||
from app.schemas.transfer import TransferInfo
|
||||
from app.schemas.types import EventType
|
||||
from app.startup.composition import outbox as composition
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runtime(tmp_path, monkeypatch):
|
||||
"""
|
||||
隔离事件订阅、错误通知和文件数据库,真实执行宿主投递及租约边界。
|
||||
"""
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'events.db'}")
|
||||
OutboxMessage.__table__.create(engine)
|
||||
factory = sessionmaker(engine, expire_on_commit=False)
|
||||
for attribute, value in {
|
||||
"__broadcast_subscribers": {},
|
||||
"__disabled_handlers": set(),
|
||||
"__disabled_classes": set(),
|
||||
"__handler_instance_resolvers": {},
|
||||
"__lifecycle_state": "running",
|
||||
"__error_policy": EventErrorPolicy(notifier=lambda: None, emit_system_error=lambda _data: None),
|
||||
}.items():
|
||||
monkeypatch.setattr(eventmanager, f"_EventManager{attribute}", value)
|
||||
monkeypatch.setattr(composition, "SessionFactory", factory)
|
||||
monkeypatch.setattr(composition, "EventManager", lambda: eventmanager)
|
||||
yield factory, SqlAlchemyOutboxDispatchStore(factory)
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def _stage(factory, count=1, event_type=EventType.TransferComplete):
|
||||
"""
|
||||
暂存保持旧插件载荷格式的整理或订阅事件,返回稳定业务事件键。
|
||||
"""
|
||||
keys = []
|
||||
with factory() as session:
|
||||
for index in range(count):
|
||||
key = f"{durable_event_topic(event_type)}:task-{index}:v1"
|
||||
keys.append(key)
|
||||
SqlAlchemyOutboxStager(session).stage(
|
||||
OutboxIntent(key, durable_event_topic(event_type), {
|
||||
"sequence": index,
|
||||
"subscribe_id": index,
|
||||
"transferinfo": {"success": True},
|
||||
}),
|
||||
datetime.now(timezone.utc),
|
||||
)
|
||||
session.commit()
|
||||
return keys
|
||||
|
||||
|
||||
def _rows(factory):
|
||||
"""
|
||||
返回脱离会话的消息快照,便于核对跨线程结算。
|
||||
"""
|
||||
with factory() as session:
|
||||
return session.scalars(select(OutboxMessage).order_by(OutboxMessage.id)).all()
|
||||
|
||||
|
||||
def _expand(factory, count=1):
|
||||
"""
|
||||
只处理先写入的父事件,确认此阶段不执行任何插件代码。
|
||||
"""
|
||||
dispatcher = composition.build_outbox_dispatcher()
|
||||
for _ in range(count):
|
||||
assert dispatcher.dispatch_one()
|
||||
return [row for row in _rows(factory) if row.topic.startswith(EVENT_HANDLER_TOPIC_PREFIX)]
|
||||
|
||||
|
||||
def test_blocked_handler_does_not_delay_the_other_handlers_entire_batch(runtime):
|
||||
"""
|
||||
首个插件未返回时另一个插件已收到全部十二集,慢插件仍持有租约和关闭屏障。
|
||||
"""
|
||||
factory, store = runtime
|
||||
started, release, received_all = Event(), Event(), Event()
|
||||
fast_calls, slow_calls = [], []
|
||||
|
||||
def slow(event):
|
||||
"""
|
||||
用可释放屏障代替插件的六十秒等待,不遗留后台执行。
|
||||
"""
|
||||
slow_calls.append(event.event_data["sequence"])
|
||||
started.set()
|
||||
assert release.wait(10)
|
||||
|
||||
def fast(event):
|
||||
"""
|
||||
验证对象恢复与关联标识,并记录整批接收顺序。
|
||||
"""
|
||||
assert isinstance(event.event_data["transferinfo"], TransferInfo)
|
||||
assert event.correlation_id == get_correlation_id() == event.event_data["idempotency_key"]
|
||||
fast_calls.append(event.event_data["sequence"])
|
||||
if len(fast_calls) == 12:
|
||||
received_all.set()
|
||||
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, slow)
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, fast)
|
||||
keys = _stage(factory, 12)
|
||||
children = _expand(factory, 12)
|
||||
assert len(children) == 24
|
||||
assert slow_calls == fast_calls == []
|
||||
assert all(row.status == "completed" for row in _rows(factory)[:12])
|
||||
assert {row.payload["event_key"] for row in children} == set(keys)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
recovery = OutboxRecovery(composition.build_outbox_dispatcher, pool.submit, max_inflight=2)
|
||||
try:
|
||||
assert recovery.dispatch(20) == 2
|
||||
assert started.wait(2)
|
||||
assert received_all.wait(5)
|
||||
assert fast_calls == list(range(12))
|
||||
assert slow_calls == [0]
|
||||
# 模拟租约过期后再次扫描,活跃处理器仍不允许启动第二个拥有者。
|
||||
slow_row = next(row for row in _rows(factory) if row.status == "processing")
|
||||
with factory() as session:
|
||||
session.get(OutboxMessage, slow_row.id).lease_until = "2000-01-01T00:00:00+00:00"
|
||||
session.commit()
|
||||
recovery.dispatch(20)
|
||||
renewed = next(row for row in _rows(factory) if row.id == slow_row.id)
|
||||
assert renewed.attempt == 1
|
||||
assert renewed.lease_until > datetime.now(timezone.utc).isoformat()
|
||||
assert not recovery.stop(0)
|
||||
assert slow_calls == [0]
|
||||
finally:
|
||||
release.set()
|
||||
assert recovery.stop(5)
|
||||
assert len(slow_calls) == 1
|
||||
assert sum(row.status == "pending" for row in _rows(factory)) == 11
|
||||
# 新恢复器重放尚未认领的记录,已经成功的接收方不会再收到整批事件。
|
||||
dispatcher = composition.build_outbox_dispatcher()
|
||||
while dispatcher.dispatch_one():
|
||||
pass
|
||||
assert slow_calls == list(range(12))
|
||||
assert fast_calls == list(range(12))
|
||||
assert all(row.status == "completed" for row in _rows(factory))
|
||||
assert store.claim(datetime.now(timezone.utc), datetime.now(timezone.utc)) is None
|
||||
|
||||
|
||||
def test_parent_replay_reuses_committed_snapshot_even_after_subscriptions_change(runtime):
|
||||
"""
|
||||
展开成功但父消息结算前崩溃时,重放不会再次投递或把新订阅加入旧事件。
|
||||
"""
|
||||
factory, store = runtime
|
||||
original, added_later = Mock(), Mock()
|
||||
|
||||
def first(event):
|
||||
"""
|
||||
记录原订阅处理器收到的稳定事件键。
|
||||
"""
|
||||
original(event.event_data["idempotency_key"])
|
||||
|
||||
def later(event):
|
||||
"""
|
||||
后加入的订阅不应进入已冻结的父事件快照。
|
||||
"""
|
||||
added_later(event)
|
||||
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, first)
|
||||
keys = _stage(factory)
|
||||
now = datetime.now(timezone.utc)
|
||||
parent = store.claim(now, now + timedelta(seconds=1))
|
||||
composition.build_outbox_handlers()[parent.topic](parent)
|
||||
assert _rows(factory)[0].payload[EVENT_FANOUT_COMPLETE_FIELD]
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, later)
|
||||
replay = store.claim(now + timedelta(seconds=2), now + timedelta(seconds=62))
|
||||
assert replay.message_id == parent.message_id and replay.attempt == 2
|
||||
composition.build_outbox_handlers()[replay.topic](replay)
|
||||
assert store.complete(replay.message_id, replay.attempt, now)
|
||||
# 模拟进程恢复,使用新的派发器与短会话完成已持久化的子消息。
|
||||
assert composition.build_outbox_dispatcher().dispatch_one()
|
||||
assert not composition.build_outbox_dispatcher().dispatch_one()
|
||||
original.assert_called_once_with(keys[0])
|
||||
added_later.assert_not_called()
|
||||
assert len(_rows(factory)) == 2
|
||||
|
||||
|
||||
def test_fanout_transaction_rolls_back_all_children_and_marker(runtime):
|
||||
"""
|
||||
子记录写入冲突时全事务回滚,父消息仍可安全重试展开。
|
||||
"""
|
||||
factory, store = runtime
|
||||
_stage(factory)
|
||||
now = datetime.now(timezone.utc)
|
||||
parent = store.claim(now, now + timedelta(seconds=60))
|
||||
duplicate = OutboxIntent("same-child", "event.handler.test", {})
|
||||
with pytest.raises(IntegrityError):
|
||||
store.stage_fanout(parent, (duplicate, duplicate), now)
|
||||
rows = _rows(factory)
|
||||
assert len(rows) == 1
|
||||
assert not rows[0].payload.get(EVENT_FANOUT_COMPLETE_FIELD)
|
||||
store.stage_fanout(parent, (duplicate,), now)
|
||||
assert len(_rows(factory)) == 2
|
||||
|
||||
|
||||
def test_stale_parent_cannot_expand_event(runtime):
|
||||
"""
|
||||
旧 attempt 不得写入子消息或覆盖新拥有者的展开状态。
|
||||
"""
|
||||
factory, store = runtime
|
||||
_stage(factory)
|
||||
now = datetime.now(timezone.utc)
|
||||
stale = store.claim(now, now + timedelta(seconds=1))
|
||||
current = store.claim(now + timedelta(seconds=2), now + timedelta(seconds=62))
|
||||
assert current.attempt == 2
|
||||
with pytest.raises(OutboxLeaseLostError):
|
||||
store.stage_fanout(stale, (OutboxIntent("stale-child", "event.handler.test", {}),), now)
|
||||
assert len(_rows(factory)) == 1
|
||||
|
||||
|
||||
def test_handler_failure_only_retries_its_own_delivery(runtime):
|
||||
"""
|
||||
失败处理器重试至死信,已完成的同事件处理器只执行一次。
|
||||
"""
|
||||
factory, store = runtime
|
||||
attempts, successful = [], []
|
||||
|
||||
def failing(event):
|
||||
"""
|
||||
模拟每次投递均失败的第三方处理器。
|
||||
"""
|
||||
attempts.append(event.event_data["idempotency_key"])
|
||||
raise RuntimeError("插件事件失败")
|
||||
|
||||
def peer(event):
|
||||
"""
|
||||
记录不应跟随其他处理器重试的成功接收方。
|
||||
"""
|
||||
successful.append(event.event_data["idempotency_key"])
|
||||
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, failing)
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, peer)
|
||||
keys = _stage(factory)
|
||||
_expand(factory)
|
||||
now = [datetime.now(timezone.utc)]
|
||||
dispatcher = OutboxDispatcher(store, composition.build_outbox_handlers(),
|
||||
fallback_handler=composition.dispatch_event_handler, clock=lambda: now[0])
|
||||
assert dispatcher.dispatch_one() and dispatcher.dispatch_one()
|
||||
for _ in range(4):
|
||||
now[0] += timedelta(hours=1)
|
||||
assert dispatcher.dispatch_one()
|
||||
assert not dispatcher.dispatch_one()
|
||||
assert attempts == keys * 5
|
||||
assert successful == keys
|
||||
rows = _rows(factory)
|
||||
assert [row.status for row in rows] == ["completed", "dead", "completed"]
|
||||
assert rows[1].payload["handler_id"] == EventRegistry.handler_identifier(failing)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("event_type", list(DURABLE_EVENT_TOPICS))
|
||||
def test_all_durable_topics_keep_payload_and_disabled_handler_semantics(runtime, event_type):
|
||||
"""
|
||||
所有持久事件保留业务载荷与幂等键,仅为启用处理器创建可恢复投递。
|
||||
"""
|
||||
factory, _store = runtime
|
||||
received = []
|
||||
|
||||
def enabled(event):
|
||||
"""
|
||||
记录实际收到的事件类型、数据和关联上下文。
|
||||
"""
|
||||
received.append((event.event_type, event.event_data, event.correlation_id))
|
||||
|
||||
def disabled(_event):
|
||||
"""
|
||||
禁用处理器不能执行,也不能生成子投递记录。
|
||||
"""
|
||||
raise AssertionError("禁用处理器不应执行")
|
||||
|
||||
eventmanager.add_event_listener(event_type, enabled)
|
||||
eventmanager.add_event_listener(event_type, disabled)
|
||||
eventmanager.disable_event_handler(disabled)
|
||||
keys = _stage(factory, event_type=event_type)
|
||||
children = _expand(factory)
|
||||
assert len(children) == 1
|
||||
assert len(children[0].topic) <= 100 and len(children[0].event_key) <= 255
|
||||
assert children[0].payload["handler_id"] == EventRegistry.handler_identifier(enabled)
|
||||
assert composition.build_outbox_dispatcher().dispatch_one()
|
||||
assert len(received) == 1
|
||||
kind, payload, correlation_id = received[0]
|
||||
assert kind == event_type
|
||||
assert payload["idempotency_key"] == keys[0] == correlation_id
|
||||
assert payload["sequence"] == 0
|
||||
|
||||
|
||||
def test_unregistered_target_is_retryable_and_does_not_call_peers(runtime):
|
||||
"""
|
||||
插件重载窗口内目标缺失时保留待重试消息,恢复同一标识后可继续投递。
|
||||
"""
|
||||
factory, store = runtime
|
||||
received = []
|
||||
|
||||
def handler(event):
|
||||
"""
|
||||
收集重载后接续投递的事件键。
|
||||
"""
|
||||
received.append(event.event_data["idempotency_key"])
|
||||
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, handler)
|
||||
keys = _stage(factory)
|
||||
_expand(factory)
|
||||
eventmanager.remove_event_listener(EventType.TransferComplete, handler)
|
||||
dispatcher = composition.build_outbox_dispatcher()
|
||||
assert dispatcher.dispatch_one()
|
||||
assert _rows(factory)[1].status == "pending"
|
||||
assert "尚未注册" in _rows(factory)[1].last_error
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, handler)
|
||||
later = datetime.now(timezone.utc) + timedelta(seconds=2)
|
||||
replay = store.claim(later, later + timedelta(seconds=60))
|
||||
dispatcher.dispatch_claimed(replay)
|
||||
assert received == keys
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_async_handler_waits_for_real_completion_without_blocking_peer(runtime):
|
||||
"""
|
||||
协程插件继续在主循环执行,独立线程拥有租约直到异步处理实际返回。
|
||||
"""
|
||||
factory, _store = runtime
|
||||
started, release, peer_done = asyncio.Event(), asyncio.Event(), Event()
|
||||
|
||||
async def slow(_event):
|
||||
"""
|
||||
用协程屏障保持处理器在途,验证没有提前结算。
|
||||
"""
|
||||
started.set()
|
||||
await release.wait()
|
||||
|
||||
def peer(_event):
|
||||
"""
|
||||
同事件的同步接收方应在协程屏障释放前完成。
|
||||
"""
|
||||
peer_done.set()
|
||||
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, slow)
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, peer)
|
||||
_stage(factory)
|
||||
_expand(factory)
|
||||
loop_owner = main_loop_registry.register(asyncio.get_running_loop())
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
recovery = OutboxRecovery(composition.build_outbox_dispatcher, pool.submit)
|
||||
try:
|
||||
assert recovery.dispatch(20) == 2
|
||||
await asyncio.wait_for(started.wait(), 2)
|
||||
assert await asyncio.to_thread(peer_done.wait, 2)
|
||||
assert any(row.status == "processing" for row in _rows(factory))
|
||||
assert not recovery.stop(0)
|
||||
finally:
|
||||
release.set()
|
||||
assert await asyncio.to_thread(recovery.stop, 5)
|
||||
finally:
|
||||
main_loop_registry.release(loop_owner)
|
||||
assert all(row.status == "completed" for row in _rows(factory))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topic,event_type,with_fallback", [
|
||||
("unknown", EventType.TransferComplete, True),
|
||||
("unknown", EventType.TransferComplete, False),
|
||||
(f"{EVENT_HANDLER_TOPIC_PREFIX}invalid", EventType.ConfigChanged, True),
|
||||
])
|
||||
def test_unknown_topic_or_event_remains_a_failed_delivery(runtime, topic, event_type, with_fallback):
|
||||
"""
|
||||
动态主题回调不接受未登记主题,不能把未知效果误记为成功。
|
||||
"""
|
||||
factory, store = runtime
|
||||
with factory() as session:
|
||||
SqlAlchemyOutboxStager(session).stage(
|
||||
OutboxIntent("unknown", topic, {"event_type": event_type.value}), datetime.now(timezone.utc),
|
||||
)
|
||||
session.commit()
|
||||
dispatcher = composition.build_outbox_dispatcher() if with_fallback else OutboxDispatcher(store, {})
|
||||
assert dispatcher.dispatch_one()
|
||||
row = _rows(factory)[0]
|
||||
assert row.status == "pending"
|
||||
assert ("未登记" if with_fallback else "unknown") in row.last_error
|
||||
|
||||
|
||||
def test_stopped_bus_cannot_settle_parent_or_child_delivery(runtime, monkeypatch):
|
||||
"""
|
||||
总线封口后不能把尚未展开的父消息或未执行的子投递误记为完成。
|
||||
"""
|
||||
factory, store = runtime
|
||||
|
||||
def handler(_event):
|
||||
"""
|
||||
封口后的任何调用都应先被生命周期边界拒绝。
|
||||
"""
|
||||
raise AssertionError("停止状态不应执行处理器")
|
||||
|
||||
eventmanager.add_event_listener(EventType.TransferComplete, handler)
|
||||
_stage(factory, 2)
|
||||
_expand(factory)
|
||||
monkeypatch.setattr(eventmanager, "_EventManager__lifecycle_state", "stopping")
|
||||
dispatcher = composition.build_outbox_dispatcher()
|
||||
assert dispatcher.dispatch_one() and dispatcher.dispatch_one()
|
||||
assert not dispatcher.dispatch_one()
|
||||
rows = _rows(factory)
|
||||
assert [row.status for row in rows] == ["completed", "pending", "pending"]
|
||||
assert all("stopping" in row.last_error for row in rows[1:])
|
||||
assert not rows[1].payload.get(EVENT_FANOUT_COMPLETE_FIELD)
|
||||
assert store.claim(datetime.now(timezone.utc), datetime.now(timezone.utc)) is None
|
||||
Reference in New Issue
Block a user