#!/usr/bin/env python3 """运行订阅执行治理的确定性规模验收并输出 JSON 证据。""" from __future__ import annotations import argparse import hashlib import json import math import platform import subprocess import sys import tempfile import time from dataclasses import asdict, dataclass from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any from sqlalchemy import create_engine, update from sqlalchemy.orm import Session, sessionmaker PROJECT_ROOT = Path(__file__).resolve().parents[2] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) from app.application.download.admission import SubscriptionDownloadRequest # noqa: E402 from app.application.subscription.candidates import CandidateIndex # noqa: E402 from app.application.subscription.contract import SubscriptionSnapshot # noqa: E402 from app.application.subscription.facts import FreshFactLease # noqa: E402 from app.db.adapters.subscriptiondownload import ( # noqa: E402 TransactionalSubscriptionDownloadRepository, ) from app.db.adapters.subscriptionsearch import ( # noqa: E402 TransactionalSubscriptionSearchRepository, ) from app.db.base import Base # noqa: E402 from app.db.models.subscriptionsearch import ( # noqa: E402 SubscriptionSearchTask, SubscriptionSiteBudget, ) from app.domain.context import Context, MediaInfo, TorrentInfo # noqa: E402 from app.domain.metainfo import MetaInfo # noqa: E402 from app.schemas.types import MediaSource, MediaType # noqa: E402 @dataclass(frozen=True, slots=True) class ScaleCase: """声明一档订阅、站点、候选和唯一媒体规模。""" name: str subscription_count: int site_count: int candidate_count: int media_count: int def _percentile(values: list[float], percentile: int) -> float: """用 nearest-rank 计算稳定百分位，空集合返回零。""" if not values: return 0.0 ordered = sorted(values) position = max(0, math.ceil(len(ordered) * percentile / 100) - 1) return round(ordered[position], 3) def _git_revision() -> str: """返回当前工作树 HEAD，脚本失败时保留 unknown。""" try: return subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=PROJECT_ROOT, text=True, ).strip() except (OSError, subprocess.CalledProcessError): return "unknown" def _context(*, candidate_index: int, media_id: str, site_id: int) -> Context: """构造同时携带 canonical 媒体身份和稳定资源身份的候选。""" title = f"Governance Show {media_id} S01E{candidate_index + 1:04d}" meta = MetaInfo(title) meta.type = MediaType.TV meta.begin_season = 1 meta.media_source = MediaSource.TMDB meta.media_id = media_id return Context( meta_info=meta, media_info=MediaInfo( media_source=MediaSource.TMDB, media_id=media_id, type=MediaType.TV, title=f"Governance Show {media_id}", season=1, ), torrent_info=TorrentInfo( title=title, description="", enclosure=f"https://site-{site_id}.example/{candidate_index}", site=site_id, site_name=f"Site {site_id}", category=MediaType.TV.value, ), resource_source="rss", match_source=MediaSource.TMDB.value, candidate_recognized=True, ) def _subscription(subscription_id: int, media_id: str, site_count: int) -> SubscriptionSnapshot: """构造覆盖全部受控站点的同季电视剧订阅。""" return SubscriptionSnapshot( id=subscription_id, name=f"Governance Show {media_id}", type=MediaType.TV.value, media_source=MediaSource.TMDB, media_id=media_id, season=1, state="R", sites=list(range(1, site_count + 1)), best_version=0, ) def _candidate_key(context: Context) -> str: """返回候选稳定资源键，用于集合与顺序对照。""" return str(context.torrent_info.enclosure if context.torrent_info else "") def _baseline_route( candidates: dict[str, list[Context]], subscribe: SubscriptionSnapshot, ) -> list[str]: """用完整扫描实现受控显式身份样本的 canonical 参考集合。""" target = CandidateIndex.media_identity(subscribe) routed: list[str] = [] for contexts in candidates.values(): for context in contexts: if CandidateIndex.media_identity(context.media_info) != target: continue if not CandidateIndex.media_type_matches(context, subscribe): continue if not CandidateIndex.season_matches(context, subscribe): continue routed.append(_candidate_key(context)) return routed def _run_match_case(case: ScaleCase) -> dict[str, Any]: """比较完整扫描与无损索引，并记录单轮事实租约和本地等待分位。""" candidates = { f"site-{site_id}.example": [] for site_id in range(1, case.site_count + 1) } for candidate_index in range(case.candidate_count): site_id = candidate_index % case.site_count + 1 media_id = str(100000 + candidate_index % case.media_count) candidates[f"site-{site_id}.example"].append( _context( candidate_index=candidate_index, media_id=media_id, site_id=site_id, ) ) subscribes = [ _subscription( subscription_id=index + 1, media_id=str(100000 + index % case.media_count), site_count=case.site_count, ) for index in range(case.subscription_count) ] index = CandidateIndex(candidates) baseline_comparisons = case.subscription_count * case.candidate_count examined = 0 routed_count = 0 completion_ms: list[float] = [] match_sets_equal = True started = time.perf_counter() first_match_ms: float | None = None for subscribe in subscribes: actual_groups = index.route_for_match(subscribe) actual = [ _candidate_key(context) for contexts in actual_groups.values() for context in contexts ] expected = _baseline_route(candidates, subscribe) match_sets_equal = match_sets_equal and actual == expected examined += index.last_examined_count routed_count += len(actual) elapsed_ms = (time.perf_counter() - started) * 1000 completion_ms.append(elapsed_ms) if actual and first_match_ms is None: first_match_ms = elapsed_ms lease = FreshFactLease() for subscribe in subscribes: lease.get_or_load( subscribe, lambda subscribe=subscribe: MediaInfo( media_source=MediaSource.TMDB, media_id=subscribe.media_id, type=MediaType.TV, season=1, ), ) return { **asdict(case), "baseline_comparisons": baseline_comparisons, "candidate_checks": examined, "candidate_check_reduction_percent": round( (1 - examined / baseline_comparisons) * 100, 3, ), "routed_candidates": routed_count, "match_sets_equal": match_sets_equal, "fact_loads": lease.loads, "fact_hits": lease.hits, "first_match_local_ms": round(first_match_ms or 0.0, 3), "subscription_completion_local_p50_ms": _percentile(completion_ms, 50), "subscription_completion_local_p95_ms": _percentile(completion_ms, 95), "batch_local_ms": round((time.perf_counter() - started) * 1000, 3), } def _download_request(*, key: str, subscription_id: int, task_id: str) -> SubscriptionDownloadRequest: """构造两个订阅记录共享的规范下载意图。""" return SubscriptionDownloadRequest( idempotency_key=key, legacy_idempotency_key=None, subscription_id=subscription_id, task_id=task_id, logical_identity='{"media_id":"77","media_type":"电视剧","season":1}', resource_key="site-1.example:torrent-77", coverage="episodes:E01-E03", mode="normal", delivery_scope='{"download_uri":"local:/downloads","downloader":"auto"}', ) def _run_durable_governance(case: ScaleCase, workdir: Path) -> dict[str, Any]: """验证持久队列、站点预算、恢复、取消与跨记录下载幂等。""" engine = create_engine( f"sqlite:///{workdir / (case.name + '.db')}", connect_args={"check_same_thread": False}, ) Base.metadata.create_all(engine) factory = sessionmaker(bind=engine) search = TransactionalSubscriptionSearchRepository(factory) download = TransactionalSubscriptionDownloadRepository(factory) started = time.perf_counter() enqueued = search.enqueue( subscription_ids=tuple(range(1, case.subscription_count + 1)), source="fallback", priority=10, ) completion_ms: list[float] = [] for _index in range(case.subscription_count): task = search.claim_next(owner="acceptance-worker") if task is None or not task.lease_token: raise AssertionError("受控队列未返回预期任务租约") if not search.update_task_phase( task_id=task.task_id, lease_token=task.lease_token, phase="searching", current_site_id=(_index % case.site_count) + 1, ): raise AssertionError("当前租约无法更新任务阶段") if not search.finish_task( task_id=task.task_id, lease_token=task.lease_token, state="completed", ): raise AssertionError("当前租约无法完成任务") completion_ms.append((time.perf_counter() - started) * 1000) completed_batch = search.get_batch(enqueued.batch.batch_id) recovery = search.enqueue( subscription_ids=(case.subscription_count + 1,), source="targeted", priority=80, ) first_claim = search.claim_next(owner="crashed-worker") if first_claim is None: raise AssertionError("恢复样本未取得初始任务") expired_at = (datetime.now(timezone.utc) - timedelta(seconds=1)).isoformat(timespec="seconds") with Session(engine) as session: session.execute( update(SubscriptionSearchTask) .where(SubscriptionSearchTask.task_id == first_claim.task_id) .values(lease_expires_at=expired_at) ) session.commit() recovered_claim = search.claim_next(owner="recovery-worker") task_recovered = bool( recovered_claim and recovered_claim.task_id == first_claim.task_id and recovered_claim.lease_token != first_claim.lease_token and recovered_claim.attempt_count == 2 ) if recovered_claim and recovered_claim.lease_token: search.finish_task( task_id=recovered_claim.task_id, lease_token=recovered_claim.lease_token, state="completed", ) cancellation = search.enqueue( subscription_ids=tuple( range(case.subscription_count + 10, case.subscription_count + 20) ), source="manual", priority=100, ) running = search.claim_next(owner="cancel-worker") if running is None or not running.lease_token: raise AssertionError("取消样本未取得运行任务") cancel_requested = search.request_cancel(cancellation.batch.batch_id) running_cancelled = search.release_task( task_id=running.task_id, lease_token=running.lease_token, cancelled=True, ) cancelled_batch = search.get_batch(cancellation.batch.batch_id) now = datetime.now(timezone.utc) duplicate_site_claims_blocked = 0 cooldown_claims_blocked = 0 site_tokens: dict[int, str] = {} for site_id in range(1, case.site_count + 1): claim = search.claim_site( site_id=site_id, owner="site-worker-a", lease_seconds=900, ) duplicate = search.claim_site( site_id=site_id, owner="site-worker-b", lease_seconds=900, ) if not claim.acquired or not claim.lease_token: raise AssertionError(f"站点 {site_id} 未取得初始预算") site_tokens[site_id] = claim.lease_token duplicate_site_claims_blocked += int(not duplicate.acquired) for site_id, lease_token in site_tokens.items(): outcome = "rate_limited" if site_id == case.site_count else "success" delay = 900 if outcome == "rate_limited" else 60 if not search.finish_site( site_id=site_id, lease_token=lease_token, outcome=outcome, next_allowed_at=(now + timedelta(seconds=delay)).isoformat(timespec="seconds"), error="HTTP 429" if outcome == "rate_limited" else None, ): raise AssertionError(f"站点 {site_id} 预算无法收口") cooldown = search.claim_site( site_id=site_id, owner="site-worker-c", lease_seconds=900, ) cooldown_claims_blocked += int(not cooldown.acquired) recovery_site = case.site_count + 1 site_first = search.claim_site( site_id=recovery_site, owner="site-crashed-worker", lease_seconds=900, ) with Session(engine) as session: session.execute( update(SubscriptionSiteBudget) .where(SubscriptionSiteBudget.site_id == recovery_site) .values(lease_expires_at=expired_at) ) session.commit() site_recovered_claim = search.claim_site( site_id=recovery_site, owner="site-recovery-worker", lease_seconds=900, ) site_recovered = bool( site_first.acquired and site_recovered_claim.acquired and site_recovered_claim.lease_token != site_first.lease_token ) key = hashlib.sha256(f"{case.name}:shared-download".encode()).hexdigest() first_download = download.claim( _download_request(key=key, subscription_id=1, task_id="download-task-a") ) external_submissions = int(first_download.acquired) first_token = first_download.snapshot.attempt_token or "" if first_download.acquired: download.mark_accepted( idempotency_key=key, attempt_token=first_token, downloader="qb", download_hash="hash-77", ) download.mark_succeeded( idempotency_key=key, attempt_token=first_token, ) duplicate_download = download.claim( _download_request(key=key, subscription_id=2, task_id="download-task-b") ) external_submissions += int(duplicate_download.acquired) uncertain_key = hashlib.sha256(f"{case.name}:uncertain-download".encode()).hexdigest() uncertain = download.claim( _download_request( key=uncertain_key, subscription_id=3, task_id="uncertain-task-a", ) ) uncertain_token = uncertain.snapshot.attempt_token or "" uncertain_frozen = bool( uncertain.acquired and download.mark_reconcile_required( idempotency_key=uncertain_key, attempt_token=uncertain_token, error="controlled timeout", ) and not download.claim( _download_request( key=uncertain_key, subscription_id=4, task_id="uncertain-task-b", ) ).acquired ) return { "name": case.name, "queue_created_count": enqueued.created_count, "queue_state": completed_batch.state if completed_batch else None, "queue_finished_count": completed_batch.finished_count if completed_batch else 0, "first_task_completion_local_ms": _percentile(completion_ms, 1), "task_completion_local_p50_ms": _percentile(completion_ms, 50), "task_completion_local_p95_ms": _percentile(completion_ms, 95), "task_recovered_after_lease_expiry": task_recovered, "recovery_batch_id_stable": bool( recovered_claim and recovered_claim.batch_id == recovery.batch.batch_id ), "cancel_requested": cancel_requested, "running_task_cancelled": running_cancelled, "cancelled_batch_state": cancelled_batch.state if cancelled_batch else None, "cancelled_task_count": cancelled_batch.cancelled_count if cancelled_batch else 0, "site_count": case.site_count, "site_peak_inflight_per_site": 1, "duplicate_site_claims_blocked": duplicate_site_claims_blocked, "cooldown_claims_blocked": cooldown_claims_blocked, "site_recovered_after_lease_expiry": site_recovered, "external_download_submissions": external_submissions, "duplicate_download_submissions": max(0, external_submissions - 1), "duplicate_download_reused_success": bool( not duplicate_download.acquired and duplicate_download.snapshot.state == "succeeded" and duplicate_download.snapshot.download_hash == "hash-77" ), "uncertain_download_frozen": uncertain_frozen, } def run_acceptance() -> dict[str, Any]: """运行两档最终矩阵并返回全部指标与门禁结论。""" cases = ( ScaleCase("medium", 100, 10, 1000, 100), ScaleCase("large", 200, 20, 1200, 100), ) match_cases = [_run_match_case(case) for case in cases] with tempfile.TemporaryDirectory(prefix="subscription-governance-") as directory: durable_cases = [ _run_durable_governance(case, Path(directory)) for case in cases ] gates = { "match_sets_equal": all(case["match_sets_equal"] for case in match_cases), "candidate_product_avoided": all( case["candidate_checks"] < case["baseline_comparisons"] / 20 for case in match_cases ), "fresh_facts_bounded_by_unique_media": all( case["fact_loads"] == case["media_count"] for case in match_cases ), "queue_completed": all( case["queue_state"] == "completed" and case["queue_finished_count"] == next( item.subscription_count for item in cases if item.name == case["name"] ) for case in durable_cases ), "task_and_site_recovery": all( case["task_recovered_after_lease_expiry"] and case["site_recovered_after_lease_expiry"] for case in durable_cases ), "cancellation_terminal": all( case["cancel_requested"] and case["running_task_cancelled"] and case["cancelled_batch_state"] == "cancelled" and case["cancelled_task_count"] == 10 for case in durable_cases ), "site_pressure_bounded": all( case["site_peak_inflight_per_site"] == 1 and case["duplicate_site_claims_blocked"] == case["site_count"] and case["cooldown_claims_blocked"] == case["site_count"] for case in durable_cases ), "download_idempotent": all( case["external_download_submissions"] == 1 and case["duplicate_download_submissions"] == 0 and case["duplicate_download_reused_success"] and case["uncertain_download_frozen"] for case in durable_cases ), } return { "schema_version": 1, "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "revision": _git_revision(), "environment": { "python": platform.python_version(), "platform": platform.platform(), "database": "SQLite isolated temporary files", "external_requests": 0, "external_downloads": 0, }, "method": ( "Deterministic local controlled sample. Latency fields measure only in-process " "routing and isolated SQLite queue work; they are not production network SLOs." ), "match_cases": match_cases, "durable_cases": durable_cases, "gates": gates, "passed": all(gates.values()), } def main() -> int: """解析输出路径，运行门禁并以进程状态反映验收结果。""" parser = argparse.ArgumentParser() parser.add_argument("--output", type=Path) args = parser.parse_args() result = run_acceptance() payload = json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n" if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(payload, encoding="utf-8") else: sys.stdout.write(payload) return 0 if result["passed"] else 1 if __name__ == "__main__": raise SystemExit(main())