Separate tutorial and Skill review gates

This commit is contained in:
LinuxDev9002
2026-07-18 20:23:00 -07:00
parent b523765944
commit a77417dbc7
6 changed files with 136 additions and 7 deletions

View File

@@ -426,7 +426,5 @@ git -C "$repo" apply --check "$patch_file"
git -C "$repo" apply "$patch_file"
applied=true
git -C "$repo" worktree remove --force "$author_repo"
worktree_preserved=false
run_state=complete
echo "manifest=$manifest"

View File

@@ -5,7 +5,8 @@ readonly SKILL_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
readonly PROCESS_SKILL="$SKILL_DIR/SKILL.md"
readonly ACCEPTANCE="$SKILL_DIR/references/tutorial-acceptance.md"
readonly WRITING_GUIDE="$SKILL_DIR/references/drafting-process.md"
readonly REVIEW_GUIDE="$SKILL_DIR/references/review-checklist.md"
readonly TUTORIAL_REVIEW_GUIDE="$SKILL_DIR/references/review-checklist.md"
readonly SKILL_REVIEW_GUIDE="$SKILL_DIR/references/skill-review-checklist.md"
readonly PRECEDENT_GUIDE="$SKILL_DIR/references/repository-precedents.md"
readonly VERIFY="$SKILL_DIR/scripts/verify_external_review.py"
readonly STYLE_DIR="$SKILL_DIR/../bpf-tutorial-writing-style"
@@ -75,13 +76,20 @@ declare -a policy_files=(
"$PROCESS_SKILL"
"$ACCEPTANCE"
"$WRITING_GUIDE"
"$REVIEW_GUIDE"
"$PRECEDENT_GUIDE"
"$STYLE_SKILL"
"$STYLE_GUIDELINES"
"$STYLE_REPOSITORY"
"$STYLE_PROSE"
)
case "$scope" in
tutorial)
policy_files+=("$TUTORIAL_REVIEW_GUIDE")
;;
skill)
policy_files+=("$SKILL_REVIEW_GUIDE")
;;
esac
for required_file in "${policy_files[@]}" "$VERIFY"; do
[[ -r $required_file ]] || {
echo "repository-local Skill dependency is missing: $required_file" >&2
@@ -365,7 +373,7 @@ finalize() {
run_state=failed_or_interrupted
fi
write_manifest
if ((exit_status != 0)); then
if ((exit_status != 0)) && [[ $run_state != complete ]]; then
echo "review did not complete; prompt, trace, and manifest were preserved under $run_dir" >&2
fi
}
@@ -444,3 +452,7 @@ fi
run_state=complete
echo "gate=$gate"
echo "manifest=$manifest"
if [[ $gate == FAIL ]]; then
echo "review completed with GATE: FAIL; prompt, trace, snapshot, and manifest were preserved under $run_dir" >&2
exit 3
fi

View File

@@ -0,0 +1,92 @@
#!/usr/bin/env python3
"""Integration tests for external-review gate exit behavior."""
from __future__ import annotations
import json
import os
import pathlib
import subprocess
import tempfile
import unittest
SCRIPT = pathlib.Path(__file__).with_name("run-external-review.sh")
REPO = SCRIPT.parents[4]
class RunExternalReviewTest(unittest.TestCase):
def setUp(self) -> None:
self.temporary = tempfile.TemporaryDirectory()
self.root = pathlib.Path(self.temporary.name)
binary = self.root / "bin"
binary.mkdir()
grok = binary / "grok"
grok.write_text(
"#!/usr/bin/env python3\n"
"import json, os\n"
"print(json.dumps({'type': 'text', 'data': 'Synthetic review.\\nGATE: ' + os.environ['FAKE_GATE']}))\n"
"print(json.dumps({'type': 'end', 'stopReason': 'EndTurn', 'modelUsage': {'grok-4.5-build': {'outputTokens': 3}}}))\n",
encoding="utf-8",
)
grok.chmod(0o755)
self.task = self.root / "task.md"
self.task.write_text("Review the repository-local Skills.\n", encoding="utf-8")
self.environment = os.environ.copy()
self.environment["PATH"] = f"{binary}:{self.environment['PATH']}"
self.environment["XDG_STATE_HOME"] = str(self.root / "state")
def tearDown(self) -> None:
self.temporary.cleanup()
def run_wrapper(self, gate: str) -> subprocess.CompletedProcess[str]:
environment = self.environment.copy()
environment["FAKE_GATE"] = gate
return subprocess.run(
[
str(SCRIPT),
"--reviewer",
"grok",
"--scope",
"skill",
"--repo",
str(REPO),
"--task",
str(self.task),
"--file",
".agents/skills/write-bpf-production-tutorial/SKILL.md",
"--file",
".agents/skills/bpf-tutorial-writing-style/SKILL.md",
],
cwd=REPO,
env=environment,
text=True,
capture_output=True,
check=False,
)
def manifest_from(self, result: subprocess.CompletedProcess[str]) -> dict[str, object]:
manifest_line = next(
line for line in result.stdout.splitlines() if line.startswith("manifest=")
)
path = pathlib.Path(manifest_line.removeprefix("manifest="))
return json.loads(path.read_text(encoding="utf-8"))
def test_pass_gate_returns_success(self) -> None:
result = self.run_wrapper("PASS")
self.assertEqual(0, result.returncode, result.stderr)
manifest = self.manifest_from(result)
self.assertEqual("complete", manifest["state"])
self.assertEqual("PASS", manifest["gate"])
def test_fail_gate_returns_nonzero_but_records_completed_review(self) -> None:
result = self.run_wrapper("FAIL")
self.assertEqual(3, result.returncode, result.stderr)
self.assertIn("review completed with GATE: FAIL", result.stderr)
manifest = self.manifest_from(result)
self.assertEqual("complete", manifest["state"])
self.assertEqual("FAIL", manifest["gate"])
if __name__ == "__main__":
unittest.main()