Coverage for src/keel/checkpoint.py: 100%
257 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
1"""Resumable checkpoint helpers for keel work runs."""
3from __future__ import annotations
5import json
6import shlex
7from pathlib import Path
8from typing import Any
10from . import config as cfg
11from . import model, workspace
13CHECKPOINT_SCHEMA_VERSION = "keel.checkpoint.v1"
14DEFAULT_CHECKPOINT_PATH = ".keel/state/checkpoint.json"
15RECORD_TYPE_RUN_CHECKPOINT = "run_checkpoint"
17COMMANDS = ("ship", "work-block", "overnight")
18STEP_IDS = tuple(step.id for step in model.BACKBONE)
19_STEP_IDS_SET = frozenset(STEP_IDS)
20MERGE_STATES = ("not-started", "pending", "merged", "failed", "skipped")
21CAPTURE_STATES = ("not-started", "applied", "deferred", "skipped", "failed")
22CLOSE_STATES = ("not-started", "closed", "failed")
23LIVE_PR_STATES = ("unknown", "missing", "open", "merged", "closed")
24LIVE_WORKTREE_STATES = ("unknown", "present", "missing")
26_IDEMPOTENT_STEPS = {
27 "s0": "load-config",
28 "s1": "select-or-reconcile-issue",
29 "s2": "ensure-branch-and-worktree",
30 "s3": "rerun-guards",
31 "s4": "continue-implementation",
32 "s5": "reclassify-diff",
33 "s6": "recheck-ci",
34 "s7": "rerun-review",
35 "s8": "rerun-test",
36 "s9": "continue-fixloop",
37 "s10": "revalidate-merge-state",
38 "s11": "run-or-verify-capture",
39 "s12": "run-or-verify-close",
40}
43class CheckpointError(ValueError):
44 """Raised when a checkpoint cannot be decoded as the stable schema."""
47def checkpoint_contract_as_dict(config: cfg.ProjectConfig) -> dict[str, Any]:
48 """Return the project-neutral checkpoint storage and resume contract."""
49 path, source = configured_checkpoint_path(config)
50 return {
51 "schema_version": CHECKPOINT_SCHEMA_VERSION,
52 "format": "json",
53 "path": path,
54 "path_source": source,
55 "missing_handling": "no-checkpoint",
56 "write_owner": ["ship", "work-block", "overnight"],
57 "resume_command": "resume",
58 "checkpoint_command": "checkpoint",
59 "steps": [
60 {
61 "step_id": step.id,
62 "step_name": step.name,
63 "safe_boundary": step.id in _IDEMPOTENT_STEPS,
64 "resume_action": _IDEMPOTENT_STEPS.get(step.id),
65 }
66 for step in model.BACKBONE
67 ],
68 "consumer_neutral": True,
69 }
72def configured_checkpoint_path(config: cfg.ProjectConfig) -> tuple[str, str]:
73 """Return the configured checkpoint path and source."""
74 pack = config.policy_pack or {}
75 reports = pack.get("reports") if isinstance(pack.get("reports"), dict) else {}
76 value = reports.get("checkpoint")
77 if isinstance(value, str) and value.strip():
78 return value, "policy_pack.reports.checkpoint"
79 return DEFAULT_CHECKPOINT_PATH, "default"
82def resolve_path(root: str | Path, config: cfg.ProjectConfig) -> Path:
83 """Resolve the checkpoint path under ``root`` and reject escapes."""
84 raw, _ = configured_checkpoint_path(config)
85 path = Path(raw)
86 if workspace.is_root_anchored(raw):
87 raise CheckpointError("checkpoint path must be relative to the project root")
88 root_path = Path(root).resolve()
89 resolved = (root_path / path).resolve()
90 try:
91 resolved.relative_to(root_path)
92 except ValueError as exc:
93 raise CheckpointError("checkpoint path escapes the project root") from exc
94 return resolved
97def build_checkpoint_record(
98 *,
99 run_id: str,
100 command: str,
101 current_step: str,
102 base_branch: str,
103 target: str | None = None,
104 issue_queue: list[int] | None = None,
105 active_issue: int | None = None,
106 branch: str | None = None,
107 worktree: str | None = None,
108 pull_request: int | None = None,
109 head_sha: str | None = None,
110 completed_steps: list[str] | None = None,
111 last_gate: str | None = None,
112 last_review: str | None = None,
113 last_check: str | None = None,
114 jury_mode: str | None = None,
115 merge_state: str = "not-started",
116 capture_state: str = "not-started",
117 close_state: str = "not-started",
118 stop_reason: str | None = None,
119) -> dict[str, Any]:
120 """Build one deterministic checkpoint record."""
121 record = {
122 "schema_version": CHECKPOINT_SCHEMA_VERSION,
123 "record_type": RECORD_TYPE_RUN_CHECKPOINT,
124 "run_id": run_id,
125 "command": command,
126 "target": target,
127 "queue": {
128 "issues": list(issue_queue or ()),
129 "active_issue": active_issue,
130 },
131 "position": {
132 "current_step": current_step,
133 "completed_steps": list(completed_steps or ()),
134 },
135 "identifiers": {
136 "base_branch": base_branch,
137 "branch": branch,
138 "worktree": worktree,
139 "pull_request": pull_request,
140 "head_sha": head_sha,
141 },
142 "state": {
143 "last_gate": last_gate,
144 "last_review": last_review,
145 "last_check": last_check,
146 "jury_mode": jury_mode,
147 "merge": merge_state,
148 "capture": capture_state,
149 "close": close_state,
150 "stop_reason": stop_reason,
151 },
152 "resume": {
153 "safe_boundary": current_step in _IDEMPOTENT_STEPS,
154 "action": _IDEMPOTENT_STEPS.get(current_step),
155 "must_reconcile_live_state": True,
156 "repeat_policy": {
157 "merge": "never-repeat-if-live-pr-merged",
158 "comments": "idempotent-anchor-or-skip",
159 "worktree": "refuse-unrelated-existing-path",
160 },
161 },
162 }
163 validate_checkpoint(record)
164 return record
167def encode_checkpoint(record: dict[str, Any]) -> str:
168 """Encode one checkpoint as stable JSON."""
169 validate_checkpoint(record)
170 return json.dumps(record, indent=2, sort_keys=True) + "\n"
173def parse_checkpoint(text: str) -> dict[str, Any]:
174 """Parse and validate one checkpoint record."""
175 try:
176 record = json.loads(text)
177 except json.JSONDecodeError as exc:
178 raise CheckpointError("invalid JSON") from exc
179 validate_checkpoint(record)
180 return record
183def read_checkpoint(path: str | Path) -> dict[str, Any] | None:
184 """Read a checkpoint; a missing checkpoint means there is no resumable run."""
185 checkpoint_path = Path(path)
186 if not checkpoint_path.exists():
187 return None
188 return parse_checkpoint(checkpoint_path.read_text(encoding="utf-8"))
191def write_checkpoint(path: str | Path, record: dict[str, Any]) -> None:
192 """Write one validated checkpoint, replacing the previous resume point atomically."""
193 workspace.write_text_atomic(path, encode_checkpoint(record))
196def resume_plan_as_dict(
197 record: dict[str, Any] | None,
198 *,
199 live_pr_state: str = "unknown",
200 live_worktree_state: str = "unknown",
201 live_head_sha: str | None = None,
202) -> dict[str, Any]:
203 """Return a deterministic dry-run resume plan after live-state reconciliation.
205 ``live_head_sha`` is the branch's actual head. The checkpoint records the head it
206 was written at, and nothing compared the two (#635), so a branch that moved since
207 resumed with stale context and no signal. A mismatch warns rather than blocks: the
208 common cause is a legitimate push that crashed before the next checkpoint, and s10
209 still fails closed on its own (``ledger.gates_pass_for_head`` pins the merge to the
210 live head).
211 """
212 if live_pr_state not in LIVE_PR_STATES:
213 raise CheckpointError("unsupported live_pr_state")
214 if live_worktree_state not in LIVE_WORKTREE_STATES:
215 raise CheckpointError("unsupported live_worktree_state")
216 if record is None:
217 return {
218 "status": "no-checkpoint",
219 "can_resume": False,
220 "reason": "checkpoint file is missing",
221 "next_step": None,
222 "resume_action": None,
223 "reconcile": _live_state(live_pr_state, live_worktree_state),
224 "warnings": [],
225 }
226 validate_checkpoint(record)
227 warnings: list[str] = []
228 state = record["state"]
229 identifiers = record["identifiers"]
230 current_step = record["position"]["current_step"]
231 status = "ready"
232 can_resume = True
233 reason = "resume from the recorded safe boundary"
234 next_step = current_step
235 action = record["resume"]["action"]
237 # A checkpoint claiming `merge: merged` used to win over *any* live state, so a
238 # checkpoint written optimistically before the merge landed sent every later resume
239 # straight to capture and close — closing the issue and flipping status:done on a PR
240 # that never merged. The post-hoc auditor cannot catch it either: `closeorder`
241 # attests the merge *decision*, not the merge. Live evidence that the PR is not
242 # merged is the stronger signal and makes this ambiguous.
243 recorded_head = identifiers.get("head_sha")
244 if live_head_sha and recorded_head and live_head_sha != recorded_head:
245 warnings.append(
246 f"branch head moved since the checkpoint was written "
247 f"({recorded_head[:12]} -> {live_head_sha[:12]}); the recorded context may be "
248 "stale — re-read the diff before resuming"
249 )
251 claimed_merged = state["merge"] == "merged"
252 contradicted = (
253 claimed_merged
254 and live_pr_state in {"open", "closed", "missing"}
255 and identifiers.get("pull_request")
256 )
257 if contradicted:
258 status = "ambiguous"
259 can_resume = False
260 reason = (
261 "checkpoint records the merge as complete but live state reports the "
262 f"pull request {live_pr_state}"
263 )
264 warnings.append(
265 "confirm whether the merge actually landed before resuming; resuming here "
266 "would close the issue for a merge that may never have happened"
267 )
268 elif live_pr_state == "merged" or claimed_merged:
269 if state["capture"] in {"not-started", "failed"}:
270 status = "needs-capture"
271 next_step = "s11"
272 action = _IDEMPOTENT_STEPS["s11"]
273 reason = "merge is complete; resume at capture without repeating merge"
274 elif state["close"] in {"not-started", "failed"}:
275 status = "needs-close"
276 next_step = "s12"
277 action = _IDEMPOTENT_STEPS["s12"]
278 reason = "capture is complete or skipped; resume at close"
279 else:
280 status = "complete"
281 can_resume = False
282 next_step = None
283 action = None
284 reason = "merge, capture, and close are already complete"
285 elif (
286 state["merge"] == "pending"
287 and live_pr_state == "unknown"
288 and identifiers.get("pull_request")
289 ):
290 # A crash *during* the merge is the genuinely ambiguous case, and it used to
291 # resume as a plain `pr-open / next: s10` with no warning at all (#635). Only
292 # live evidence resolves it: `merged` and `open` are handled above/below and
293 # both answer the question. `unknown` does not, so refuse to guess.
294 status = "ambiguous"
295 can_resume = False
296 reason = (
297 "checkpoint records the merge as in-flight and live state is unknown; "
298 "whether the merge landed cannot be determined from the checkpoint alone"
299 )
300 warnings.append(
301 "check whether the pull request merged before resuming; resuming here "
302 "would re-attempt a merge that may already have landed"
303 )
304 elif live_pr_state == "closed" and identifiers.get("pull_request"):
305 status = "ambiguous"
306 can_resume = False
307 reason = "checkpoint references a pull request that live state reports closed"
308 warnings.append("reopen the PR or reconcile the checkpoint before resuming")
309 elif live_worktree_state == "missing" and identifiers.get("worktree"):
310 status = "ambiguous"
311 can_resume = False
312 reason = "checkpoint references a worktree that live state reports missing"
313 warnings.append("recreate or reconcile the recorded worktree before resuming")
314 elif live_pr_state == "missing" and identifiers.get("pull_request"):
315 status = "ambiguous"
316 can_resume = False
317 reason = "checkpoint references a pull request that live state reports missing"
318 warnings.append("verify whether the PR was deleted or the checkpoint is stale")
319 elif current_step == "s6":
320 status = "waiting-on-ci"
321 reason = "resume by rechecking CI before review, test, or merge"
322 elif identifiers.get("pull_request"):
323 status = "pr-open"
324 reason = "resume against the recorded pull request after refreshing live state"
326 return {
327 "status": status,
328 "can_resume": can_resume,
329 "reason": reason,
330 "next_step": next_step,
331 "resume_action": action,
332 "checkpoint": record,
333 "reconcile": _live_state(live_pr_state, live_worktree_state),
334 "warnings": warnings,
335 }
338COVERAGE_STATES = ("covered", "missing", "stale-step")
341def _checkpoint_write_command(
342 run_id: str,
343 expected_step: str,
344 *,
345 path: str = "<project.yaml>",
346 root: str | None = None,
347 pull_request: int | None = None,
348 head_sha: str | None = None,
349 active_issue: int | None = None,
350 branch: str | None = None,
351) -> str:
352 path_token = path if path == "<project.yaml>" else shlex.quote(path)
353 parts = ["keel", "checkpoint", path_token]
354 if root and root != ".":
355 parts += ["--root", shlex.quote(root)]
356 parts += ["--write", "--run-id", shlex.quote(run_id), "--step", shlex.quote(expected_step)]
357 for flag, val in (
358 ("--active-issue", active_issue),
359 ("--branch", branch),
360 ("--pull-request", pull_request),
361 ("--head-sha", head_sha),
362 ):
363 if val is not None and val != "":
364 parts += [flag, shlex.quote(str(val))]
365 return " ".join(parts)
368def covering_checkpoint(
369 record: dict[str, Any] | None,
370 run_id: str,
371 expected_step: str,
372 *,
373 path: str = "<project.yaml>",
374 root: str | None = None,
375 pull_request: int | None = None,
376 head_sha: str | None = None,
377) -> dict[str, Any]:
378 """Decide whether a checkpoint covers ``run_id`` at ``expected_step``.
380 Pure and deterministic — no I/O. ``record`` is the current checkpoint
381 (or ``None`` when no checkpoint file exists); ``expected_step`` must be a
382 backbone step id. A run is *covered* when its checkpoint is for the same
383 ``run_id`` and the run actually progressed to ``expected_step`` (the
384 recorded ``current_step`` is at or past it, or the step is in
385 ``completed_steps``). The result distinguishes:
387 * ``covered`` — a current checkpoint for this run reached ``expected_step``;
388 * ``missing`` — no checkpoint, or a checkpoint for a different run;
389 * ``stale-step`` — a checkpoint for this run that has not reached the step.
390 """
391 if expected_step not in STEP_IDS:
392 raise CheckpointError("expected_step must be a backbone step id")
393 expected_index = STEP_IDS.index(expected_step)
394 prefix = f"no current checkpoint for run {run_id} at step {expected_step}"
395 if record is None:
396 cmd = _checkpoint_write_command(
397 run_id,
398 expected_step,
399 path=path,
400 root=root,
401 pull_request=pull_request,
402 head_sha=head_sha,
403 )
404 return {
405 "status": "missing",
406 "covered": False,
407 "run_id": run_id,
408 "expected_step": expected_step,
409 "checkpoint_run_id": None,
410 "checkpoint_step": None,
411 "reason": f"{prefix}; write it: `{cmd}`",
412 }
413 validate_checkpoint(record)
414 checkpoint_run_id = record.get("run_id")
415 position = record["position"]
416 current_step = position["current_step"]
417 completed = position.get("completed_steps", [])
418 identifiers = record.get("identifiers", {})
419 queue = record.get("queue", {})
420 cmd = _checkpoint_write_command(
421 run_id,
422 expected_step,
423 path=path,
424 root=root,
425 pull_request=pull_request if pull_request is not None else identifiers.get("pull_request"),
426 head_sha=head_sha if head_sha is not None else identifiers.get("head_sha"),
427 active_issue=queue.get("active_issue"),
428 branch=identifiers.get("branch"),
429 )
430 base = {
431 "run_id": run_id,
432 "expected_step": expected_step,
433 "checkpoint_run_id": checkpoint_run_id,
434 "checkpoint_step": current_step,
435 }
436 if checkpoint_run_id != run_id:
437 return {
438 **base,
439 "status": "missing",
440 "covered": False,
441 "reason": f"{prefix} (checkpoint is for run {checkpoint_run_id}); write it: `{cmd}`",
442 }
443 reached = expected_step in completed or STEP_IDS.index(current_step) >= expected_index
444 if not reached:
445 return {
446 **base,
447 "status": "stale-step",
448 "covered": False,
449 "reason": f"{prefix} (run is at {current_step}); advance it: `{cmd}`",
450 }
451 return {
452 **base,
453 "status": "covered",
454 "covered": True,
455 "reason": f"checkpoint for run {run_id} reached step {expected_step}",
456 }
459#: The step whose completion ``keel merge`` records (see :func:`mark_merged`).
460MERGE_STEP = "s10"
463def mark_merged(record: dict[str, Any] | None, run_id: str) -> dict[str, Any] | None:
464 """The checkpoint after ``run_id``'s merge landed, or ``None`` when it is not that run's.
466 Pure. ``keel merge`` refuses unless a checkpoint covers the run at s10, and once the merge
467 lands that checkpoint has to say so: left alone it reads ``merge: not-started`` at s10, and
468 ``keel status`` calls a merged run *waiting on the merge window* for as long as it exists
469 (#1448). The merged record says ``merge: merged``, counts s10 as completed and moves the
470 run to the next step, and clears the stop that held it; a run already past s10 keeps its
471 position.
472 """
473 if record is None or record.get("run_id") != run_id:
474 return None
475 validate_checkpoint(record)
476 position = record["position"]
477 completed = list(position.get("completed_steps", []))
478 if MERGE_STEP not in completed:
479 completed.append(MERGE_STEP)
480 merge_index = STEP_IDS.index(MERGE_STEP)
481 current = position["current_step"]
482 if STEP_IDS.index(current) <= merge_index:
483 current = STEP_IDS[merge_index + 1]
484 merged = {
485 **record,
486 "position": {**position, "current_step": current, "completed_steps": completed},
487 # The stop that held the run at s10 is over once the merge lands; left in place, it
488 # would keep `keel status` reading the run as interrupted or waiting on it.
489 "state": {**record["state"], "merge": "merged", "stop_reason": None},
490 "resume": {
491 **record["resume"],
492 "safe_boundary": current in _IDEMPOTENT_STEPS,
493 "action": _IDEMPOTENT_STEPS.get(current),
494 },
495 }
496 validate_checkpoint(merged)
497 return merged
500def find_orphans(
501 *,
502 live_branches: list[str] | None = None,
503 live_pull_requests: list[int] | None = None,
504 checkpoint_record: dict[str, Any] | None = None,
505 ledger_records: list[dict[str, Any]] | None = None,
506) -> dict[str, Any]:
507 """Return live branches/PRs with no covering checkpoint or ledger record.
509 Pure and deterministic — no I/O. An *orphan* is a branch or pull request
510 that exists on the git/transport side but is referenced by neither the
511 current checkpoint nor any ledger record, i.e. keel has no covering state
512 for it (the GAP-13 hazard from the git side). Advisory only.
513 """
514 known_branches, known_prs = _known_references(checkpoint_record, ledger_records)
515 orphan_branches = [branch for branch in (live_branches or []) if branch not in known_branches]
516 orphan_prs = [pr for pr in (live_pull_requests or []) if pr not in known_prs]
517 return {
518 "branches": orphan_branches,
519 "pull_requests": orphan_prs,
520 "orphan_count": len(orphan_branches) + len(orphan_prs),
521 "known_branches": sorted(known_branches),
522 "known_pull_requests": sorted(known_prs),
523 }
526def _known_references(
527 checkpoint_record: dict[str, Any] | None,
528 ledger_records: list[dict[str, Any]] | None,
529) -> tuple[set[str], set[int]]:
530 branches: set[str] = set()
531 prs: set[int] = set()
532 if isinstance(checkpoint_record, dict):
533 raw_identifiers = checkpoint_record.get("identifiers")
534 identifiers = raw_identifiers if isinstance(raw_identifiers, dict) else {}
535 branch = identifiers.get("branch")
536 if isinstance(branch, str) and branch:
537 branches.add(branch)
538 pull_request = identifiers.get("pull_request")
539 if isinstance(pull_request, int):
540 prs.add(pull_request)
541 for record in ledger_records or []:
542 if not isinstance(record, dict):
543 continue
544 git = record.get("git") if isinstance(record.get("git"), dict) else {}
545 branch = git.get("branch")
546 if isinstance(branch, str) and branch:
547 branches.add(branch)
548 pr_entry = record.get("pull_request")
549 if isinstance(pr_entry, dict):
550 number = pr_entry.get("number")
551 if isinstance(number, int):
552 prs.add(number)
553 return branches, prs
556def validate_checkpoint(record: Any) -> None:
557 """Validate the stable checkpoint shape."""
558 if not isinstance(record, dict):
559 raise CheckpointError("checkpoint must be an object")
560 if record.get("schema_version") != CHECKPOINT_SCHEMA_VERSION:
561 raise CheckpointError("unsupported schema_version")
562 if record.get("record_type") != RECORD_TYPE_RUN_CHECKPOINT:
563 raise CheckpointError("unsupported record_type")
564 if record.get("command") not in COMMANDS:
565 raise CheckpointError("unsupported command")
566 queue = record.get("queue")
567 if not isinstance(queue, dict) or not isinstance(queue.get("issues"), list):
568 raise CheckpointError("queue must be an object with issues")
569 position = record.get("position")
570 if not isinstance(position, dict) or position.get("current_step") not in _IDEMPOTENT_STEPS:
571 raise CheckpointError("unsupported current_step")
572 completed = position.get("completed_steps")
573 if not isinstance(completed, list):
574 raise CheckpointError("unsupported completed_steps")
575 try:
576 if not _STEP_IDS_SET.issuperset(completed):
577 raise CheckpointError("unsupported completed_steps")
578 except TypeError as err:
579 raise CheckpointError("unsupported completed_steps") from err
580 identifiers = record.get("identifiers")
581 if not isinstance(identifiers, dict) or "base_branch" not in identifiers:
582 raise CheckpointError("identifiers must include base_branch")
583 state = record.get("state")
584 if not isinstance(state, dict):
585 raise CheckpointError("state must be an object")
586 if state.get("merge") not in MERGE_STATES:
587 raise CheckpointError("unsupported merge state")
588 if state.get("capture") not in CAPTURE_STATES:
589 raise CheckpointError("unsupported capture state")
590 if state.get("close") not in CLOSE_STATES:
591 raise CheckpointError("unsupported close state")
592 resume = record.get("resume")
593 if not isinstance(resume, dict):
594 raise CheckpointError("resume must be an object")
595 if resume.get("action") != _IDEMPOTENT_STEPS[position["current_step"]]:
596 raise CheckpointError("resume action does not match current_step")
597 if not isinstance(resume.get("repeat_policy"), dict):
598 raise CheckpointError("resume repeat_policy must be an object")
601def _live_state(live_pr_state: str, live_worktree_state: str) -> dict[str, str]:
602 return {
603 "pull_request": live_pr_state,
604 "worktree": live_worktree_state,
605 "source": "live-git-github-state-or-adapter-supplied-dry-run-state",
606 }