Coverage for src/keel/captureverify.py: 100%
73 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-02 20:26 +0000
1"""Pure capture reconciliation: cross-check merged PRs against the ledger.
3``keel capture-verify`` historically trusted the agent to pass every merged PR
4via ``--merged-pr`` and to self-report ``--capture-status applied`` with no
5proof. This module hardens that accounting with three additive checks, all
6pure data in / pure findings out (no network, subprocess, clock, or random):
81. **missing-marker** — every PR in the derived merged set must have a valid
9 capture marker in the ledger. A merged PR with no marker is a finding, so a
10 merged PR can no longer silently vanish from capture accounting by being
11 omitted from the args.
122. **applied-without-artifact** — an ``applied`` capture must carry a durable
13 capture artifact reference (path/hash) in its ledger record. ``applied``
14 with no artifact is a finding. ``deferred``/``skipped`` need no artifact.
153. **reviewer-count-mismatch** — the ledger record's ``actors.reviewers`` count
16 for a PR is cross-checked against the evidence-side review-verdict count for
17 that PR. Recording more reviewers than verdicts posted is a finding.
19The CLI does the I/O (transport query for merged PRs, marker/verdict fetch) and
20feeds the results here. The base pass/fail semantics of ``verify_session`` are
21preserved; these are strictly additional findings.
22"""
24from __future__ import annotations
26from typing import Any
28from . import capture, ledger
30RECONCILE_SCHEMA_VERSION = "keel.capture-verify-reconcile.v1"
32FINDING_MISSING_MARKER = "missing-marker"
33FINDING_INVALID_MARKER = "invalid-marker"
34FINDING_APPLIED_WITHOUT_ARTIFACT = "applied-without-artifact"
35FINDING_REVIEWER_COUNT_MISMATCH = "reviewer-count-mismatch"
36#: A **note**, never a finding: the capture was applied to a sink outside the checkout,
37#: so the path in the ledger is host-specific and names nothing on any other host.
38#: That is the sink's design, not a gap — reporting it as `applied-without-artifact`
39#: accused a run that did exactly what it was configured to do (#1185).
40NOTE_APPLIED_ELSEWHERE = "applied-elsewhere"
43def reconcile(
44 records: list[dict[str, Any]],
45 merged_prs: list[int] | tuple[int, ...],
46 *,
47 verdict_counts: dict[int, int] | None = None,
48) -> dict[str, Any]:
49 """Cross-check the derived merged-PR set against the ledger.
51 ``records`` is the run ledger. ``merged_prs`` is the authoritative merged-PR
52 set (derived from the transport, not the agent's args). ``verdict_counts``
53 maps a PR number to the evidence-side review-verdict count; a PR omitted from
54 the mapping skips the reviewer cross-check (the count is unknown offline, so
55 it degrades to advisory rather than failing).
57 Returns a structured report: per-PR results plus a flat findings list and a
58 summary. ``ok`` is true only when no findings were raised.
59 """
60 counts = verdict_counts or {}
61 results = [_reconcile_pr(records, pr, counts) for pr in merged_prs]
62 findings = [finding for result in results for finding in result["findings"]]
63 notes = [note for result in results for note in result.get("notes", ())]
64 by_type = {
65 FINDING_MISSING_MARKER: 0,
66 FINDING_INVALID_MARKER: 0,
67 FINDING_APPLIED_WITHOUT_ARTIFACT: 0,
68 FINDING_REVIEWER_COUNT_MISMATCH: 0,
69 }
70 for finding in findings:
71 by_type[finding["type"]] += 1
72 return {
73 "schema_version": RECONCILE_SCHEMA_VERSION,
74 "ok": not findings,
75 "merged_prs": list(merged_prs),
76 "results": results,
77 "findings": findings,
78 "notes": notes,
79 "summary": {
80 "checked": len(results),
81 "findings": len(findings),
82 "notes": len(notes),
83 **by_type,
84 },
85 }
88def _reconcile_pr(
89 records: list[dict[str, Any]],
90 pr_number: int,
91 verdict_counts: dict[int, int],
92) -> dict[str, Any]:
93 verification = capture._verify_pr(records, pr_number)
94 record = ledger.latest_ship_run_for_pr(records, pr_number)
95 findings: list[dict[str, Any]] = []
96 notes: list[dict[str, Any]] = []
98 if not verification["ok"]:
99 if verification["status"] == "missing":
100 findings.append(
101 _finding(
102 FINDING_MISSING_MARKER,
103 pr_number,
104 "merged PR has no capture marker in the ledger",
105 )
106 )
107 else:
108 reason = verification.get("reason") or "invalid capture marker in the ledger"
109 findings.append(
110 _finding(
111 FINDING_INVALID_MARKER,
112 pr_number,
113 f"merged PR has an invalid capture marker: {reason}",
114 invalid_reason=reason,
115 )
116 )
118 artifact = _capture_artifact(record)
119 scope = _capture_artifact_scope(record, artifact)
120 # Both branches are about an **applied** row, and the note has to say so as
121 # plainly as the finding does: it asserts that a host wrote a file. Attached on
122 # the scope alone it appeared beside `invalid-marker`, and on a `deferred` row —
123 # claiming a write for a run that captured nothing.
124 applied = verification.get("status") == "applied"
125 if applied and not artifact and scope != capture.ARTIFACT_SCOPE_MACHINE:
126 findings.append(
127 _finding(
128 FINDING_APPLIED_WITHOUT_ARTIFACT,
129 pr_number,
130 "capture status is applied but no capture artifact was recorded",
131 )
132 )
133 elif applied and scope == capture.ARTIFACT_SCOPE_MACHINE:
134 # Decided from the record, not from the filesystem: this module is pure, and
135 # "can this host read it" is the wrong question anyway — the path is host-specific
136 # wherever it is read, including on the machine that wrote it a month later.
137 notes.append(
138 _note(
139 NOTE_APPLIED_ELSEWHERE,
140 pr_number,
141 "capture artifact is outside the checkout, so it is readable only on the "
142 + (f"host that wrote it: {artifact}" if artifact else "host that wrote it"),
143 )
144 )
146 recorded_reviewers = _recorded_reviewer_count(record)
147 verdicts = verdict_counts.get(pr_number)
148 if verdicts is not None and recorded_reviewers > verdicts:
149 findings.append(
150 _finding(
151 FINDING_REVIEWER_COUNT_MISMATCH,
152 pr_number,
153 f"ledger records {recorded_reviewers} reviewer(s) but only "
154 f"{verdicts} review verdict(s) were posted",
155 recorded_reviewers=recorded_reviewers,
156 posted_verdicts=verdicts,
157 )
158 )
160 return {
161 "pr": pr_number,
162 "ok": not findings,
163 "marker_status": verification["status"],
164 "marker": verification.get("marker"),
165 "artifact": artifact,
166 "recorded_reviewers": recorded_reviewers,
167 "posted_verdicts": verdicts,
168 "findings": findings,
169 "notes": notes,
170 }
173def _finding(finding_type: str, pr_number: int, reason: str, **extra: Any) -> dict[str, Any]:
174 finding = {"type": finding_type, "pr": pr_number, "reason": reason}
175 finding.update(extra)
176 return finding
179def _note(note_type: str, pr_number: int, message: str) -> dict[str, Any]:
180 """A note has a finding's shape and none of its authority: it never fails a run."""
181 return {"type": note_type, "pr": pr_number, "message": message}
184def _capture_artifact_scope(record: dict[str, Any] | None, artifact: str | None) -> str | None:
185 """The recorded scope, or the one the path's shape implies for an older record.
187 Records written before `artifact_scope` existed carry only the path, and an anchored
188 path was always an outside sink — so nothing already in a ledger changes meaning.
189 """
190 if not isinstance(record, dict):
191 return None
192 block = record.get("capture")
193 if isinstance(block, dict):
194 recorded = block.get("artifact_scope")
195 if isinstance(recorded, str) and recorded.strip():
196 return recorded.strip()
197 return capture.artifact_scope(artifact)
200def _capture_artifact(record: dict[str, Any] | None) -> str | None:
201 if not isinstance(record, dict):
202 return None
203 block = record.get("capture")
204 if not isinstance(block, dict):
205 return None
206 artifact = block.get("artifact")
207 return artifact if isinstance(artifact, str) and artifact.strip() else None
210def _recorded_reviewer_count(record: dict[str, Any] | None) -> int:
211 if not isinstance(record, dict):
212 return 0
213 actors = record.get("actors")
214 if not isinstance(actors, dict):
215 return 0
216 reviewers = actors.get("reviewers")
217 if not isinstance(reviewers, list):
218 return 0
219 return sum(1 for reviewer in reviewers if isinstance(reviewer, str) and reviewer.strip())