Coverage for src/keel/ledger.py: 100%

376 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 19:17 +0000

1"""Structured run ledger helpers for keel workflows.""" 

2 

3from __future__ import annotations 

4 

5import json 

6from collections.abc import Callable, Collection, Mapping 

7from pathlib import Path 

8from typing import Any 

9 

10from . import capture, consent, redaction, workspace 

11from . import config as cfg 

12 

13LEDGER_SCHEMA_VERSION = "keel.run-ledger.v1" 

14CAPTURE_HEALTH_SCHEMA_VERSION = "keel.capture-health.v1" 

15DEFAULT_LEDGER_PATH = ".keel/state/run-ledger.jsonl" 

16RECORD_TYPE_SHIP_RUN = "ship_run" 

17#: The operator's live consent as ``swarm-run --live`` delegated it to its workers (#1400): 

18#: one record when the run delegates it, and one more per cluster pull request its 

19#: workers open. See :func:`build_consent_delegation_record`. 

20RECORD_TYPE_CONSENT_DELEGATION = "consent_delegation" 

21#: The follow-up issue ``keel merge`` filed for the minor findings a ``COMMENT`` jury 

22#: consensus left when ``team.jury.comment_consensus`` passed the gate (#1570). One per 

23#: pull request and head, so a retried merge reuses the issue instead of filing another. 

24RECORD_TYPE_JURY_FOLLOWUP = "jury_followup" 

25# The record kinds this keel reads. A reader skips any other kind (#1400): the ledger is 

26# append-only and shared by every keel on a checkout, so a kind a newer keel adds must not 

27# make an older keel refuse the history it can still read. The writer stays strict — 

28# :func:`encode_record` only ever writes a kind named here. 

29#: :data:`KNOWN_RECORD_TYPES` in its published order (the ledger contract's 

30#: ``record_types``). A keel older than 1.26.0 refuses a ledger holding any kind but the 

31#: first: those readers did not yet skip a kind they did not know. 

32RECORD_TYPES: tuple[str, ...] = ( 

33 RECORD_TYPE_SHIP_RUN, 

34 RECORD_TYPE_CONSENT_DELEGATION, 

35 RECORD_TYPE_JURY_FOLLOWUP, 

36) 

37KNOWN_RECORD_TYPES: frozenset[str] = frozenset(RECORD_TYPES) 

38#: What :func:`parse_records` and :func:`read_records` return unless a reader asks for 

39#: more. Every reader keel had before #1400 reads ship runs and nothing else; it gets ship 

40#: runs and nothing else, so a delegation record — which names a pull request and a head 

41#: — can never be read as a ship run by one that forgot to check the kind. 

42SHIP_RUN_RECORDS: tuple[str, ...] = (RECORD_TYPE_SHIP_RUN,) 

43#: ``delegated``: written once, before any worker starts. ``pull_request``: written once 

44#: per cluster whose worker opened its pull request, naming it and the head it pushed. 

45CONSENT_DELEGATION_EVENTS: tuple[str, ...] = ("delegated", "pull_request") 

46UNKNOWN_RECORD_TYPE_HANDLING = "skip-with-warning" 

47# A skipped kind is named in a warning; a hostile or corrupt line must not flood it. 

48_KIND_DISPLAY_LIMIT = 80 

49 

50 

51class LedgerError(ValueError): 

52 """Raised when a ledger file cannot be decoded as the stable schema.""" 

53 

54 

55def ledger_contract_as_dict(config: cfg.ProjectConfig) -> dict[str, Any]: 

56 """Return the project-neutral ledger storage and schema contract.""" 

57 path, source = configured_ledger_path(config) 

58 return { 

59 "schema_version": LEDGER_SCHEMA_VERSION, 

60 "format": "jsonl", 

61 "path": path, 

62 "path_source": source, 

63 "missing_handling": "treat-as-empty", 

64 "append_owner": ["ship", "swarm-run", "merge"], 

65 "readers": [ 

66 "morning", 

67 "wrap", 

68 "overnight", 

69 "capture-verification", 

70 "consent-verify", 

71 "ledger", 

72 ], 

73 "consumer_neutral": True, 

74 "capture_redaction": redaction.contract_as_dict(config), 

75 "capture_contract": capture.contract_as_dict(config), 

76 "capture_health": capture_health_contract_as_dict(), 

77 "record_types": list(RECORD_TYPES), 

78 "default_read_record_types": list(SHIP_RUN_RECORDS), 

79 "unknown_record_types": UNKNOWN_RECORD_TYPE_HANDLING, 

80 } 

81 

82 

83def capture_health_contract_as_dict() -> dict[str, Any]: 

84 """Return the ledger-derived capture-health summary contract.""" 

85 return { 

86 "schema_version": CAPTURE_HEALTH_SCHEMA_VERSION, 

87 "source": "run-ledger ship_run records", 

88 "readers": ["morning", "wrap", "status", "ledger"], 

89 "consumer_neutral": True, 

90 "missing_ledger_handling": "clean-empty-history", 

91 "dry_run": { 

92 "no_mutations": True, 

93 "safe_reconcile_actions_only": True, 

94 }, 

95 "states": ["clean", "needs-reconcile"], 

96 "item_statuses": ["applied", "deferred", "skipped", "missing-marker"], 

97 } 

98 

99 

100def configured_ledger_path(config: cfg.ProjectConfig) -> tuple[str, str]: 

101 """Return the configured ledger path and the config source that supplied it.""" 

102 pack = config.policy_pack or {} 

103 reports = pack.get("reports") if isinstance(pack.get("reports"), dict) else {} 

104 value = reports.get("run_ledger") 

105 if isinstance(value, str) and value.strip(): 

106 return value, "policy_pack.reports.run_ledger" 

107 return DEFAULT_LEDGER_PATH, "default" 

108 

109 

110def resolve_path(root: str | Path, config: cfg.ProjectConfig) -> Path: 

111 """Resolve the configured ledger path under ``root`` and reject escapes.""" 

112 raw, _ = configured_ledger_path(config) 

113 path = Path(raw) 

114 if workspace.is_root_anchored(raw): 

115 raise LedgerError("run ledger path must be relative to the project root") 

116 root_path = Path(root).resolve() 

117 resolved = (root_path / path).resolve() 

118 try: 

119 resolved.relative_to(root_path) 

120 except ValueError as exc: 

121 raise LedgerError("run ledger path escapes the project root") from exc 

122 return resolved 

123 

124 

125def build_ship_run_record( 

126 *, 

127 command: str, 

128 base_branch: str, 

129 changed_files: list[str] | None, 

130 declared_files: list[str] | None = None, 

131 outcomes: list[Any], 

132 verdict: Any, 

133 assessment: Any, 

134 issue_intake: dict[str, Any] | None = None, 

135 target: str | None = None, 

136 run_id: str | None = None, 

137 issue_number: int | None = None, 

138 pr_number: int | None = None, 

139 branch: str | None = None, 

140 head_sha: str | None = None, 

141 capture_status: str | None = None, 

142 capture_reason: str | None = None, 

143 capture_artifact: str | None = None, 

144 #: The files the *capture* is about, when they are not the files this run's git 

145 #: diff reported. On a post-merge s11 the local diff is empty and the sink reads 

146 #: the PR's files from the host; the ledger has to hash the same list or the 

147 #: dedupe compares two fingerprints of one lesson. ``None`` keeps 

148 #: ``changed_files``, which is every other caller. 

149 capture_changed_files: list[str] | tuple[str, ...] | None = None, 

150 capture_retrieved: list[str] | tuple[str, ...] = (), 

151 capture_not_run: bool = False, 

152 issue_title: str | None = None, 

153 issue_labels: list[str] | tuple[str, ...] = (), 

154 existing_records: list[dict[str, Any]] | None = None, 

155 config: cfg.ProjectConfig | None = None, 

156 implementer: str | None = None, 

157 reviewer_agents: list[str] | None = None, 

158 tester: str | None = None, 

159 host_agent: str | None = None, 

160 transport: str | None = None, 

161 profile: str | None = None, 

162 jury_mode: str | None = None, 

163 jury_panel: dict[str, Any] | None = None, 

164 implement_mode: str | None = None, 

165 implement_phases: list[dict[str, Any]] | None = None, 

166 implement_loop: dict[str, Any] | None = None, 

167 consent_status: str | None = None, 

168 consent_scopes: list[str] | tuple[str, ...] | None = None, 

169 run_controls: dict[str, Any] | None = None, 

170 fix_attribution: dict[str, Any] | None = None, 

171) -> dict[str, Any]: 

172 """Build one deterministic consumer-neutral ship ledger record.""" 

173 return { 

174 "schema_version": LEDGER_SCHEMA_VERSION, 

175 "record_type": RECORD_TYPE_SHIP_RUN, 

176 "command": command, 

177 "run_id": run_id, 

178 "target": target, 

179 "issue": {"number": issue_number}, 

180 "pull_request": {"number": pr_number}, 

181 "git": { 

182 "base_branch": base_branch, 

183 "branch": branch, 

184 "head_sha": head_sha, 

185 }, 

186 # `None` means git could not be read, and stays distinct from `[]` all the way 

187 # into the record: a consumer of the ledger (or of the closure comment rendered 

188 # from it) must not read "we could not see the diff" as "the diff was empty" — 

189 # that is the conflation this whole family of fixes exists to remove, and a 

190 # record claiming TIER-3 with zero files is self-contradictory besides. 

191 "changes": { 

192 "file_count": None if changed_files is None else len(changed_files), 

193 "files": None if changed_files is None else list(changed_files), 

194 "unreadable": changed_files is None, 

195 }, 

196 "declared": _declared_block(declared_files), 

197 "gates": [ 

198 { 

199 "gate": outcome.gate, 

200 "ok": outcome.ok, 

201 "skipped": outcome.skipped, 

202 "timed_out": outcome.timed_out, 

203 # getattr: outcomes reach here from adapters and fixtures that predate 

204 # these fields. Defaulting not_run to False keeps an older producer's 

205 # record readable; on_fail defaults to the strict value so a record 

206 # that *does* carry not_run without a severity fails closed. 

207 "not_run": getattr(outcome, "not_run", False), 

208 "on_fail": getattr(outcome, "on_fail", "block"), 

209 "error": outcome.error, 

210 "finding_count": len(outcome.findings), 

211 # Which panel the jury gate's verdict is (#1437): one keel convened for 

212 # this run, or the one already posted for this head and reused. 

213 **_jury_source(outcome), 

214 } 

215 for outcome in outcomes 

216 ], 

217 "verdict": { 

218 "blocked": verdict.blocked, 

219 "counts": dict(verdict.counts), 

220 }, 

221 "assessment": { 

222 "tier": assessment.tier, 

223 "reviewers": assessment.reviewers, 

224 "window_open": assessment.window_open, 

225 "ci_ok": assessment.ci_ok, 

226 "merge": { 

227 "action": assessment.merge.action, 

228 "reason": assessment.merge.reason, 

229 }, 

230 "halted": assessment.halted, 

231 "bypassed_window": assessment.bypassed_window, 

232 }, 

233 "actors": { 

234 "implementer": implementer, 

235 "reviewers": list(reviewer_agents or ()), 

236 "tester": tester, 

237 # Who took each s9 fix round, from the run-events file (#1016). An escalated 

238 # round was not fixed by the implementer, and a closure comment rendered from 

239 # `implementer` alone says it was. 

240 "fixers": _fixers(fix_attribution), 

241 "attribution_sentence": _attribution_sentence(fix_attribution), 

242 }, 

243 "run_context": _run_context( 

244 host_agent=host_agent, 

245 transport=transport, 

246 profile=profile, 

247 jury_mode=jury_mode, 

248 jury_panel=jury_panel, 

249 implement_mode=implement_mode, 

250 implement_phases=implement_phases, 

251 implement_loop=implement_loop, 

252 consent_status=consent_status, 

253 consent_scopes=consent_scopes, 

254 ), 

255 "run_controls": run_controls, 

256 "issue_intake": issue_intake, 

257 "capture": capture.record_marker( 

258 pr_number=pr_number, 

259 status=capture_status, 

260 reason=capture_reason, 

261 artifact=capture_artifact, 

262 retrieved=capture_retrieved, 

263 title=issue_title, 

264 labels=issue_labels, 

265 # Not `changed_files`: `changes.files` above records what this run's git 

266 # diff reported, which must stay None-preserving, while the capture 

267 # fingerprint has to be the one the document on disk was written with. 

268 changed_files=( 

269 capture_changed_files if capture_changed_files is not None else changed_files 

270 ), 

271 existing_records=existing_records or [], 

272 config=config, 

273 not_run=capture_not_run, 

274 ), 

275 } 

276 

277 

278def _fixers(fix_attribution: dict[str, Any] | None) -> list[dict[str, Any]]: 

279 """The per-round fixer records from a ``keel.fix-attribution.v1`` document.""" 

280 rounds = fix_attribution.get("rounds") if isinstance(fix_attribution, dict) else None 

281 if not isinstance(rounds, list): 

282 return [] 

283 return [item for item in rounds if isinstance(item, dict)] 

284 

285 

286def _attribution_sentence(fix_attribution: dict[str, Any] | None) -> str | None: 

287 """The rendered *"implemented by agy, fixed by opus in round 2"* phrase, when recorded.""" 

288 if not isinstance(fix_attribution, dict): 

289 return None 

290 sentence = fix_attribution.get("sentence") 

291 return sentence if isinstance(sentence, str) and sentence.strip() else None 

292 

293 

294def _declared_block(declared_files: list[str] | None) -> dict[str, Any] | None: 

295 """Build the implementer's declared-scope block, or ``None`` when unset. 

296 

297 ``declared_files`` is the implementer's contract of which files the change is 

298 *supposed* to touch (distinct from the observed ``changes`` diff). When the 

299 implementer does not declare a scope, the block is omitted so readers can 

300 degrade to advisory back-compat behavior. 

301 """ 

302 if declared_files is None: 

303 return None 

304 files = [str(path) for path in declared_files] 

305 return {"file_count": len(files), "files": files} 

306 

307 

308def declared_files_for_record(record: dict[str, Any]) -> list[str] | None: 

309 """Return the implementer's declared file list from a ship_run ``record``. 

310 

311 Returns ``None`` when no declared scope was recorded (a missing or malformed 

312 ``declared`` block), letting ``scope-verify`` degrade to an advisory pass. 

313 """ 

314 declared = record.get("declared") 

315 if not isinstance(declared, dict): 

316 return None 

317 files = declared.get("files") 

318 if not isinstance(files, list): 

319 return None 

320 return [str(path) for path in files] 

321 

322 

323def build_run_context( 

324 *, 

325 host_agent: str | None, 

326 transport: str | None, 

327 consent_status: str | None, 

328 consent_scopes: list[str] | tuple[str, ...] | None, 

329) -> dict[str, Any]: 

330 """The ``run_context`` block :func:`build_ship_run_record` writes, for a writer that 

331 builds its record another way — ``swarm-land``'s landing record (#1422). 

332 

333 The profile, the jury and the s4 fields are left unset: a landing knows none of them. 

334 """ 

335 return _run_context( 

336 host_agent=host_agent, 

337 transport=transport, 

338 profile=None, 

339 jury_mode=None, 

340 jury_panel=None, 

341 consent_status=consent_status, 

342 consent_scopes=consent_scopes, 

343 ) 

344 

345 

346def _run_context( 

347 *, 

348 host_agent: str | None, 

349 transport: str | None, 

350 profile: str | None, 

351 jury_mode: str | None, 

352 jury_panel: dict[str, Any] | None, 

353 consent_status: str | None, 

354 consent_scopes: list[str] | tuple[str, ...] | None, 

355 implement_mode: str | None = None, 

356 implement_phases: list[dict[str, Any]] | None = None, 

357 implement_loop: dict[str, Any] | None = None, 

358) -> dict[str, Any]: 

359 """Build the deterministic consumer-neutral preflight run-context block. 

360 

361 Every field is optional; a missing scalar degrades to ``None`` so the 

362 closure renderer can present ``unknown``/``none`` without a schema change. 

363 Consent is a small summary: a status and the approved mutation scopes, 

364 reusing the operator/approve-scope inputs already resolved by the caller. 

365 

366 ``implement_mode``/``implement_phases`` are the s4 profile (#1020). A ``default`` run 

367 records ``None`` and an empty phase list, so a record written before the knob existed 

368 reads identically to one written by a run that did not use it. 

369 """ 

370 scopes = [str(scope) for scope in (consent_scopes or ()) if str(scope).strip()] 

371 return { 

372 "host_agent": host_agent if _nonblank(host_agent) else None, 

373 "transport": transport if _nonblank(transport) else None, 

374 "profile": profile if _nonblank(profile) else None, 

375 "jury_mode": jury_mode if _nonblank(jury_mode) else None, 

376 # The panel-availability probe this run measured, or `None` when the tier 

377 # never named a panel (#1066). Recorded so a reader of the ledger can tell 

378 # a jury-reviewed change from one a host bench reviewed because the panel 

379 # could not be staffed, without re-deriving it from a machine that has 

380 # since changed. 

381 "jury_panel": dict(jury_panel) if isinstance(jury_panel, dict) else None, 

382 # The s4 profile and, under `tdd`, one record per phase — the ledger is where a 

383 # closure comment and a later audit learn that this change was written test-first 

384 # and which commit each half of s4 produced. 

385 "implement_mode": implement_mode if _nonblank(implement_mode) else None, 

386 "implement_phases": [dict(phase) for phase in implement_phases or ()], 

387 # The s4 iteration loop (#1165): its policy and one record per iteration, or 

388 # `None` for a run that neither configured nor recorded one. 

389 "implement_loop": dict(implement_loop) if isinstance(implement_loop, dict) else None, 

390 "consent": { 

391 "status": consent_status if _nonblank(consent_status) else None, 

392 "scopes": scopes, 

393 }, 

394 } 

395 

396 

397def _nonblank(value: Any) -> bool: 

398 return isinstance(value, str) and bool(value.strip()) 

399 

400 

401def encode_record(record: dict[str, Any]) -> str: 

402 """Encode one ledger record as stable JSONL.""" 

403 _validate_record(record) 

404 return json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n" 

405 

406 

407def parse_records( 

408 text: str, 

409 *, 

410 warn: Callable[[str], None] | None = None, 

411 kinds: Collection[str] = SHIP_RUN_RECORDS, 

412) -> list[dict[str, Any]]: 

413 """Parse ledger JSONL text into validated records of the kinds this keel knows. 

414 

415 **Every record of a known kind is validated; only those of ``kinds`` are returned** 

416 — ship runs unless the reader asks for more (:data:`SHIP_RUN_RECORDS`). A reader that 

417 wants a ``consent_delegation`` record (``consent-verify``) or every kind (``keel 

418 ledger``) names it; every other reader is handed ship runs alone, as it always was. 

419 

420 **Forward compatible** (#1400): a well-formed record whose ``record_type`` names a 

421 kind outside :data:`KNOWN_RECORD_TYPES` is skipped, never returned and never a 

422 reason to refuse the ledger — so no reader can mistake it for a ship run, and a 

423 kind a newer keel appends does not lock an older keel out of its own history. 

424 ``warn`` is told once per skipped kind, with the line it was first seen on. 

425 

426 Everything else is refused exactly as before: invalid JSON, a non-object, a wrong 

427 ``schema_version``, a record with no ``record_type`` at all (or one that is not a 

428 non-blank string), and — through :func:`_validate_record` — any record of a known 

429 kind. Only a record that *names* a kind can be set aside as one. 

430 """ 

431 records: list[dict[str, Any]] = [] 

432 skipped: set[str] = set() 

433 for line_number, raw in enumerate(text.splitlines(), start=1): 

434 if not raw.strip(): 

435 continue 

436 try: 

437 record = json.loads(raw) 

438 except json.JSONDecodeError as exc: 

439 raise LedgerError(f"line {line_number}: invalid JSON") from exc 

440 kind = _unknown_kind(record) 

441 if kind is not None: 

442 if warn is not None and kind not in skipped: 

443 warn(unknown_kind_warning(kind, line_number)) 

444 skipped.add(kind) 

445 continue 

446 _validate_record(record, line_number=line_number) 

447 if record["record_type"] in kinds: 

448 records.append(record) 

449 return records 

450 

451 

452def unknown_kind_warning(kind: str, line_number: int) -> str: 

453 """The one warning a reader gives for a record kind it skips.""" 

454 shown = kind if len(kind) <= _KIND_DISPLAY_LIMIT else kind[:_KIND_DISPLAY_LIMIT] + "..." 

455 return ( 

456 f"run ledger line {line_number}: skipped record_type {shown!r}, which this keel " 

457 "does not know (a newer keel may have written it); later records of that type " 

458 "are skipped too" 

459 ) 

460 

461 

462def read_records( 

463 path: str | Path, 

464 *, 

465 warn: Callable[[str], None] | None = None, 

466 kinds: Collection[str] = SHIP_RUN_RECORDS, 

467) -> list[dict[str, Any]]: 

468 """Read a ledger file; a missing ledger is a valid empty history. 

469 

470 Record kinds this keel does not know are skipped, and only ``kinds`` are returned, as 

471 :func:`parse_records` does. 

472 """ 

473 ledger_path = Path(path) 

474 if not ledger_path.exists(): 

475 return [] 

476 return parse_records(ledger_path.read_text(encoding="utf-8"), warn=warn, kinds=kinds) 

477 

478 

479def build_consent_delegation_record( 

480 delegation: Mapping[str, Any], 

481 *, 

482 recorded_at: str, 

483 pull_request: Mapping[str, Any] | None = None, 

484) -> dict[str, Any]: 

485 """A ``consent_delegation`` record for ``delegation`` (#1400). 

486 

487 ``delegation`` is :meth:`keel.swarm_worker.ConsentDelegation.to_dict`: who consented, 

488 to which scopes, how, when, and for which run and clusters. Without ``pull_request`` 

489 the record is the ``delegated`` event, written before any worker starts; with it, the 

490 ``pull_request`` event for one cluster — ``{"cluster", "number", "branch", 

491 "head_sha", "url"}`` — written once that cluster's worker has opened its pull request. 

492 

493 Two events, never an update: the ledger is append-only, and the pull request number is 

494 known only after ``gh pr create``. Each record carries the whole delegation, so a 

495 reader needs one record to answer for a pull request. The parent's verbatim consent 

496 record is not copied: every field a reader needs is named here. 

497 """ 

498 record: dict[str, Any] = { 

499 "schema_version": LEDGER_SCHEMA_VERSION, 

500 "record_type": RECORD_TYPE_CONSENT_DELEGATION, 

501 "event": "delegated" if pull_request is None else "pull_request", 

502 "swarm_id": delegation.get("swarm_id"), 

503 "clusters": list(delegation.get("clusters") or ()), 

504 "scopes": list(delegation.get("scopes") or ()), 

505 "operator": delegation.get("operator"), 

506 "source": delegation.get("source"), 

507 "mode": delegation.get("mode"), 

508 "delegated_at": delegation.get("delegated_at"), 

509 "recorded_at": recorded_at, 

510 "pull_request": None if pull_request is None else dict(pull_request), 

511 } 

512 _validate_record(record) 

513 return record 

514 

515 

516def build_jury_followup_record( 

517 *, 

518 pr_number: int, 

519 head_sha: str, 

520 issue_number: int, 

521 issue_url: str | None, 

522 findings: list[str] | tuple[str, ...], 

523 recorded_at: str, 

524) -> dict[str, Any]: 

525 """A ``jury_followup`` record: the issue filed for a COMMENT panel's minors (#1570).""" 

526 record: dict[str, Any] = { 

527 "schema_version": LEDGER_SCHEMA_VERSION, 

528 "record_type": RECORD_TYPE_JURY_FOLLOWUP, 

529 "pull_request": {"number": pr_number}, 

530 "head_sha": head_sha, 

531 "issue": {"number": issue_number, "url": issue_url}, 

532 "findings": list(findings), 

533 "recorded_at": recorded_at, 

534 } 

535 _validate_record(record) 

536 return record 

537 

538 

539def jury_followup_for_head( 

540 records: list[dict[str, Any]], 

541 pr_number: int, 

542 head_sha: str, 

543 *, 

544 covered_heads: Collection[str] = (), 

545) -> dict[str, Any] | None: 

546 """Latest ``jury_followup`` record for ``pr_number`` at ``head_sha`` or ``covered_heads``.""" 

547 accepted = {head_sha, *covered_heads} 

548 match: dict[str, Any] | None = None 

549 for record in records: 

550 if record.get("record_type") != RECORD_TYPE_JURY_FOLLOWUP: 

551 continue 

552 if (record.get("pull_request") or {}).get("number") == pr_number and record.get( 

553 "head_sha" 

554 ) in accepted: 

555 match = record 

556 return match 

557 

558 

559def consent_delegation_for_pr( 

560 records: list[dict[str, Any]], pr_number: int 

561) -> dict[str, Any] | None: 

562 """The latest ``pull_request`` delegation record naming ``pr_number``, or ``None``. 

563 

564 The pull request number is the only key: keel wrote it from ``gh pr create``'s answer, 

565 so it names the pull request the worker opened. A branch name is not a key — anyone can 

566 open a pull request from a fork with a ``swarm/<id>/<cluster>`` branch — and a reader 

567 that gates on the record must still check the pull request's head against the record's 

568 ``head_sha`` (:func:`keel.consentverify.consent_for_pr` does). 

569 """ 

570 match: dict[str, Any] | None = None 

571 for record in records: 

572 if record.get("record_type") != RECORD_TYPE_CONSENT_DELEGATION: 

573 continue 

574 if (record.get("pull_request") or {}).get("number") == pr_number: 

575 match = record 

576 return match 

577 

578 

579def latest_ship_run_for_pr( 

580 records: list[dict[str, Any]], 

581 pr_number: int, 

582) -> dict[str, Any] | None: 

583 """Return the last ship_run record whose pull_request matches ``pr_number``. 

584 

585 Records are appended in chronological order, so the last match is the most 

586 recent ship run for that PR. Returns ``None`` when no record matches. 

587 

588 **Scoped by pull request, not by head, and a caller that gates on the record must 

589 say so itself.** A pull request outlives its heads, so the record this returns may 

590 have been written for a commit that is no longer the head — right for a reader 

591 asking "what happened on this PR" (capture health, scope verification), wrong for 

592 anything a merge decision hangs on. The two callers that gate check the head 

593 themselves: :func:`gates_pass_for_head` selects on ``git.head_sha`` here, and 

594 :func:`keel.juryavail.shipped` refuses a record whose head is not the one being 

595 verified — without which an earlier head's fallback weakened the current head's 

596 contract (#1068 round 2). 

597 """ 

598 match: dict[str, Any] | None = None 

599 for record in records: 

600 if record.get("record_type") != RECORD_TYPE_SHIP_RUN: 

601 continue 

602 pull_request = record.get("pull_request") 

603 number = pull_request.get("number") if isinstance(pull_request, dict) else None 

604 if number == pr_number: 

605 match = record 

606 return match 

607 

608 

609def _jury_source(outcome: Any) -> dict[str, Any]: 

610 """The jury gate's provenance fields for a ledger gate entry (#1437). 

611 

612 ``source: reused`` with the posted comment it was read from, or ``source: ran`` for a 

613 jury this run executed; nothing for any other gate, or for a jury nobody ran. 

614 """ 

615 reused = getattr(outcome, "reused_from", None) 

616 if reused is not None: 

617 return {"source": "reused", "reused_from": reused.as_dict()} 

618 if outcome.gate == "jury" and not getattr(outcome, "not_run", False): 

619 return {"source": "ran"} 

620 return {} 

621 

622 

623def record_gates_passed(record: dict[str, Any]) -> bool: 

624 """Return whether a ship_run record's gates count as a clean pass. 

625 

626 A pass requires that the run was not blocked by findings and that every 

627 recorded gate either ran clean (``ok``) or was deliberately skipped, with no 

628 gate reporting an error. A missing or malformed ``gates``/``verdict`` block 

629 degrades to "not a pass" so a corrupt record can never authorize a merge. 

630 

631 A gate marked ``not_run`` was never executed by the runner that wrote the 

632 record — an ``agentic`` gate reaching the command-only runner. For a 

633 ``block``-severity gate that is **not** a pass: certifying "gates passed" for a 

634 blocking review nobody ran is exactly the fail-open this check exists to stop. 

635 Soft (``warn``/``suggest``) not-run gates are tolerated, matching ``skipped``. 

636 Records written before this field existed have no ``not_run`` key and are 

637 unaffected. 

638 """ 

639 verdict = record.get("verdict") 

640 if not isinstance(verdict, dict) or verdict.get("blocked") is not False: 

641 return False 

642 gates = record.get("gates") 

643 if not isinstance(gates, list) or not gates: 

644 return False 

645 for gate in gates: 

646 if not isinstance(gate, dict): 

647 return False 

648 if gate.get("error"): 

649 return False 

650 if not (gate.get("ok") is True or gate.get("skipped") is True): 

651 return False 

652 # Missing `on_fail` defaults to the strict value *here*, at read time. Defaulting 

653 # only at write time protects records keel wrote and no others: any producer that 

654 # learns `not_run` without its sibling key would otherwise fail open in exactly 

655 # the certification path this check exists to close. 

656 # Fail closed on anything not explicitly soft: a missing key, a JSON-round-tripped 

657 # `None`, or a severity name keel does not know all mean "we cannot tell this was 

658 # optional", and this is the certification path. 

659 if gate.get("not_run") is True and gate.get("on_fail") not in ("warn", "suggest"): 

660 return False 

661 return True 

662 

663 

664def gates_pass_for_head( 

665 records: list[dict[str, Any]], 

666 pr_number: int, 

667 head_sha: str, 

668 covered_heads: Collection[str] = (), 

669) -> tuple[bool, dict[str, Any] | None]: 

670 """Find a passing gates run recorded against ``head_sha`` for ``pr_number``. 

671 

672 Returns ``(matched, record)``. ``matched`` is ``True`` only when the **latest** 

673 ship_run record for the PR carrying the exact current ``head_sha`` passed its 

674 gates (see :func:`record_gates_passed`); the record is returned on a match and 

675 ``None`` otherwise. A blank ``head_sha`` never matches — an unknown head must 

676 not be authorized by a stale green run. This is a pure function: it reads only 

677 its arguments and performs no I/O. 

678 

679 Latest-wins, not any-pass. Re-gating the same head is ordinary — a flaky suite 

680 settles, a fix-loop re-runs, a dependency lands — so the same ``head_sha`` can 

681 carry a green record followed by a red one. Scanning for *any* green would let 

682 the superseded pass authorize the merge and the later red never be consulted: 

683 a fail-open in exactly the gate that exists to hold the merge closed. Only the 

684 most recent verdict for the head counts. 

685 

686 ``covered_heads`` are heads the current one descends from by capture commits alone 

687 (#1203): the learning lands on the pull request's own branch after the gates ran, so 

688 the gates-pass is recorded against a head the branch has moved one lesson past. A 

689 record for one of them counts, and **latest-wins still holds across the whole set** — 

690 a red record for any of them after a green one is the verdict. The set is produced by 

691 `capture.capture_only_descent` and nothing else, so it admits a markdown file inside 

692 the sink, never code; CI still runs on the new head and `keel merge` still requires 

693 that rollup to be green. 

694 """ 

695 if not isinstance(head_sha, str) or not head_sha.strip(): 

696 return False, None 

697 accepted = {head_sha, *covered_heads} 

698 latest: dict[str, Any] | None = None 

699 for record in records: 

700 if record.get("record_type") != RECORD_TYPE_SHIP_RUN: 

701 continue 

702 pull_request = record.get("pull_request") 

703 number = pull_request.get("number") if isinstance(pull_request, dict) else None 

704 if number != pr_number: 

705 continue 

706 git = record.get("git") 

707 record_sha = git.get("head_sha") if isinstance(git, dict) else None 

708 if record_sha not in accepted: 

709 continue 

710 latest = record 

711 if latest is None or not record_gates_passed(latest): 

712 return False, None 

713 return True, latest 

714 

715 

716def _latest_per_pr(records: list[dict[str, Any]]) -> list[dict[str, Any]]: 

717 """One record per pull request — the last — keeping records that name none. 

718 

719 A pull request has one capture, and since #1157 it can leave more than one 

720 marked record: the writer's clash is keyed by ``(pull request, head)``, so a 

721 superseded head's marker survives beside the merged head's. Counting rows 

722 rather than pull requests then reported one merged pull request twice — 

723 ``applied: 1, skipped: 1`` for a single capture — in morning, wrap and status. 

724 Insertion order is preserved so these readers still list captures in the order 

725 the ledger recorded them. 

726 """ 

727 latest: dict[int, dict[str, Any]] = {} 

728 unkeyed: list[dict[str, Any]] = [] 

729 for record in records: 

730 pull_request = record.get("pull_request") 

731 number = pull_request.get("number") if isinstance(pull_request, dict) else None 

732 if isinstance(number, int): 

733 latest[number] = record 

734 else: 

735 unkeyed.append(record) 

736 # Identity, not equality: two equal rows are still two rows. A set of id()s makes 

737 # the filter one pass; `any(record is kept for kept in keep)` was records × kept, 

738 # and the ledger is append-only, so that product only grows (#1355). Every id is 

739 # of an object `records` still holds, so none can be reused mid-comprehension. 

740 kept_ids = {id(kept) for kept in [*latest.values(), *unkeyed]} 

741 return [record for record in records if id(record) in kept_ids] 

742 

743 

744def capture_health_summary(records: list[dict[str, Any]]) -> dict[str, Any]: 

745 """Summarize capture visibility for morning, wrap, status, and ledger readers.""" 

746 merged_records = _latest_per_pr([r for r in records if _is_merged_ship_run(r)]) 

747 items = [_capture_health_item(record) for record in merged_records] 

748 counts = { 

749 "applied": 0, 

750 "marker_only": 0, 

751 "create_learning": 0, 

752 "duplicate_learning": 0, 

753 "deferred": 0, 

754 "skipped": 0, 

755 "missing_marker": 0, 

756 "needs_reconcile": 0, 

757 } 

758 skipped_by_reason: dict[str, int] = {} 

759 for item in items: 

760 status = item["status"] 

761 learning = item["learning_decision"] 

762 if status == "applied": 

763 counts["applied"] += 1 

764 elif status == "deferred": 

765 counts["deferred"] += 1 

766 elif status == "skipped": 

767 counts["skipped"] += 1 

768 skipped_by_reason[item["reason"] or "unspecified"] = ( 

769 skipped_by_reason.get(item["reason"] or "unspecified", 0) + 1 

770 ) 

771 else: 

772 counts["missing_marker"] += 1 

773 if learning == "marker-only": 

774 counts["marker_only"] += 1 

775 elif learning == "create-learning": 

776 counts["create_learning"] += 1 

777 elif learning == "duplicate": 

778 counts["duplicate_learning"] += 1 

779 if item["needs_reconcile"]: 

780 counts["needs_reconcile"] += 1 

781 return { 

782 "schema_version": CAPTURE_HEALTH_SCHEMA_VERSION, 

783 "status": "needs-reconcile" if counts["needs_reconcile"] else "clean", 

784 "record_count": len(merged_records), 

785 "counts": counts, 

786 "skipped_by_reason": dict(sorted(skipped_by_reason.items())), 

787 "items": items, 

788 "reconcile_actions": [action for item in items for action in item["reconcile_actions"]], 

789 "dry_run": { 

790 "no_mutations": True, 

791 "description": "Morning and wrap surface these actions; they do not mutate " 

792 "ledger, GitHub, or capture destinations.", 

793 }, 

794 } 

795 

796 

797def _is_merged_ship_run(record: Any) -> bool: 

798 if not isinstance(record, dict): 

799 return False 

800 if record.get("record_type") != RECORD_TYPE_SHIP_RUN: 

801 return False 

802 capture_block = record.get("capture") 

803 # The operator declared this run never reached capture, which settles the 

804 # question before the assessment is consulted: that block records what ship 

805 # *recommended*, and a recommendation to merge is read here as proof the PR 

806 # merged. For a run re-recording gates after a rebase both are true at once — 

807 # gates green, nothing merged — and the record would otherwise be counted as a 

808 # merged PR whose marker went missing, a capture gap that is not there (#945). 

809 # Only an explicit declaration does this; a plain ``status: None`` still falls 

810 # through, so a marker that genuinely went missing is still reported. 

811 if isinstance(capture_block, dict) and capture_block.get("not_run") is True: 

812 return False 

813 assessment = record.get("assessment") 

814 if isinstance(assessment, dict): 

815 merge = assessment.get("merge") 

816 if isinstance(merge, dict) and merge.get("action") == "merge": 

817 return True 

818 if isinstance(capture_block, dict) and capture_block.get("status") in ( 

819 "applied", 

820 "deferred", 

821 "skipped", 

822 ): 

823 return True 

824 return False 

825 

826 

827def _capture_marker(record: dict[str, Any]) -> str | None: 

828 capture = record.get("capture") 

829 marker = capture.get("marker") if isinstance(capture, dict) else None 

830 return marker if isinstance(marker, str) and marker.strip() else None 

831 

832 

833def record_head_sha(record: Mapping[str, Any]) -> str | None: 

834 """The head a ship_run record was written for, or ``None`` when it names none.""" 

835 git = record.get("git") 

836 head = git.get("head_sha") if isinstance(git, Mapping) else None 

837 return head.strip() if isinstance(head, str) and head.strip() else None 

838 

839 

840def capture_marker_for_head( 

841 records: list[dict[str, Any]], 

842 *, 

843 pr_number: int | None, 

844 head_sha: str | None, 

845) -> dict[str, Any] | None: 

846 """The recorded capture marker for this ``(pull request, head)``, if any. 

847 

848 The same rule :func:`existing_capture_marker` enforces, asked **before** a 

849 record exists. The clash is keyed on the pair and nothing else, so a caller 

850 about to do durable work for this run can find out whether its append will 

851 land — a learning file written ahead of an append that then no-ops is a 

852 document on disk that no ledger record will ever point at. 

853 """ 

854 if not isinstance(pr_number, int): 

855 return None 

856 head = head_sha.strip() if isinstance(head_sha, str) and head_sha.strip() else None 

857 for existing in records: 

858 if existing.get("record_type") != RECORD_TYPE_SHIP_RUN: 

859 continue 

860 other = existing.get("pull_request") 

861 if (other.get("number") if isinstance(other, dict) else None) != pr_number: 

862 continue 

863 if record_head_sha(existing) != head: 

864 continue 

865 if _capture_marker(existing) is not None: 

866 return existing 

867 return None 

868 

869 

870def existing_capture_marker( 

871 records: list[dict[str, Any]], record: dict[str, Any] 

872) -> dict[str, Any] | None: 

873 """The already-recorded capture marker ``record`` would duplicate, if any. 

874 

875 One capture marker per **(pull request, head)**, enforced at write time. It was 

876 once only *detected*, and only afterwards: :func:`keel.capture.verify_session` 

877 refuses the whole session on a second one ("multiple capture markers found for 

878 merged PR"), ``capture-reconcile`` returns ``blocked`` with no actions to offer, 

879 and nothing in this module can remove a line — so the recovery was editing the 

880 ledger by hand, which is forging audit history to make a gate pass. 

881 

882 Re-running the same append is the most natural thing to do after a crash mid-s11, 

883 which made the obvious recovery the very action that bricks the run. Checking here 

884 costs one pass over records the caller already holds. Returns the conflicting 

885 record so the caller can name it; ``None`` when the append is new. 

886 

887 **Scoped to the head, because that is how the merge gate reads it** (#1157). Keyed 

888 by pull request alone, this refused every later head once *any* record carried a 

889 marker — including the very first run, red. :func:`gates_pass_for_head` and 

890 :func:`keel.juryavail.is_ship_run_for_head` then asked for a passing record on the 

891 *current* head, which no run was any longer allowed to write. A pull request whose 

892 first ship run failed became permanently unmergeable, and the documented exit was 

893 editing an append-only audit ledger to make a gate pass, which is the one action 

894 this design exists to prevent. The writer now keys the record the way every reader 

895 that gates does: retrying the same head still clashes, a new head is a new run. 

896 

897 A record naming no head is compared to other records naming no head — the same 

898 rule, applied to the value they have, rather than an exemption from it. 

899 """ 

900 if _capture_marker(record) is None: 

901 return None 

902 pull_request = record.get("pull_request") 

903 pr = pull_request.get("number") if isinstance(pull_request, dict) else None 

904 if not isinstance(pr, int): 

905 return None 

906 return capture_marker_for_head(records, pr_number=pr, head_sha=record_head_sha(record)) 

907 

908 

909def append_record(path: str | Path, record: dict[str, Any]) -> None: 

910 """Append one validated JSONL record, creating parent directories as needed.""" 

911 ledger_path = Path(path) 

912 ledger_path.parent.mkdir(parents=True, exist_ok=True) 

913 workspace.ensure_runtime_gitignore_for(ledger_path) 

914 with ledger_path.open("a", encoding="utf-8") as handle: 

915 handle.write(encode_record(record)) 

916 

917 

918def sanitize_record( 

919 record: dict[str, Any], 

920 config: cfg.ProjectConfig | None = None, 

921) -> dict[str, Any]: 

922 """Apply capture redaction before a ledger record becomes durable.""" 

923 result = redaction.sanitize(record, redaction.policy_from_config(config)) 

924 sanitized = dict(result.value) 

925 sanitized["redaction"] = result.audit 

926 return sanitized 

927 

928 

929def _unknown_kind(record: Any) -> str | None: 

930 """The ``record_type`` of a record of a kind this keel does not know, else ``None``. 

931 

932 Only a record that is otherwise in this ledger's schema and *names* its kind 

933 qualifies: a missing, blank or non-string ``record_type`` is not a kind to skip but 

934 a malformed record, and :func:`_validate_record` refuses it as it always has. 

935 """ 

936 if not isinstance(record, dict) or record.get("schema_version") != LEDGER_SCHEMA_VERSION: 

937 return None 

938 kind = record.get("record_type") 

939 if not isinstance(kind, str) or not kind.strip() or kind in KNOWN_RECORD_TYPES: 

940 return None 

941 return kind 

942 

943 

944def _validate_record(record: Any, *, line_number: int | None = None) -> None: 

945 prefix = f"line {line_number}: " if line_number is not None else "" 

946 if not isinstance(record, dict): 

947 raise LedgerError(f"{prefix}record must be an object") 

948 if record.get("schema_version") != LEDGER_SCHEMA_VERSION: 

949 raise LedgerError(f"{prefix}unsupported schema_version") 

950 kind = record.get("record_type") 

951 if kind == RECORD_TYPE_CONSENT_DELEGATION: 

952 if problem := _consent_delegation_problem(record): 

953 raise LedgerError(f"{prefix}invalid consent_delegation record: {problem}") 

954 elif kind == RECORD_TYPE_JURY_FOLLOWUP: 

955 if problem := _jury_followup_problem(record): 

956 raise LedgerError(f"{prefix}invalid jury_followup record: {problem}") 

957 elif kind != RECORD_TYPE_SHIP_RUN: 

958 raise LedgerError(f"{prefix}unsupported record_type") 

959 

960 

961def _positive_int(value: Any) -> bool: 

962 return isinstance(value, int) and not isinstance(value, bool) and value > 0 

963 

964 

965def _jury_followup_problem(record: dict[str, Any]) -> str: 

966 """What is wrong with a ``jury_followup`` record, or ``""`` when nothing is.""" 

967 if not _positive_int((record.get("pull_request") or {}).get("number")): 

968 return "pull_request.number must be a positive integer" 

969 for key in ("head_sha", "recorded_at"): 

970 if not _nonblank(record.get(key)): 

971 return f"{key} must be a non-blank string" 

972 if not _positive_int((record.get("issue") or {}).get("number")): 

973 return "issue.number must be a positive integer" 

974 findings = record.get("findings") 

975 if not isinstance(findings, list) or not findings or not all(_nonblank(f) for f in findings): 

976 return "findings must be a non-empty list of non-blank strings" 

977 return "" 

978 

979 

980def _consent_delegation_problem(record: dict[str, Any]) -> str: 

981 """What is wrong with a ``consent_delegation`` record, or ``""`` when nothing is. 

982 

983 Strict about every field a reader relies on; a field this keel does not name is 

984 tolerated, so a later keel can add one without making this one refuse the ledger. 

985 """ 

986 event = record.get("event") 

987 if event not in CONSENT_DELEGATION_EVENTS: 

988 return f"event must be one of {', '.join(CONSENT_DELEGATION_EVENTS)}" 

989 for key in ("swarm_id", "operator", "source", "mode", "delegated_at", "recorded_at"): 

990 if not _nonblank(record.get(key)): 

991 return f"{key} must be a non-blank string" 

992 clusters = record.get("clusters") 

993 if ( 

994 not isinstance(clusters, list) 

995 or not clusters 

996 or not all(_nonblank(c) for c in clusters) 

997 or len(set(clusters)) != len(clusters) 

998 ): 

999 return "clusters must be a non-empty list of distinct non-blank strings" 

1000 scopes = record.get("scopes") 

1001 if not isinstance(scopes, list) or not scopes or not all(_nonblank(s) for s in scopes): 

1002 return "scopes must be a non-empty list of non-blank strings" 

1003 try: 

1004 normalized = list(consent.normalize_scopes(scopes)) 

1005 except ValueError as exc: 

1006 return str(exc) 

1007 if normalized != scopes: 

1008 return f"scopes must be normalised: {normalized}" 

1009 return _delegated_pull_request_problem(event, record.get("pull_request"), clusters) 

1010 

1011 

1012def _delegated_pull_request_problem(event: Any, pull_request: Any, clusters: list[Any]) -> str: 

1013 """What is wrong with a delegation record's ``pull_request`` for its ``event``.""" 

1014 if event == "delegated": 

1015 return "" if pull_request is None else "a delegated event names no pull request" 

1016 if not isinstance(pull_request, dict): 

1017 return "a pull_request event names its pull request" 

1018 if pull_request.get("cluster") not in clusters: 

1019 return "pull_request.cluster must be one of the delegation's clusters" 

1020 number = pull_request.get("number") 

1021 if isinstance(number, bool) or not isinstance(number, int) or number < 1: 

1022 return "pull_request.number must be a positive integer" 

1023 for key in ("branch", "head_sha"): 

1024 if not _nonblank(pull_request.get(key)): 

1025 return f"pull_request.{key} must be a non-blank string" 

1026 url = pull_request.get("url") 

1027 if url is not None and not isinstance(url, str): 

1028 return "pull_request.url must be a string" 

1029 return "" 

1030 

1031 

1032def _capture_health_item(record: dict[str, Any]) -> dict[str, Any]: 

1033 block = record.get("capture") if isinstance(record.get("capture"), dict) else {} 

1034 status = block.get("status") 

1035 marker = block.get("marker") 

1036 reason = block.get("marker_reason") or block.get("reason") 

1037 learning = block.get("learning") if isinstance(block.get("learning"), dict) else {} 

1038 item_status = _capture_health_status(status, marker) 

1039 needs_reconcile = item_status in {"missing-marker", "deferred"} 

1040 pr_number = (record.get("pull_request") or {}).get("number") 

1041 item = { 

1042 "run_id": record.get("run_id"), 

1043 "issue": (record.get("issue") or {}).get("number"), 

1044 "pull_request": pr_number, 

1045 "status": item_status, 

1046 "capture_status": status, 

1047 "marker": marker if isinstance(marker, str) and marker.strip() else None, 

1048 "reason": reason if isinstance(reason, str) and reason.strip() else None, 

1049 "learning_decision": learning.get("decision"), 

1050 "learning_reason": learning.get("reason"), 

1051 "needs_reconcile": needs_reconcile, 

1052 "reconcile_actions": [], 

1053 } 

1054 if needs_reconcile: 

1055 item["reconcile_actions"].append(_capture_reconcile_action(pr_number, item_status)) 

1056 return item 

1057 

1058 

1059def _capture_health_status(status: Any, marker: Any) -> str: 

1060 if not isinstance(marker, str) or not marker.strip(): 

1061 return "missing-marker" 

1062 if status == "deferred": 

1063 return "deferred" 

1064 if status == "skipped": 

1065 return "skipped" 

1066 return "applied" 

1067 

1068 

1069def _capture_reconcile_action(pr_number: Any, status: str) -> dict[str, Any]: 

1070 return { 

1071 "type": "capture-reconcile", 

1072 "reason": status, 

1073 "pr": pr_number if isinstance(pr_number, int) else None, 

1074 "command": ( 

1075 f"keel capture-reconcile .keel/project.yaml --root . --merged-pr {pr_number}" 

1076 if isinstance(pr_number, int) 

1077 else "keel capture-reconcile .keel/project.yaml --root ." 

1078 ), 

1079 "dry_run": True, 

1080 }