Coverage for src/keel/ledger.py: 100%
376 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
1"""Structured run ledger helpers for keel workflows."""
3from __future__ import annotations
5import json
6from collections.abc import Callable, Collection, Mapping
7from pathlib import Path
8from typing import Any
10from . import capture, consent, redaction, workspace
11from . import config as cfg
13LEDGER_SCHEMA_VERSION = "keel.run-ledger.v1"
14CAPTURE_HEALTH_SCHEMA_VERSION = "keel.capture-health.v1"
15DEFAULT_LEDGER_PATH = ".keel/state/run-ledger.jsonl"
16RECORD_TYPE_SHIP_RUN = "ship_run"
17#: The operator's live consent as ``swarm-run --live`` delegated it to its workers (#1400):
18#: one record when the run delegates it, and one more per cluster pull request its
19#: workers open. See :func:`build_consent_delegation_record`.
20RECORD_TYPE_CONSENT_DELEGATION = "consent_delegation"
21#: The follow-up issue ``keel merge`` filed for the minor findings a ``COMMENT`` jury
22#: consensus left when ``team.jury.comment_consensus`` passed the gate (#1570). One per
23#: pull request and head, so a retried merge reuses the issue instead of filing another.
24RECORD_TYPE_JURY_FOLLOWUP = "jury_followup"
25# The record kinds this keel reads. A reader skips any other kind (#1400): the ledger is
26# append-only and shared by every keel on a checkout, so a kind a newer keel adds must not
27# make an older keel refuse the history it can still read. The writer stays strict —
28# :func:`encode_record` only ever writes a kind named here.
29#: :data:`KNOWN_RECORD_TYPES` in its published order (the ledger contract's
30#: ``record_types``). A keel older than 1.26.0 refuses a ledger holding any kind but the
31#: first: those readers did not yet skip a kind they did not know.
32RECORD_TYPES: tuple[str, ...] = (
33 RECORD_TYPE_SHIP_RUN,
34 RECORD_TYPE_CONSENT_DELEGATION,
35 RECORD_TYPE_JURY_FOLLOWUP,
36)
37KNOWN_RECORD_TYPES: frozenset[str] = frozenset(RECORD_TYPES)
38#: What :func:`parse_records` and :func:`read_records` return unless a reader asks for
39#: more. Every reader keel had before #1400 reads ship runs and nothing else; it gets ship
40#: runs and nothing else, so a delegation record — which names a pull request and a head
41#: — can never be read as a ship run by one that forgot to check the kind.
42SHIP_RUN_RECORDS: tuple[str, ...] = (RECORD_TYPE_SHIP_RUN,)
43#: ``delegated``: written once, before any worker starts. ``pull_request``: written once
44#: per cluster whose worker opened its pull request, naming it and the head it pushed.
45CONSENT_DELEGATION_EVENTS: tuple[str, ...] = ("delegated", "pull_request")
46UNKNOWN_RECORD_TYPE_HANDLING = "skip-with-warning"
47# A skipped kind is named in a warning; a hostile or corrupt line must not flood it.
48_KIND_DISPLAY_LIMIT = 80
51class LedgerError(ValueError):
52 """Raised when a ledger file cannot be decoded as the stable schema."""
55def ledger_contract_as_dict(config: cfg.ProjectConfig) -> dict[str, Any]:
56 """Return the project-neutral ledger storage and schema contract."""
57 path, source = configured_ledger_path(config)
58 return {
59 "schema_version": LEDGER_SCHEMA_VERSION,
60 "format": "jsonl",
61 "path": path,
62 "path_source": source,
63 "missing_handling": "treat-as-empty",
64 "append_owner": ["ship", "swarm-run", "merge"],
65 "readers": [
66 "morning",
67 "wrap",
68 "overnight",
69 "capture-verification",
70 "consent-verify",
71 "ledger",
72 ],
73 "consumer_neutral": True,
74 "capture_redaction": redaction.contract_as_dict(config),
75 "capture_contract": capture.contract_as_dict(config),
76 "capture_health": capture_health_contract_as_dict(),
77 "record_types": list(RECORD_TYPES),
78 "default_read_record_types": list(SHIP_RUN_RECORDS),
79 "unknown_record_types": UNKNOWN_RECORD_TYPE_HANDLING,
80 }
83def capture_health_contract_as_dict() -> dict[str, Any]:
84 """Return the ledger-derived capture-health summary contract."""
85 return {
86 "schema_version": CAPTURE_HEALTH_SCHEMA_VERSION,
87 "source": "run-ledger ship_run records",
88 "readers": ["morning", "wrap", "status", "ledger"],
89 "consumer_neutral": True,
90 "missing_ledger_handling": "clean-empty-history",
91 "dry_run": {
92 "no_mutations": True,
93 "safe_reconcile_actions_only": True,
94 },
95 "states": ["clean", "needs-reconcile"],
96 "item_statuses": ["applied", "deferred", "skipped", "missing-marker"],
97 }
100def configured_ledger_path(config: cfg.ProjectConfig) -> tuple[str, str]:
101 """Return the configured ledger path and the config source that supplied it."""
102 pack = config.policy_pack or {}
103 reports = pack.get("reports") if isinstance(pack.get("reports"), dict) else {}
104 value = reports.get("run_ledger")
105 if isinstance(value, str) and value.strip():
106 return value, "policy_pack.reports.run_ledger"
107 return DEFAULT_LEDGER_PATH, "default"
110def resolve_path(root: str | Path, config: cfg.ProjectConfig) -> Path:
111 """Resolve the configured ledger path under ``root`` and reject escapes."""
112 raw, _ = configured_ledger_path(config)
113 path = Path(raw)
114 if workspace.is_root_anchored(raw):
115 raise LedgerError("run ledger path must be relative to the project root")
116 root_path = Path(root).resolve()
117 resolved = (root_path / path).resolve()
118 try:
119 resolved.relative_to(root_path)
120 except ValueError as exc:
121 raise LedgerError("run ledger path escapes the project root") from exc
122 return resolved
125def build_ship_run_record(
126 *,
127 command: str,
128 base_branch: str,
129 changed_files: list[str] | None,
130 declared_files: list[str] | None = None,
131 outcomes: list[Any],
132 verdict: Any,
133 assessment: Any,
134 issue_intake: dict[str, Any] | None = None,
135 target: str | None = None,
136 run_id: str | None = None,
137 issue_number: int | None = None,
138 pr_number: int | None = None,
139 branch: str | None = None,
140 head_sha: str | None = None,
141 capture_status: str | None = None,
142 capture_reason: str | None = None,
143 capture_artifact: str | None = None,
144 #: The files the *capture* is about, when they are not the files this run's git
145 #: diff reported. On a post-merge s11 the local diff is empty and the sink reads
146 #: the PR's files from the host; the ledger has to hash the same list or the
147 #: dedupe compares two fingerprints of one lesson. ``None`` keeps
148 #: ``changed_files``, which is every other caller.
149 capture_changed_files: list[str] | tuple[str, ...] | None = None,
150 capture_retrieved: list[str] | tuple[str, ...] = (),
151 capture_not_run: bool = False,
152 issue_title: str | None = None,
153 issue_labels: list[str] | tuple[str, ...] = (),
154 existing_records: list[dict[str, Any]] | None = None,
155 config: cfg.ProjectConfig | None = None,
156 implementer: str | None = None,
157 reviewer_agents: list[str] | None = None,
158 tester: str | None = None,
159 host_agent: str | None = None,
160 transport: str | None = None,
161 profile: str | None = None,
162 jury_mode: str | None = None,
163 jury_panel: dict[str, Any] | None = None,
164 implement_mode: str | None = None,
165 implement_phases: list[dict[str, Any]] | None = None,
166 implement_loop: dict[str, Any] | None = None,
167 consent_status: str | None = None,
168 consent_scopes: list[str] | tuple[str, ...] | None = None,
169 run_controls: dict[str, Any] | None = None,
170 fix_attribution: dict[str, Any] | None = None,
171) -> dict[str, Any]:
172 """Build one deterministic consumer-neutral ship ledger record."""
173 return {
174 "schema_version": LEDGER_SCHEMA_VERSION,
175 "record_type": RECORD_TYPE_SHIP_RUN,
176 "command": command,
177 "run_id": run_id,
178 "target": target,
179 "issue": {"number": issue_number},
180 "pull_request": {"number": pr_number},
181 "git": {
182 "base_branch": base_branch,
183 "branch": branch,
184 "head_sha": head_sha,
185 },
186 # `None` means git could not be read, and stays distinct from `[]` all the way
187 # into the record: a consumer of the ledger (or of the closure comment rendered
188 # from it) must not read "we could not see the diff" as "the diff was empty" —
189 # that is the conflation this whole family of fixes exists to remove, and a
190 # record claiming TIER-3 with zero files is self-contradictory besides.
191 "changes": {
192 "file_count": None if changed_files is None else len(changed_files),
193 "files": None if changed_files is None else list(changed_files),
194 "unreadable": changed_files is None,
195 },
196 "declared": _declared_block(declared_files),
197 "gates": [
198 {
199 "gate": outcome.gate,
200 "ok": outcome.ok,
201 "skipped": outcome.skipped,
202 "timed_out": outcome.timed_out,
203 # getattr: outcomes reach here from adapters and fixtures that predate
204 # these fields. Defaulting not_run to False keeps an older producer's
205 # record readable; on_fail defaults to the strict value so a record
206 # that *does* carry not_run without a severity fails closed.
207 "not_run": getattr(outcome, "not_run", False),
208 "on_fail": getattr(outcome, "on_fail", "block"),
209 "error": outcome.error,
210 "finding_count": len(outcome.findings),
211 # Which panel the jury gate's verdict is (#1437): one keel convened for
212 # this run, or the one already posted for this head and reused.
213 **_jury_source(outcome),
214 }
215 for outcome in outcomes
216 ],
217 "verdict": {
218 "blocked": verdict.blocked,
219 "counts": dict(verdict.counts),
220 },
221 "assessment": {
222 "tier": assessment.tier,
223 "reviewers": assessment.reviewers,
224 "window_open": assessment.window_open,
225 "ci_ok": assessment.ci_ok,
226 "merge": {
227 "action": assessment.merge.action,
228 "reason": assessment.merge.reason,
229 },
230 "halted": assessment.halted,
231 "bypassed_window": assessment.bypassed_window,
232 },
233 "actors": {
234 "implementer": implementer,
235 "reviewers": list(reviewer_agents or ()),
236 "tester": tester,
237 # Who took each s9 fix round, from the run-events file (#1016). An escalated
238 # round was not fixed by the implementer, and a closure comment rendered from
239 # `implementer` alone says it was.
240 "fixers": _fixers(fix_attribution),
241 "attribution_sentence": _attribution_sentence(fix_attribution),
242 },
243 "run_context": _run_context(
244 host_agent=host_agent,
245 transport=transport,
246 profile=profile,
247 jury_mode=jury_mode,
248 jury_panel=jury_panel,
249 implement_mode=implement_mode,
250 implement_phases=implement_phases,
251 implement_loop=implement_loop,
252 consent_status=consent_status,
253 consent_scopes=consent_scopes,
254 ),
255 "run_controls": run_controls,
256 "issue_intake": issue_intake,
257 "capture": capture.record_marker(
258 pr_number=pr_number,
259 status=capture_status,
260 reason=capture_reason,
261 artifact=capture_artifact,
262 retrieved=capture_retrieved,
263 title=issue_title,
264 labels=issue_labels,
265 # Not `changed_files`: `changes.files` above records what this run's git
266 # diff reported, which must stay None-preserving, while the capture
267 # fingerprint has to be the one the document on disk was written with.
268 changed_files=(
269 capture_changed_files if capture_changed_files is not None else changed_files
270 ),
271 existing_records=existing_records or [],
272 config=config,
273 not_run=capture_not_run,
274 ),
275 }
278def _fixers(fix_attribution: dict[str, Any] | None) -> list[dict[str, Any]]:
279 """The per-round fixer records from a ``keel.fix-attribution.v1`` document."""
280 rounds = fix_attribution.get("rounds") if isinstance(fix_attribution, dict) else None
281 if not isinstance(rounds, list):
282 return []
283 return [item for item in rounds if isinstance(item, dict)]
286def _attribution_sentence(fix_attribution: dict[str, Any] | None) -> str | None:
287 """The rendered *"implemented by agy, fixed by opus in round 2"* phrase, when recorded."""
288 if not isinstance(fix_attribution, dict):
289 return None
290 sentence = fix_attribution.get("sentence")
291 return sentence if isinstance(sentence, str) and sentence.strip() else None
294def _declared_block(declared_files: list[str] | None) -> dict[str, Any] | None:
295 """Build the implementer's declared-scope block, or ``None`` when unset.
297 ``declared_files`` is the implementer's contract of which files the change is
298 *supposed* to touch (distinct from the observed ``changes`` diff). When the
299 implementer does not declare a scope, the block is omitted so readers can
300 degrade to advisory back-compat behavior.
301 """
302 if declared_files is None:
303 return None
304 files = [str(path) for path in declared_files]
305 return {"file_count": len(files), "files": files}
308def declared_files_for_record(record: dict[str, Any]) -> list[str] | None:
309 """Return the implementer's declared file list from a ship_run ``record``.
311 Returns ``None`` when no declared scope was recorded (a missing or malformed
312 ``declared`` block), letting ``scope-verify`` degrade to an advisory pass.
313 """
314 declared = record.get("declared")
315 if not isinstance(declared, dict):
316 return None
317 files = declared.get("files")
318 if not isinstance(files, list):
319 return None
320 return [str(path) for path in files]
323def build_run_context(
324 *,
325 host_agent: str | None,
326 transport: str | None,
327 consent_status: str | None,
328 consent_scopes: list[str] | tuple[str, ...] | None,
329) -> dict[str, Any]:
330 """The ``run_context`` block :func:`build_ship_run_record` writes, for a writer that
331 builds its record another way — ``swarm-land``'s landing record (#1422).
333 The profile, the jury and the s4 fields are left unset: a landing knows none of them.
334 """
335 return _run_context(
336 host_agent=host_agent,
337 transport=transport,
338 profile=None,
339 jury_mode=None,
340 jury_panel=None,
341 consent_status=consent_status,
342 consent_scopes=consent_scopes,
343 )
346def _run_context(
347 *,
348 host_agent: str | None,
349 transport: str | None,
350 profile: str | None,
351 jury_mode: str | None,
352 jury_panel: dict[str, Any] | None,
353 consent_status: str | None,
354 consent_scopes: list[str] | tuple[str, ...] | None,
355 implement_mode: str | None = None,
356 implement_phases: list[dict[str, Any]] | None = None,
357 implement_loop: dict[str, Any] | None = None,
358) -> dict[str, Any]:
359 """Build the deterministic consumer-neutral preflight run-context block.
361 Every field is optional; a missing scalar degrades to ``None`` so the
362 closure renderer can present ``unknown``/``none`` without a schema change.
363 Consent is a small summary: a status and the approved mutation scopes,
364 reusing the operator/approve-scope inputs already resolved by the caller.
366 ``implement_mode``/``implement_phases`` are the s4 profile (#1020). A ``default`` run
367 records ``None`` and an empty phase list, so a record written before the knob existed
368 reads identically to one written by a run that did not use it.
369 """
370 scopes = [str(scope) for scope in (consent_scopes or ()) if str(scope).strip()]
371 return {
372 "host_agent": host_agent if _nonblank(host_agent) else None,
373 "transport": transport if _nonblank(transport) else None,
374 "profile": profile if _nonblank(profile) else None,
375 "jury_mode": jury_mode if _nonblank(jury_mode) else None,
376 # The panel-availability probe this run measured, or `None` when the tier
377 # never named a panel (#1066). Recorded so a reader of the ledger can tell
378 # a jury-reviewed change from one a host bench reviewed because the panel
379 # could not be staffed, without re-deriving it from a machine that has
380 # since changed.
381 "jury_panel": dict(jury_panel) if isinstance(jury_panel, dict) else None,
382 # The s4 profile and, under `tdd`, one record per phase — the ledger is where a
383 # closure comment and a later audit learn that this change was written test-first
384 # and which commit each half of s4 produced.
385 "implement_mode": implement_mode if _nonblank(implement_mode) else None,
386 "implement_phases": [dict(phase) for phase in implement_phases or ()],
387 # The s4 iteration loop (#1165): its policy and one record per iteration, or
388 # `None` for a run that neither configured nor recorded one.
389 "implement_loop": dict(implement_loop) if isinstance(implement_loop, dict) else None,
390 "consent": {
391 "status": consent_status if _nonblank(consent_status) else None,
392 "scopes": scopes,
393 },
394 }
397def _nonblank(value: Any) -> bool:
398 return isinstance(value, str) and bool(value.strip())
401def encode_record(record: dict[str, Any]) -> str:
402 """Encode one ledger record as stable JSONL."""
403 _validate_record(record)
404 return json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n"
407def parse_records(
408 text: str,
409 *,
410 warn: Callable[[str], None] | None = None,
411 kinds: Collection[str] = SHIP_RUN_RECORDS,
412) -> list[dict[str, Any]]:
413 """Parse ledger JSONL text into validated records of the kinds this keel knows.
415 **Every record of a known kind is validated; only those of ``kinds`` are returned**
416 — ship runs unless the reader asks for more (:data:`SHIP_RUN_RECORDS`). A reader that
417 wants a ``consent_delegation`` record (``consent-verify``) or every kind (``keel
418 ledger``) names it; every other reader is handed ship runs alone, as it always was.
420 **Forward compatible** (#1400): a well-formed record whose ``record_type`` names a
421 kind outside :data:`KNOWN_RECORD_TYPES` is skipped, never returned and never a
422 reason to refuse the ledger — so no reader can mistake it for a ship run, and a
423 kind a newer keel appends does not lock an older keel out of its own history.
424 ``warn`` is told once per skipped kind, with the line it was first seen on.
426 Everything else is refused exactly as before: invalid JSON, a non-object, a wrong
427 ``schema_version``, a record with no ``record_type`` at all (or one that is not a
428 non-blank string), and — through :func:`_validate_record` — any record of a known
429 kind. Only a record that *names* a kind can be set aside as one.
430 """
431 records: list[dict[str, Any]] = []
432 skipped: set[str] = set()
433 for line_number, raw in enumerate(text.splitlines(), start=1):
434 if not raw.strip():
435 continue
436 try:
437 record = json.loads(raw)
438 except json.JSONDecodeError as exc:
439 raise LedgerError(f"line {line_number}: invalid JSON") from exc
440 kind = _unknown_kind(record)
441 if kind is not None:
442 if warn is not None and kind not in skipped:
443 warn(unknown_kind_warning(kind, line_number))
444 skipped.add(kind)
445 continue
446 _validate_record(record, line_number=line_number)
447 if record["record_type"] in kinds:
448 records.append(record)
449 return records
452def unknown_kind_warning(kind: str, line_number: int) -> str:
453 """The one warning a reader gives for a record kind it skips."""
454 shown = kind if len(kind) <= _KIND_DISPLAY_LIMIT else kind[:_KIND_DISPLAY_LIMIT] + "..."
455 return (
456 f"run ledger line {line_number}: skipped record_type {shown!r}, which this keel "
457 "does not know (a newer keel may have written it); later records of that type "
458 "are skipped too"
459 )
462def read_records(
463 path: str | Path,
464 *,
465 warn: Callable[[str], None] | None = None,
466 kinds: Collection[str] = SHIP_RUN_RECORDS,
467) -> list[dict[str, Any]]:
468 """Read a ledger file; a missing ledger is a valid empty history.
470 Record kinds this keel does not know are skipped, and only ``kinds`` are returned, as
471 :func:`parse_records` does.
472 """
473 ledger_path = Path(path)
474 if not ledger_path.exists():
475 return []
476 return parse_records(ledger_path.read_text(encoding="utf-8"), warn=warn, kinds=kinds)
479def build_consent_delegation_record(
480 delegation: Mapping[str, Any],
481 *,
482 recorded_at: str,
483 pull_request: Mapping[str, Any] | None = None,
484) -> dict[str, Any]:
485 """A ``consent_delegation`` record for ``delegation`` (#1400).
487 ``delegation`` is :meth:`keel.swarm_worker.ConsentDelegation.to_dict`: who consented,
488 to which scopes, how, when, and for which run and clusters. Without ``pull_request``
489 the record is the ``delegated`` event, written before any worker starts; with it, the
490 ``pull_request`` event for one cluster — ``{"cluster", "number", "branch",
491 "head_sha", "url"}`` — written once that cluster's worker has opened its pull request.
493 Two events, never an update: the ledger is append-only, and the pull request number is
494 known only after ``gh pr create``. Each record carries the whole delegation, so a
495 reader needs one record to answer for a pull request. The parent's verbatim consent
496 record is not copied: every field a reader needs is named here.
497 """
498 record: dict[str, Any] = {
499 "schema_version": LEDGER_SCHEMA_VERSION,
500 "record_type": RECORD_TYPE_CONSENT_DELEGATION,
501 "event": "delegated" if pull_request is None else "pull_request",
502 "swarm_id": delegation.get("swarm_id"),
503 "clusters": list(delegation.get("clusters") or ()),
504 "scopes": list(delegation.get("scopes") or ()),
505 "operator": delegation.get("operator"),
506 "source": delegation.get("source"),
507 "mode": delegation.get("mode"),
508 "delegated_at": delegation.get("delegated_at"),
509 "recorded_at": recorded_at,
510 "pull_request": None if pull_request is None else dict(pull_request),
511 }
512 _validate_record(record)
513 return record
516def build_jury_followup_record(
517 *,
518 pr_number: int,
519 head_sha: str,
520 issue_number: int,
521 issue_url: str | None,
522 findings: list[str] | tuple[str, ...],
523 recorded_at: str,
524) -> dict[str, Any]:
525 """A ``jury_followup`` record: the issue filed for a COMMENT panel's minors (#1570)."""
526 record: dict[str, Any] = {
527 "schema_version": LEDGER_SCHEMA_VERSION,
528 "record_type": RECORD_TYPE_JURY_FOLLOWUP,
529 "pull_request": {"number": pr_number},
530 "head_sha": head_sha,
531 "issue": {"number": issue_number, "url": issue_url},
532 "findings": list(findings),
533 "recorded_at": recorded_at,
534 }
535 _validate_record(record)
536 return record
539def jury_followup_for_head(
540 records: list[dict[str, Any]],
541 pr_number: int,
542 head_sha: str,
543 *,
544 covered_heads: Collection[str] = (),
545) -> dict[str, Any] | None:
546 """Latest ``jury_followup`` record for ``pr_number`` at ``head_sha`` or ``covered_heads``."""
547 accepted = {head_sha, *covered_heads}
548 match: dict[str, Any] | None = None
549 for record in records:
550 if record.get("record_type") != RECORD_TYPE_JURY_FOLLOWUP:
551 continue
552 if (record.get("pull_request") or {}).get("number") == pr_number and record.get(
553 "head_sha"
554 ) in accepted:
555 match = record
556 return match
559def consent_delegation_for_pr(
560 records: list[dict[str, Any]], pr_number: int
561) -> dict[str, Any] | None:
562 """The latest ``pull_request`` delegation record naming ``pr_number``, or ``None``.
564 The pull request number is the only key: keel wrote it from ``gh pr create``'s answer,
565 so it names the pull request the worker opened. A branch name is not a key — anyone can
566 open a pull request from a fork with a ``swarm/<id>/<cluster>`` branch — and a reader
567 that gates on the record must still check the pull request's head against the record's
568 ``head_sha`` (:func:`keel.consentverify.consent_for_pr` does).
569 """
570 match: dict[str, Any] | None = None
571 for record in records:
572 if record.get("record_type") != RECORD_TYPE_CONSENT_DELEGATION:
573 continue
574 if (record.get("pull_request") or {}).get("number") == pr_number:
575 match = record
576 return match
579def latest_ship_run_for_pr(
580 records: list[dict[str, Any]],
581 pr_number: int,
582) -> dict[str, Any] | None:
583 """Return the last ship_run record whose pull_request matches ``pr_number``.
585 Records are appended in chronological order, so the last match is the most
586 recent ship run for that PR. Returns ``None`` when no record matches.
588 **Scoped by pull request, not by head, and a caller that gates on the record must
589 say so itself.** A pull request outlives its heads, so the record this returns may
590 have been written for a commit that is no longer the head — right for a reader
591 asking "what happened on this PR" (capture health, scope verification), wrong for
592 anything a merge decision hangs on. The two callers that gate check the head
593 themselves: :func:`gates_pass_for_head` selects on ``git.head_sha`` here, and
594 :func:`keel.juryavail.shipped` refuses a record whose head is not the one being
595 verified — without which an earlier head's fallback weakened the current head's
596 contract (#1068 round 2).
597 """
598 match: dict[str, Any] | None = None
599 for record in records:
600 if record.get("record_type") != RECORD_TYPE_SHIP_RUN:
601 continue
602 pull_request = record.get("pull_request")
603 number = pull_request.get("number") if isinstance(pull_request, dict) else None
604 if number == pr_number:
605 match = record
606 return match
609def _jury_source(outcome: Any) -> dict[str, Any]:
610 """The jury gate's provenance fields for a ledger gate entry (#1437).
612 ``source: reused`` with the posted comment it was read from, or ``source: ran`` for a
613 jury this run executed; nothing for any other gate, or for a jury nobody ran.
614 """
615 reused = getattr(outcome, "reused_from", None)
616 if reused is not None:
617 return {"source": "reused", "reused_from": reused.as_dict()}
618 if outcome.gate == "jury" and not getattr(outcome, "not_run", False):
619 return {"source": "ran"}
620 return {}
623def record_gates_passed(record: dict[str, Any]) -> bool:
624 """Return whether a ship_run record's gates count as a clean pass.
626 A pass requires that the run was not blocked by findings and that every
627 recorded gate either ran clean (``ok``) or was deliberately skipped, with no
628 gate reporting an error. A missing or malformed ``gates``/``verdict`` block
629 degrades to "not a pass" so a corrupt record can never authorize a merge.
631 A gate marked ``not_run`` was never executed by the runner that wrote the
632 record — an ``agentic`` gate reaching the command-only runner. For a
633 ``block``-severity gate that is **not** a pass: certifying "gates passed" for a
634 blocking review nobody ran is exactly the fail-open this check exists to stop.
635 Soft (``warn``/``suggest``) not-run gates are tolerated, matching ``skipped``.
636 Records written before this field existed have no ``not_run`` key and are
637 unaffected.
638 """
639 verdict = record.get("verdict")
640 if not isinstance(verdict, dict) or verdict.get("blocked") is not False:
641 return False
642 gates = record.get("gates")
643 if not isinstance(gates, list) or not gates:
644 return False
645 for gate in gates:
646 if not isinstance(gate, dict):
647 return False
648 if gate.get("error"):
649 return False
650 if not (gate.get("ok") is True or gate.get("skipped") is True):
651 return False
652 # Missing `on_fail` defaults to the strict value *here*, at read time. Defaulting
653 # only at write time protects records keel wrote and no others: any producer that
654 # learns `not_run` without its sibling key would otherwise fail open in exactly
655 # the certification path this check exists to close.
656 # Fail closed on anything not explicitly soft: a missing key, a JSON-round-tripped
657 # `None`, or a severity name keel does not know all mean "we cannot tell this was
658 # optional", and this is the certification path.
659 if gate.get("not_run") is True and gate.get("on_fail") not in ("warn", "suggest"):
660 return False
661 return True
664def gates_pass_for_head(
665 records: list[dict[str, Any]],
666 pr_number: int,
667 head_sha: str,
668 covered_heads: Collection[str] = (),
669) -> tuple[bool, dict[str, Any] | None]:
670 """Find a passing gates run recorded against ``head_sha`` for ``pr_number``.
672 Returns ``(matched, record)``. ``matched`` is ``True`` only when the **latest**
673 ship_run record for the PR carrying the exact current ``head_sha`` passed its
674 gates (see :func:`record_gates_passed`); the record is returned on a match and
675 ``None`` otherwise. A blank ``head_sha`` never matches — an unknown head must
676 not be authorized by a stale green run. This is a pure function: it reads only
677 its arguments and performs no I/O.
679 Latest-wins, not any-pass. Re-gating the same head is ordinary — a flaky suite
680 settles, a fix-loop re-runs, a dependency lands — so the same ``head_sha`` can
681 carry a green record followed by a red one. Scanning for *any* green would let
682 the superseded pass authorize the merge and the later red never be consulted:
683 a fail-open in exactly the gate that exists to hold the merge closed. Only the
684 most recent verdict for the head counts.
686 ``covered_heads`` are heads the current one descends from by capture commits alone
687 (#1203): the learning lands on the pull request's own branch after the gates ran, so
688 the gates-pass is recorded against a head the branch has moved one lesson past. A
689 record for one of them counts, and **latest-wins still holds across the whole set** —
690 a red record for any of them after a green one is the verdict. The set is produced by
691 `capture.capture_only_descent` and nothing else, so it admits a markdown file inside
692 the sink, never code; CI still runs on the new head and `keel merge` still requires
693 that rollup to be green.
694 """
695 if not isinstance(head_sha, str) or not head_sha.strip():
696 return False, None
697 accepted = {head_sha, *covered_heads}
698 latest: dict[str, Any] | None = None
699 for record in records:
700 if record.get("record_type") != RECORD_TYPE_SHIP_RUN:
701 continue
702 pull_request = record.get("pull_request")
703 number = pull_request.get("number") if isinstance(pull_request, dict) else None
704 if number != pr_number:
705 continue
706 git = record.get("git")
707 record_sha = git.get("head_sha") if isinstance(git, dict) else None
708 if record_sha not in accepted:
709 continue
710 latest = record
711 if latest is None or not record_gates_passed(latest):
712 return False, None
713 return True, latest
716def _latest_per_pr(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
717 """One record per pull request — the last — keeping records that name none.
719 A pull request has one capture, and since #1157 it can leave more than one
720 marked record: the writer's clash is keyed by ``(pull request, head)``, so a
721 superseded head's marker survives beside the merged head's. Counting rows
722 rather than pull requests then reported one merged pull request twice —
723 ``applied: 1, skipped: 1`` for a single capture — in morning, wrap and status.
724 Insertion order is preserved so these readers still list captures in the order
725 the ledger recorded them.
726 """
727 latest: dict[int, dict[str, Any]] = {}
728 unkeyed: list[dict[str, Any]] = []
729 for record in records:
730 pull_request = record.get("pull_request")
731 number = pull_request.get("number") if isinstance(pull_request, dict) else None
732 if isinstance(number, int):
733 latest[number] = record
734 else:
735 unkeyed.append(record)
736 # Identity, not equality: two equal rows are still two rows. A set of id()s makes
737 # the filter one pass; `any(record is kept for kept in keep)` was records × kept,
738 # and the ledger is append-only, so that product only grows (#1355). Every id is
739 # of an object `records` still holds, so none can be reused mid-comprehension.
740 kept_ids = {id(kept) for kept in [*latest.values(), *unkeyed]}
741 return [record for record in records if id(record) in kept_ids]
744def capture_health_summary(records: list[dict[str, Any]]) -> dict[str, Any]:
745 """Summarize capture visibility for morning, wrap, status, and ledger readers."""
746 merged_records = _latest_per_pr([r for r in records if _is_merged_ship_run(r)])
747 items = [_capture_health_item(record) for record in merged_records]
748 counts = {
749 "applied": 0,
750 "marker_only": 0,
751 "create_learning": 0,
752 "duplicate_learning": 0,
753 "deferred": 0,
754 "skipped": 0,
755 "missing_marker": 0,
756 "needs_reconcile": 0,
757 }
758 skipped_by_reason: dict[str, int] = {}
759 for item in items:
760 status = item["status"]
761 learning = item["learning_decision"]
762 if status == "applied":
763 counts["applied"] += 1
764 elif status == "deferred":
765 counts["deferred"] += 1
766 elif status == "skipped":
767 counts["skipped"] += 1
768 skipped_by_reason[item["reason"] or "unspecified"] = (
769 skipped_by_reason.get(item["reason"] or "unspecified", 0) + 1
770 )
771 else:
772 counts["missing_marker"] += 1
773 if learning == "marker-only":
774 counts["marker_only"] += 1
775 elif learning == "create-learning":
776 counts["create_learning"] += 1
777 elif learning == "duplicate":
778 counts["duplicate_learning"] += 1
779 if item["needs_reconcile"]:
780 counts["needs_reconcile"] += 1
781 return {
782 "schema_version": CAPTURE_HEALTH_SCHEMA_VERSION,
783 "status": "needs-reconcile" if counts["needs_reconcile"] else "clean",
784 "record_count": len(merged_records),
785 "counts": counts,
786 "skipped_by_reason": dict(sorted(skipped_by_reason.items())),
787 "items": items,
788 "reconcile_actions": [action for item in items for action in item["reconcile_actions"]],
789 "dry_run": {
790 "no_mutations": True,
791 "description": "Morning and wrap surface these actions; they do not mutate "
792 "ledger, GitHub, or capture destinations.",
793 },
794 }
797def _is_merged_ship_run(record: Any) -> bool:
798 if not isinstance(record, dict):
799 return False
800 if record.get("record_type") != RECORD_TYPE_SHIP_RUN:
801 return False
802 capture_block = record.get("capture")
803 # The operator declared this run never reached capture, which settles the
804 # question before the assessment is consulted: that block records what ship
805 # *recommended*, and a recommendation to merge is read here as proof the PR
806 # merged. For a run re-recording gates after a rebase both are true at once —
807 # gates green, nothing merged — and the record would otherwise be counted as a
808 # merged PR whose marker went missing, a capture gap that is not there (#945).
809 # Only an explicit declaration does this; a plain ``status: None`` still falls
810 # through, so a marker that genuinely went missing is still reported.
811 if isinstance(capture_block, dict) and capture_block.get("not_run") is True:
812 return False
813 assessment = record.get("assessment")
814 if isinstance(assessment, dict):
815 merge = assessment.get("merge")
816 if isinstance(merge, dict) and merge.get("action") == "merge":
817 return True
818 if isinstance(capture_block, dict) and capture_block.get("status") in (
819 "applied",
820 "deferred",
821 "skipped",
822 ):
823 return True
824 return False
827def _capture_marker(record: dict[str, Any]) -> str | None:
828 capture = record.get("capture")
829 marker = capture.get("marker") if isinstance(capture, dict) else None
830 return marker if isinstance(marker, str) and marker.strip() else None
833def record_head_sha(record: Mapping[str, Any]) -> str | None:
834 """The head a ship_run record was written for, or ``None`` when it names none."""
835 git = record.get("git")
836 head = git.get("head_sha") if isinstance(git, Mapping) else None
837 return head.strip() if isinstance(head, str) and head.strip() else None
840def capture_marker_for_head(
841 records: list[dict[str, Any]],
842 *,
843 pr_number: int | None,
844 head_sha: str | None,
845) -> dict[str, Any] | None:
846 """The recorded capture marker for this ``(pull request, head)``, if any.
848 The same rule :func:`existing_capture_marker` enforces, asked **before** a
849 record exists. The clash is keyed on the pair and nothing else, so a caller
850 about to do durable work for this run can find out whether its append will
851 land — a learning file written ahead of an append that then no-ops is a
852 document on disk that no ledger record will ever point at.
853 """
854 if not isinstance(pr_number, int):
855 return None
856 head = head_sha.strip() if isinstance(head_sha, str) and head_sha.strip() else None
857 for existing in records:
858 if existing.get("record_type") != RECORD_TYPE_SHIP_RUN:
859 continue
860 other = existing.get("pull_request")
861 if (other.get("number") if isinstance(other, dict) else None) != pr_number:
862 continue
863 if record_head_sha(existing) != head:
864 continue
865 if _capture_marker(existing) is not None:
866 return existing
867 return None
870def existing_capture_marker(
871 records: list[dict[str, Any]], record: dict[str, Any]
872) -> dict[str, Any] | None:
873 """The already-recorded capture marker ``record`` would duplicate, if any.
875 One capture marker per **(pull request, head)**, enforced at write time. It was
876 once only *detected*, and only afterwards: :func:`keel.capture.verify_session`
877 refuses the whole session on a second one ("multiple capture markers found for
878 merged PR"), ``capture-reconcile`` returns ``blocked`` with no actions to offer,
879 and nothing in this module can remove a line — so the recovery was editing the
880 ledger by hand, which is forging audit history to make a gate pass.
882 Re-running the same append is the most natural thing to do after a crash mid-s11,
883 which made the obvious recovery the very action that bricks the run. Checking here
884 costs one pass over records the caller already holds. Returns the conflicting
885 record so the caller can name it; ``None`` when the append is new.
887 **Scoped to the head, because that is how the merge gate reads it** (#1157). Keyed
888 by pull request alone, this refused every later head once *any* record carried a
889 marker — including the very first run, red. :func:`gates_pass_for_head` and
890 :func:`keel.juryavail.is_ship_run_for_head` then asked for a passing record on the
891 *current* head, which no run was any longer allowed to write. A pull request whose
892 first ship run failed became permanently unmergeable, and the documented exit was
893 editing an append-only audit ledger to make a gate pass, which is the one action
894 this design exists to prevent. The writer now keys the record the way every reader
895 that gates does: retrying the same head still clashes, a new head is a new run.
897 A record naming no head is compared to other records naming no head — the same
898 rule, applied to the value they have, rather than an exemption from it.
899 """
900 if _capture_marker(record) is None:
901 return None
902 pull_request = record.get("pull_request")
903 pr = pull_request.get("number") if isinstance(pull_request, dict) else None
904 if not isinstance(pr, int):
905 return None
906 return capture_marker_for_head(records, pr_number=pr, head_sha=record_head_sha(record))
909def append_record(path: str | Path, record: dict[str, Any]) -> None:
910 """Append one validated JSONL record, creating parent directories as needed."""
911 ledger_path = Path(path)
912 ledger_path.parent.mkdir(parents=True, exist_ok=True)
913 workspace.ensure_runtime_gitignore_for(ledger_path)
914 with ledger_path.open("a", encoding="utf-8") as handle:
915 handle.write(encode_record(record))
918def sanitize_record(
919 record: dict[str, Any],
920 config: cfg.ProjectConfig | None = None,
921) -> dict[str, Any]:
922 """Apply capture redaction before a ledger record becomes durable."""
923 result = redaction.sanitize(record, redaction.policy_from_config(config))
924 sanitized = dict(result.value)
925 sanitized["redaction"] = result.audit
926 return sanitized
929def _unknown_kind(record: Any) -> str | None:
930 """The ``record_type`` of a record of a kind this keel does not know, else ``None``.
932 Only a record that is otherwise in this ledger's schema and *names* its kind
933 qualifies: a missing, blank or non-string ``record_type`` is not a kind to skip but
934 a malformed record, and :func:`_validate_record` refuses it as it always has.
935 """
936 if not isinstance(record, dict) or record.get("schema_version") != LEDGER_SCHEMA_VERSION:
937 return None
938 kind = record.get("record_type")
939 if not isinstance(kind, str) or not kind.strip() or kind in KNOWN_RECORD_TYPES:
940 return None
941 return kind
944def _validate_record(record: Any, *, line_number: int | None = None) -> None:
945 prefix = f"line {line_number}: " if line_number is not None else ""
946 if not isinstance(record, dict):
947 raise LedgerError(f"{prefix}record must be an object")
948 if record.get("schema_version") != LEDGER_SCHEMA_VERSION:
949 raise LedgerError(f"{prefix}unsupported schema_version")
950 kind = record.get("record_type")
951 if kind == RECORD_TYPE_CONSENT_DELEGATION:
952 if problem := _consent_delegation_problem(record):
953 raise LedgerError(f"{prefix}invalid consent_delegation record: {problem}")
954 elif kind == RECORD_TYPE_JURY_FOLLOWUP:
955 if problem := _jury_followup_problem(record):
956 raise LedgerError(f"{prefix}invalid jury_followup record: {problem}")
957 elif kind != RECORD_TYPE_SHIP_RUN:
958 raise LedgerError(f"{prefix}unsupported record_type")
961def _positive_int(value: Any) -> bool:
962 return isinstance(value, int) and not isinstance(value, bool) and value > 0
965def _jury_followup_problem(record: dict[str, Any]) -> str:
966 """What is wrong with a ``jury_followup`` record, or ``""`` when nothing is."""
967 if not _positive_int((record.get("pull_request") or {}).get("number")):
968 return "pull_request.number must be a positive integer"
969 for key in ("head_sha", "recorded_at"):
970 if not _nonblank(record.get(key)):
971 return f"{key} must be a non-blank string"
972 if not _positive_int((record.get("issue") or {}).get("number")):
973 return "issue.number must be a positive integer"
974 findings = record.get("findings")
975 if not isinstance(findings, list) or not findings or not all(_nonblank(f) for f in findings):
976 return "findings must be a non-empty list of non-blank strings"
977 return ""
980def _consent_delegation_problem(record: dict[str, Any]) -> str:
981 """What is wrong with a ``consent_delegation`` record, or ``""`` when nothing is.
983 Strict about every field a reader relies on; a field this keel does not name is
984 tolerated, so a later keel can add one without making this one refuse the ledger.
985 """
986 event = record.get("event")
987 if event not in CONSENT_DELEGATION_EVENTS:
988 return f"event must be one of {', '.join(CONSENT_DELEGATION_EVENTS)}"
989 for key in ("swarm_id", "operator", "source", "mode", "delegated_at", "recorded_at"):
990 if not _nonblank(record.get(key)):
991 return f"{key} must be a non-blank string"
992 clusters = record.get("clusters")
993 if (
994 not isinstance(clusters, list)
995 or not clusters
996 or not all(_nonblank(c) for c in clusters)
997 or len(set(clusters)) != len(clusters)
998 ):
999 return "clusters must be a non-empty list of distinct non-blank strings"
1000 scopes = record.get("scopes")
1001 if not isinstance(scopes, list) or not scopes or not all(_nonblank(s) for s in scopes):
1002 return "scopes must be a non-empty list of non-blank strings"
1003 try:
1004 normalized = list(consent.normalize_scopes(scopes))
1005 except ValueError as exc:
1006 return str(exc)
1007 if normalized != scopes:
1008 return f"scopes must be normalised: {normalized}"
1009 return _delegated_pull_request_problem(event, record.get("pull_request"), clusters)
1012def _delegated_pull_request_problem(event: Any, pull_request: Any, clusters: list[Any]) -> str:
1013 """What is wrong with a delegation record's ``pull_request`` for its ``event``."""
1014 if event == "delegated":
1015 return "" if pull_request is None else "a delegated event names no pull request"
1016 if not isinstance(pull_request, dict):
1017 return "a pull_request event names its pull request"
1018 if pull_request.get("cluster") not in clusters:
1019 return "pull_request.cluster must be one of the delegation's clusters"
1020 number = pull_request.get("number")
1021 if isinstance(number, bool) or not isinstance(number, int) or number < 1:
1022 return "pull_request.number must be a positive integer"
1023 for key in ("branch", "head_sha"):
1024 if not _nonblank(pull_request.get(key)):
1025 return f"pull_request.{key} must be a non-blank string"
1026 url = pull_request.get("url")
1027 if url is not None and not isinstance(url, str):
1028 return "pull_request.url must be a string"
1029 return ""
1032def _capture_health_item(record: dict[str, Any]) -> dict[str, Any]:
1033 block = record.get("capture") if isinstance(record.get("capture"), dict) else {}
1034 status = block.get("status")
1035 marker = block.get("marker")
1036 reason = block.get("marker_reason") or block.get("reason")
1037 learning = block.get("learning") if isinstance(block.get("learning"), dict) else {}
1038 item_status = _capture_health_status(status, marker)
1039 needs_reconcile = item_status in {"missing-marker", "deferred"}
1040 pr_number = (record.get("pull_request") or {}).get("number")
1041 item = {
1042 "run_id": record.get("run_id"),
1043 "issue": (record.get("issue") or {}).get("number"),
1044 "pull_request": pr_number,
1045 "status": item_status,
1046 "capture_status": status,
1047 "marker": marker if isinstance(marker, str) and marker.strip() else None,
1048 "reason": reason if isinstance(reason, str) and reason.strip() else None,
1049 "learning_decision": learning.get("decision"),
1050 "learning_reason": learning.get("reason"),
1051 "needs_reconcile": needs_reconcile,
1052 "reconcile_actions": [],
1053 }
1054 if needs_reconcile:
1055 item["reconcile_actions"].append(_capture_reconcile_action(pr_number, item_status))
1056 return item
1059def _capture_health_status(status: Any, marker: Any) -> str:
1060 if not isinstance(marker, str) or not marker.strip():
1061 return "missing-marker"
1062 if status == "deferred":
1063 return "deferred"
1064 if status == "skipped":
1065 return "skipped"
1066 return "applied"
1069def _capture_reconcile_action(pr_number: Any, status: str) -> dict[str, Any]:
1070 return {
1071 "type": "capture-reconcile",
1072 "reason": status,
1073 "pr": pr_number if isinstance(pr_number, int) else None,
1074 "command": (
1075 f"keel capture-reconcile .keel/project.yaml --root . --merged-pr {pr_number}"
1076 if isinstance(pr_number, int)
1077 else "keel capture-reconcile .keel/project.yaml --root ."
1078 ),
1079 "dry_run": True,
1080 }