Coverage for src/keel/evidence.py: 100%
768 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
1"""Deterministic pre-merge evidence verification.
3The ship adapter is agentic, but the artifacts it must leave behind are not:
4reviewer verdict comments/reviews, the jury verdict a gating jury requires, and the stable
5closure comment marker. This module keeps the check pure so CI can enforce it
6without trusting prose in an agent prompt.
8Classification is **header-anchored**: :func:`marker_in_header` decides what a
9comment is from its first non-empty line and nothing else, so a marker a reviewer
10quotes in prose is content rather than a classification signal (#1026). The ship
11assessment heading is anchored the same way, from its own header line (#1035).
12"""
14from __future__ import annotations
16import hashlib
17import re
18import shlex
19from collections.abc import Collection, Sequence
20from dataclasses import dataclass
21from typing import Any
23from . import agents, closure, juryavail
24from . import team as team_policy
26SCHEMA_VERSION = "keel.evidence.v1"
27#: ``reviewers.panel`` when the cross-vendor jury *is* the review for this tier
28#: (``knobs.team``'s ``review.by_tier.<n>: jury``), and the minimum distinct
29#: vendors such a panel must span. Both come from :mod:`keel.team`, the leaf
30#: module that owns the team vocabulary, so the gate and the policy cannot drift.
31JURY_PANEL = team_policy.JURY_PANEL
32DEFAULT_MINIMUM_JURY_VENDORS = team_policy.DEFAULT_MIN_VENDORS
33AGENT_LABEL_PREFIX = "agent:"
34MODEL_LABEL_PREFIX = "model:"
35REVIEW_VERDICT_MARKER = "keel.review-verdict.v1"
36#: The ``Verdict:`` tokens that **approve** a change (#1426). A review verdict counts
37#: toward ``review-verdict-N`` only when the first word of its ``Verdict:`` line is one
38#: of these, read case-insensitively by :func:`review_verdict_token`. The set is what
39#: keel and its hosts actually write, not a guess:
40#:
41#: * ``LGTM`` — the default of :func:`keel.artifacts.render_review_verdict`, what
42#: ``contracts.py``'s ``review_verdict_template`` renders for an unblocked change, and
43#: what :func:`keel.jury.map_verdict` folds ai-jury's ``APPROVE`` / ``READY`` into
44#: before ``keel review --from-jury`` posts a ballot;
45#: * ``APPROVE`` — what the host reviewers post today: every verdict on the keel PRs
46#: from #1405 onward, 631 of the 1,355 verdict comments on the repository;
47#: * ``PASS`` — 168 verdicts posted on #918–#992, written by hosts as ``Verdict: pass``.
48#:
49#: Everything else holds the merge: ``REQUEST_CHANGES`` (keel's own blocked-change
50#: template, and the jury's ``NEEDS_INFO``), ``COMMENT``, ``ABSTAIN``, a token keel does
51#: not know, and a verdict with no ``Verdict:`` line at all. Inventing an approval for a
52#: stance keel does not recognise is the mapping error that cannot be undone — the same
53#: rule :data:`keel.jury._VERDICT` states for ballots.
54APPROVING_VERDICTS = frozenset({"APPROVE", "LGTM", "PASS"})
55#: The non-approving tokens a refusal names as *requesting changes*; any other
56#: non-approving token is named by itself (``alice does not approve at <head> (verdict
57#: COMMENT)``).
58REQUEST_CHANGES_VERDICTS = frozenset({"REQUEST_CHANGES", "CHANGES_REQUESTED", "NEEDS_INFO"})
59#: The finding a non-approving review verdict at the current head raises (#1426).
60VERDICT_NOT_APPROVED_FINDING = "review-verdict-not-approved"
61#: The finding a non-approving jury verdict at the current head raises (#1429).
62JURY_NOT_APPROVED_FINDING = "jury-verdict-not-approved"
63#: The one consensus the opt-in ``team.jury.comment_consensus`` rule can accept (#1570).
64COMMENT_CONSENSUS = "COMMENT"
65#: Findings-summary labels that block under the COMMENT-consensus rule, the non-blocking
66#: labels it accepts, and the label it leaves out of the follow-up issue (advisory, never
67#: gates — ``findings.py``, #1570).
68_COMMENT_BLOCKING_LABELS = frozenset({"critical", "major"})
69_COMMENT_NON_BLOCKING_LABELS = frozenset({"minor", "nit"})
70_COMMENT_ADVISORY_LABEL = "nit"
71JURY_VERDICT_MARKER = "keel.jury-verdict.v1"
72#: The comment a live ship run posts on its PR right after creating it (#1013). It is
73#: the *primary* arming signal for the evidence gate: unlike the branch name it is
74#: written by the run itself, so a run that named its branch something else — or whose
75#: ledger lives in a per-run worktree CI cannot read — still identifies as a keel ship
76#: run. See :func:`gate_decision` for the full arming order.
77SHIP_PROVENANCE_MARKER = "keel.ship-provenance.v1"
78#: The marker the ship adapter tells a host to post when a finding is deferred rather
79#: than fixed. keel core never counts a deferral as evidence; it is listed among the
80#: classification markers so a deferral comment reads as *one* artifact instead of
81#: being classified by whichever marker its prose happens to name.
82DEFERRAL_MARKER = "keel.deferral.v1"
83SHIP_ASSESSMENT_HEADING = "### \U0001f6a2 keel ship"
84#: The banner the ``keel ship`` CLI prints above its own summary. An assessment pasted
85#: raw — without the workflow's Markdown heading — leads with this line instead, so both
86#: forms identify the comment. See :func:`_is_ship_assessment`.
87SHIP_ASSESSMENT_BANNER = "keel ship \u2014"
88DEFAULT_WAIVER_LABEL = "keel:evidence-waived"
89TRUSTED_AUTHOR_ASSOCIATIONS = frozenset({"OWNER", "MEMBER", "COLLABORATOR"})
90TRUSTED_SHIP_ASSESSMENT_BOTS = frozenset({"github-actions", "github-actions[bot]"})
92#: Every marker that *classifies* an evidence comment. Order is the order findings
93#: report them in, so a malformed-header message is byte-stable.
94CLASSIFICATION_MARKERS: tuple[str, ...] = (
95 REVIEW_VERDICT_MARKER,
96 JURY_VERDICT_MARKER,
97 SHIP_PROVENANCE_MARKER,
98 closure.CLOSURE_SCHEMA_VERSION,
99 DEFERRAL_MARKER,
100)
102#: Membership view of :data:`CLASSIFICATION_MARKERS`, which is a tuple because its **order**
103#: is the order markers are rendered in. Asking a tuple `x in ...` is a linear scan, and the
104#: two places below only ask whether every token is a known marker — a set answers that
105#: question by its type rather than by walking. The tuple stays where order matters.
106_CLASSIFICATION_MARKERS_SET = frozenset(CLASSIFICATION_MARKERS)
108#: The finding raised for a comment whose header names more than one marker.
109MALFORMED_MARKER_FINDING = "malformed-evidence-comment"
111#: The header fields keel reads off an evidence comment. Closed by design: an
112#: unlisted ``key: value`` line is prose, and prose ends the header block (#932).
113#: ``panelists`` joins ``vendors`` as a jury-verdict field, because the size of a
114#: panel that *is* the review sets the required verdict count (#1015).
115_FIELD_RE = re.compile(
116 r"^\s*(?P<key>reviewer|head|vendor|model|vendors|panelists)\s*:\s*(?P<value>\S+)\s*$",
117 re.IGNORECASE,
118)
119_HEADER_LINE_RE = re.compile(r"^[A-Za-z0-9_-]+\s*:")
120#: A ``Verdict:`` line, as :func:`keel.artifacts.render_review_verdict` writes it. Anchored
121#: at the line start, so ``AI Jury verdict:`` and a quoted ``> Verdict:`` are not one.
122_VERDICT_LINE_RE = re.compile(r"^\s*verdict\s*:(?P<value>.*)$", re.IGNORECASE)
123#: The first word of a verdict line's value: wrapper punctuation (``**``, a backtick, an
124#: emoji) is skipped, and the word ends at the first character that is not a letter,
125#: ``_`` or ``-`` — so ``APPROVE — minor nits`` and ``APPROVE, minor nits`` both read
126#: ``APPROVE``.
127_VERDICT_TOKEN_RE = re.compile(r"^[\W_]*(?P<token>[A-Za-z][A-Za-z_-]*)")
128#: A jury verdict's consensus line, as :func:`keel.artifacts.render_jury_verdict` writes it:
129#: ``AI Jury verdict: REQUEST_CHANGES.`` — the trailing full stop ends the token like any
130#: other punctuation does (#1429).
131_JURY_VERDICT_LINE_RE = re.compile(r"^\s*AI\s+Jury\s+verdict\s*:(?P<value>.*)$", re.IGNORECASE)
132_SHIP_BRANCH_RE = re.compile(r"^(feature|fix|chore|docs|test)/issue-\d+(?:-|$)")
133#: The exact wrapper a marker line may wear. Every keel renderer emits its marker as
134#: the whole first line, in one of exactly two shapes: bare
135#: (``artifacts.render_review_verdict`` / ``render_jury_verdict`` /
136#: ``render_ship_provenance``) or wrapped in an HTML comment so it renders invisibly
137#: (``closure.render_closure_comment``).
138#:
139#: These are matched literally, never with a regex. A pattern that treats ``-->`` as
140#: *the* comment terminator is wrong about HTML — a browser also ends a comment at
141#: ``--!>`` — and a classifier that disagrees with the renderer about where a comment
142#: ends is exactly the confusion this module exists to remove (CodeQL
143#: ``py/bad-tag-filter``). keel does not need to parse HTML: it needs to recognise the
144#: one shape it writes, and refuse everything else.
145_HTML_COMMENT_OPEN = "<!--"
146_HTML_COMMENT_CLOSE = "-->"
148STATUS_PASS = "pass"
149#: The finding severity that fails verification on its own, with nothing missing.
150BLOCKING_FINDING_SEVERITY = "major"
151STATUS_WAITING = "waiting"
152STATUS_FAIL = "fail"
153STATUSES = (STATUS_PASS, STATUS_WAITING, STATUS_FAIL)
155# Evidence phases. An artifact is required in the phase that produces it, mirroring
156# the step mapping stepverifier already applies (review -> s7, jury -> s8, closure ->
157# s12). The merge gate at s10 asks for PHASE_PRE_MERGE, because the closure comment
158# is a post-merge record and requiring it at s10 makes the backbone unsatisfiable.
159PHASE_PRE_MERGE = "pre-merge"
160PHASE_POST_MERGE = "post-merge"
161PHASE_ALL = "all"
162PHASES = (PHASE_PRE_MERGE, PHASE_POST_MERGE, PHASE_ALL)
165@dataclass(frozen=True)
166class EvidenceItem:
167 id: str
168 kind: str
169 required: bool
170 description: str
171 phase: str = PHASE_PRE_MERGE
173 def as_dict(self) -> dict[str, Any]:
174 return {
175 "id": self.id,
176 "kind": self.kind,
177 "required": self.required,
178 "description": self.description,
179 "phase": self.phase,
180 }
183def gate_active(labels: Sequence[str] | None, gate_label: str) -> bool:
184 """Return whether ``gate_label`` is present in ``labels`` (None/empty -> False).
186 An empty ``gate_label`` is never active, so a misconfigured (blank) label can
187 never silently match — the schema also forbids an empty ``evidence_gate_label``.
188 """
189 if not gate_label:
190 return False
191 return gate_label in set(labels or ())
194def gate_decision(
195 labels: Sequence[str] | None,
196 gate_label: str,
197 *,
198 waiver_label: str = DEFAULT_WAIVER_LABEL,
199 head_ref: str | None = None,
200 pr_comments: list[dict[str, Any]] | None = None,
201 pr_reviews: list[dict[str, Any]] | None = None,
202 ledger_records: Sequence[object] | None = None,
203) -> dict[str, Any]:
204 """Return the fail-closed evidence-gate arming decision.
206 Ship provenance arms the gate by default. The only disarm path is an explicit
207 waiver label applied by an operator; the legacy gate label remains an
208 additional arming signal for already-installed workflows.
210 The signals are consulted in this order, and the order is part of the contract
211 (documented in ``docs/keel/evidence.md``):
213 1. ``operator-waiver-label`` — the one sanctioned disarm, checked first so an
214 explicit waiver is never shadowed by an arming signal.
215 2. ``gate-label`` — the legacy opt-in label.
216 3. ``ship-provenance-comment`` — a trusted PR comment carrying
217 :data:`SHIP_PROVENANCE_MARKER`, which a live ship run posts as soon as the PR
218 exists. **Ahead of the branch regex on purpose** (#1013): the marker is written
219 by the run, the branch name is written by whoever typed it, and a ship run that
220 named its branch ``fix/2467-slug`` used to read as a non-keel PR.
221 4. ``ship-branch`` — the legacy branch-name fallback for runs that predate the
222 marker. Kept, but it is no longer the signal keel relies on.
223 5. ``ship-assessment-comment`` / ``review-verdict-marker`` / ``ship-run-ledger`` —
224 the remaining after-the-fact traces, unchanged.
226 Nothing was removed: every path that armed the gate before still arms it.
227 """
228 label_set = set(labels or ())
229 if waiver_label and waiver_label in label_set:
230 return _gate_decision(False, "operator-waiver-label", waiver_label, waived=True)
231 if gate_active(labels, gate_label):
232 return _gate_decision(True, "gate-label", gate_label)
233 if _has_trusted_ship_provenance(pr_comments or []):
234 return _gate_decision(True, "ship-provenance-comment", SHIP_PROVENANCE_MARKER)
235 if head_ref and _SHIP_BRANCH_RE.search(head_ref):
236 return _gate_decision(True, "ship-branch", head_ref)
237 if _has_trusted_ship_assessment(pr_comments or []):
238 return _gate_decision(True, "ship-assessment-comment", SHIP_ASSESSMENT_HEADING)
239 if _has_trusted_review_marker([*(pr_comments or []), *(pr_reviews or [])]):
240 return _gate_decision(True, "review-verdict-marker", REVIEW_VERDICT_MARKER)
241 if ledger_records:
242 return _gate_decision(True, "ship-run-ledger", "ship_run")
243 return _gate_decision(False, "no-ship-provenance", None)
246def _gate_decision(
247 enforced: bool,
248 reason: str,
249 source: str | None,
250 *,
251 waived: bool = False,
252) -> dict[str, Any]:
253 return {
254 "schema_version": SCHEMA_VERSION,
255 "enforced": enforced,
256 "waived": waived,
257 "reason": reason,
258 "source": source,
259 }
262def _has_trusted_ship_provenance(items: list[dict[str, Any]]) -> bool:
263 """True when a trusted PR comment carries the ship-provenance marker.
265 Trust is the same fail-closed ``author_association`` check every other evidence
266 source uses: an anonymous drive-by comment must not be able to arm — or, more to
267 the point, to *look* like it armed — the gate.
268 """
269 return any(
270 _is_trusted_source(item, enforced=True)
271 and marker_in_header(_body(item)) == SHIP_PROVENANCE_MARKER
272 for item in items
273 )
276def _has_trusted_ship_assessment(items: list[dict[str, Any]]) -> bool:
277 return any(
278 _is_ship_assessment_source(item) and _is_ship_assessment(_body(item)) for item in items
279 )
282def _is_ship_assessment_source(item: dict[str, Any]) -> bool:
283 if _is_trusted_source(item, enforced=True):
284 return True
285 user = item.get("user") if isinstance(item.get("user"), dict) else {}
286 login = user.get("login") if isinstance(user.get("login"), str) else None
287 return bool(login and login.lower() in TRUSTED_SHIP_ASSESSMENT_BOTS)
290def contract_as_dict(
291 review_contract: dict[str, Any],
292 *,
293 dry_run: bool = False,
294 enforced: bool = True,
295 deferrals: tuple[str, ...] = (),
296 phase: str = PHASE_ALL,
297) -> dict[str, Any]:
298 """Return the required evidence set derived from review/jury flags."""
299 return {
300 "schema_version": SCHEMA_VERSION,
301 "enforced": enforced,
302 "phase": phase,
303 "source": "review_merge_contract + closure_comment",
304 "dry_run_disables_gating": True,
305 "fail_closed": True,
306 "require_distinct_vendors": _require_distinct_vendors(review_contract),
307 "accepted_sources": {
308 "closure": ("trusted issue/PR comments carrying keel.closure-comment.v1"),
309 "review": (
310 "trusted PR review/comment carrying keel.review-verdict.v1 and current head, "
311 "whose reviewer's latest Verdict line approves"
312 ),
313 "jury": "trusted PR comment carrying keel.jury-verdict.v1 and current head",
314 },
315 "not_accepted": [
316 "pull_request_body",
317 "chat_summary",
318 "untrusted_public_comment",
319 "keel_ship_assessment_comment",
320 ],
321 "deferrals": list(deferrals),
322 "required": [
323 item.as_dict()
324 for item in required_items(review_contract, dry_run=False, enforced=enforced)
325 ],
326 "active_required": [
327 item.as_dict()
328 for item in required_items(
329 review_contract, dry_run=dry_run, enforced=enforced, phase=phase
330 )
331 ],
332 }
335def required_items(
336 review_contract: dict[str, Any],
337 *,
338 dry_run: bool = False,
339 enforced: bool = True,
340 phase: str = PHASE_ALL,
341) -> tuple[EvidenceItem, ...]:
342 """Return the tier/flag-derived evidence requirements for ``phase``.
344 ``phase`` selects which artifacts are in scope: ``pre-merge`` covers the
345 review verdicts and a gating jury verdict (everything that must exist before
346 s10 authorizes a merge), ``post-merge`` covers the closure comments s11
347 posts, and ``all`` — the default, so existing callers are unchanged — covers
348 both. An unknown phase raises, so a typo cannot silently drop requirements.
349 """
350 if phase not in PHASES:
351 raise ValueError(f"unknown evidence phase {phase!r}; expected one of {', '.join(PHASES)}")
352 if dry_run or not enforced:
353 return ()
354 reviewers = review_contract.get("reviewers")
355 reviewer_count = reviewers.get("count") if isinstance(reviewers, dict) else 0
356 reviewer_count = reviewer_count if isinstance(reviewer_count, int) and reviewer_count > 0 else 0
357 jury = review_contract.get("jury")
358 jury_required = (
359 isinstance(jury, dict) and bool(jury.get("enabled")) and jury.get("mode") == "gating"
360 )
361 items: list[EvidenceItem] = [
362 EvidenceItem(
363 "closure-comment-pr",
364 "closure",
365 True,
366 "PR conversation comment with keel.closure-comment.v1 marker",
367 PHASE_POST_MERGE,
368 ),
369 EvidenceItem(
370 "closure-comment-issue",
371 "closure",
372 True,
373 "Linked issue comment with keel.closure-comment.v1 marker",
374 PHASE_POST_MERGE,
375 ),
376 ]
377 # A jury panel's verdicts are panelist ballots mapped onto s7 evidence by
378 # `keel review --from-jury` (#1015): same marker, same head binding, same
379 # requirement. The description names the panel that produced them so a
380 # missing one sends the operator to the jury run rather than to a host
381 # reviewer that was never dispatched.
382 review_description = (
383 "Distinct posted ai-jury panelist verdict for the current PR"
384 if review_panel(review_contract) == JURY_PANEL
385 else "Distinct posted s7 reviewer verdict for the current PR"
386 )
387 for index in range(1, reviewer_count + 1):
388 items.append(
389 EvidenceItem(
390 f"review-verdict-{index}",
391 "review",
392 True,
393 review_description,
394 PHASE_PRE_MERGE,
395 )
396 )
397 if jury_required:
398 items.append(
399 EvidenceItem(
400 "jury-verdict",
401 "jury",
402 True,
403 "Posted gating jury verdict comment for the current PR",
404 PHASE_PRE_MERGE,
405 )
406 )
407 if phase == PHASE_ALL:
408 return tuple(items)
409 return tuple(item for item in items if item.phase == phase)
412def verify(
413 review_contract: dict[str, Any],
414 *,
415 pr_comments: list[dict[str, Any]] | None = None,
416 issue_comments: list[dict[str, Any]] | None = None,
417 pr_reviews: list[dict[str, Any]] | None = None,
418 pr_body: str | None = None,
419 pr_title: str = "",
420 pr_labels: Sequence[str] | None = None,
421 head_sha: str | None = None,
422 covered_heads: Collection[str] = (),
423 ledger_record: dict[str, Any] | None = None,
424 dry_run: bool = False,
425 enforced: bool = True,
426 deferrals: tuple[str, ...] = (),
427 phase: str = PHASE_ALL,
428 require_armed: bool = False,
429 waived: bool = False,
430) -> dict[str, Any]:
431 """Verify required evidence artifacts and return a deterministic report.
433 ``phase`` narrows the requirement set to the artifacts that phase produces;
434 see :func:`required_items`. The s10 merge gate passes ``pre-merge`` so it
435 does not demand the closure comments s11 has not written yet.
437 ``require_armed`` closes the vacuous-pass hole: with the gate unarmed there
438 are no requirements, so the report would otherwise pass without having
439 checked anything, and a green result could not be told apart from "could not
440 tell whether this was a ship run". When set, an unarmed gate is a blocking
441 finding instead. A deliberately non-ship PR still goes green through the
442 operator waiver label, which disarms explicitly rather than by accident.
444 When ``ledger_record`` is the ship_run record for this PR, a closure comment
445 only counts when its content matches the canonical render of that record
446 (closure-comment fidelity). Without a record the marker-only behavior holds.
448 Every comment is classified by :func:`marker_in_header` — the marker on its
449 header line, never one quoted in its prose. A header naming two markers is
450 malformed: it is excluded from every count and reported as an advisory
451 ``malformed-evidence-comment`` finding.
453 When the gate is active, ``pr_labels`` are additionally checked for the
454 mandatory ``agent:<vendor>`` attribution label (and cross-checked against the
455 ledger implementer vendor when a record is present); see
456 :func:`attribution_check`. The labels are separately checked against keel's own
457 vocabulary — what :func:`keel.agents.attribution` produces from the ledger's
458 ``actors.implementer`` — by :func:`attribution_vocabulary_check`, which catches a
459 hand-composed label that happens to agree with a hand-written ledger value.
460 """
461 del pr_body # Explicitly not accepted as evidence.
462 items = required_items(review_contract, dry_run=dry_run, enforced=enforced, phase=phase)
463 deferred = set(deferrals)
464 counts = _evidence_counts(
465 pr_comments=pr_comments or [],
466 issue_comments=issue_comments or [],
467 pr_reviews=pr_reviews or [],
468 head_sha=head_sha,
469 covered_heads=covered_heads,
470 enforced=enforced,
471 ledger_record=ledger_record,
472 )
473 findings = _run_context_findings(
474 pr_comments=pr_comments or [],
475 issue_comments=issue_comments or [],
476 enforced=enforced,
477 ledger_record=ledger_record,
478 )
479 mismatch = _closure_mismatch_scopes(
480 pr_comments=pr_comments or [],
481 issue_comments=issue_comments or [],
482 enforced=enforced,
483 ledger_record=ledger_record,
484 )
485 # A jury whose standing consensus does not approve (#1429) leaves `jury-verdict`
486 # unsatisfied *and* says why, by name: unsatisfied alone would read as "missing" and
487 # hide a comment that is on the pull request. It blocks only where the jury verdict is
488 # a requirement — an advisory panel's rejection is reported, never gated on, which is
489 # what advisory means.
490 jury_required = any(
491 item.id == "jury-verdict"
492 and not (item.id in deferred or item.kind in deferred or "all" in deferred)
493 for item in items
494 )
495 # The opt-in COMMENT-consensus rule (#1570): a COMMENT panel with no blocking finding
496 # and enough approving reviewers at the head satisfies `jury-verdict`. When it is on
497 # and does not apply, the refusal below says which of its conditions failed.
498 # Only where `jury-verdict` is required: an advisory panel has no requirement to meet,
499 # and its COMMENT is reported as it always was, with no follow-up filed for it.
500 consensus = (
501 comment_consensus(
502 review_contract,
503 pr_comments or [],
504 pr_reviews or [],
505 head_sha=head_sha,
506 covered_heads=covered_heads,
507 enforced=enforced,
508 pr_title=pr_title,
509 )
510 if jury_required
511 else None
512 )
513 consensus_applied = consensus is not None and consensus["applied"]
514 if consensus_applied:
515 counts["jury_verdict"] = 1
516 jury_refusal = (
517 None
518 if consensus_applied
519 else _jury_not_approved_finding(
520 pr_comments or [],
521 head_sha=head_sha,
522 covered_heads=covered_heads,
523 enforced=enforced,
524 blocking=jury_required,
525 comment_rule=consensus,
526 )
527 )
528 results = []
529 for item in items:
530 present = _is_present(item, counts)
531 is_deferred = item.id in deferred or item.kind in deferred or "all" in deferred
532 ok = present or is_deferred
533 results.append(
534 {
535 "id": item.id,
536 "kind": item.kind,
537 "required": item.required,
538 "present": present,
539 "deferred": is_deferred,
540 "ok": ok,
541 "reason": None
542 if ok
543 else (
544 jury_refusal["message"]
545 if item.id == "jury-verdict" and jury_refusal is not None
546 else _result_reason(item, mismatch)
547 ),
548 }
549 )
550 missing = [result["id"] for result in results if not result["ok"]]
551 distinct = _distinct_vendor_finding(
552 review_contract,
553 items=items,
554 deferred=deferred,
555 pr_comments=pr_comments or [],
556 pr_reviews=pr_reviews or [],
557 head_sha=head_sha,
558 covered_heads=covered_heads,
559 enforced=enforced,
560 )
561 if distinct is not None:
562 findings = [*findings, distinct]
563 substance = _verdict_substance_findings(
564 [*(pr_comments or []), *(pr_reviews or [])],
565 head_sha=head_sha,
566 covered_heads=covered_heads,
567 enforced=enforced,
568 pr_title=pr_title,
569 )
570 findings = [*findings, *substance]
571 # A reviewer whose standing verdict does not approve holds the merge by name (#1426):
572 # the counts above leave them out, and this says why instead of letting another
573 # reviewer's approval make up the number.
574 findings = [
575 *findings,
576 *_verdict_not_approved_findings(
577 [*(pr_comments or []), *(pr_reviews or [])],
578 head_sha=head_sha,
579 covered_heads=covered_heads,
580 enforced=enforced,
581 ),
582 ]
583 if jury_refusal is not None:
584 findings = [*findings, jury_refusal]
585 findings = [
586 *findings,
587 *_malformed_marker_findings(
588 [*(pr_comments or []), *(issue_comments or []), *(pr_reviews or [])],
589 enforced=enforced,
590 ),
591 ]
592 attribution = _attribution_finding(
593 pr_labels=pr_labels,
594 enforced=enforced and not dry_run,
595 ledger_record=ledger_record,
596 )
597 if attribution is not None:
598 findings = [*findings, attribution]
599 model_label = _attribution_model_finding(pr_labels=pr_labels, enforced=enforced and not dry_run)
600 if model_label is not None:
601 findings = [*findings, model_label]
602 vocabulary = _attribution_vocabulary_finding(
603 pr_labels=pr_labels,
604 enforced=enforced and not dry_run,
605 ledger_record=ledger_record,
606 )
607 if vocabulary is not None:
608 findings = [*findings, vocabulary]
609 unarmed = _unarmed_finding(
610 enforced=enforced,
611 dry_run=dry_run,
612 require_armed=require_armed,
613 waived=waived,
614 )
615 if unarmed is not None:
616 findings = [*findings, unarmed]
617 blocking_findings = [f for f in findings if f["severity"] == BLOCKING_FINDING_SEVERITY]
618 has_mismatch = bool(mismatch)
619 if not missing and not blocking_findings:
620 status = STATUS_PASS
621 elif blocking_findings or has_mismatch:
622 status = STATUS_FAIL
623 else:
624 status = STATUS_WAITING
625 return {
626 "schema_version": SCHEMA_VERSION,
627 "status": status,
628 "dry_run": dry_run,
629 "enforced": enforced,
630 "phase": phase,
631 "required_count": len(items),
632 "missing": missing,
633 "results": results,
634 "counts": counts,
635 "findings": findings,
636 "comment_consensus": consensus,
637 }
640def comment_consensus(
641 review_contract: dict[str, Any],
642 pr_comments: list[dict[str, Any]],
643 pr_reviews: list[dict[str, Any]],
644 *,
645 head_sha: str | None,
646 covered_heads: Collection[str] = (),
647 enforced: bool = True,
648 pr_title: str = "",
649) -> dict[str, Any] | None:
650 """Whether the opt-in COMMENT-consensus rule satisfies ``jury-verdict`` here (#1570).
652 ``None`` when the rule has nothing to say: it is off on this contract
653 (``jury.comment_consensus`` is not a count), no jury verdict stands for the head, or
654 the standing consensus is not ``COMMENT`` — an approval needs no rule, and
655 ``REQUEST_CHANGES``, ``ABSTAIN``, ``NO_QUORUM`` or an unreadable line is a hard stop
656 this rule never relaxes.
658 Otherwise a record: ``applied`` and, when it is not, the ``reason`` naming the first
659 condition that failed. All of these must hold, and each fails closed:
661 * **a resolved head** (:func:`keel.juryavail.is_pinnable_head`). This rule removes a
662 requirement, so — like the vendor count (#1069) — it is never read off a blank head;
663 * **a findings summary keel rendered** (:func:`keel.artifacts.jury_verdict_summary`),
664 because "unreadable" is never the same answer as "no findings";
665 * **no** ``critical`` / ``major`` **item** in it, and every item labeled ``minor`` or
666 ``nit`` (any unrecognised or missing severity blocks, #1570);
667 * **no review verdict at the head that fails to approve**, and **at least
668 ``min_approvals`` distinct approving ones** — the same tally the
669 ``review-verdict-N`` items count (:func:`_review_tally`: trusted, head-pinned, with
670 substance, judged at each reviewer's latest word), at the head under verification or
671 a head it covers by capture commits alone.
673 ``minors`` are the summary's remaining items, nits left out: what ``keel merge`` files
674 as the follow-up issue when the rule is what passed the gate.
675 """
676 jury = review_contract.get("jury")
677 required = jury.get("comment_consensus") if isinstance(jury, dict) else None
678 if isinstance(required, bool) or not isinstance(required, int):
679 return None
680 standing = _standing_jury_verdict(
681 pr_comments, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced
682 )
683 if standing is None:
684 return None
685 body = _body(standing)
686 if jury_verdict_token(body) != COMMENT_CONSENSUS:
687 return None
688 record: dict[str, Any] = {
689 "applied": False,
690 "reason": None,
691 "head": _verdict_head(standing, body),
692 "required_approvals": required,
693 "approvals": 0,
694 "minors": [],
695 }
696 if not juryavail.is_pinnable_head(head_sha):
697 record["reason"] = "no head was resolved to hold the review verdicts to"
698 return record
699 # Imported here: `artifacts` renders the verdict this module reads and imports it.
700 from . import artifacts
702 summary = artifacts.jury_verdict_summary(body)
703 if summary is None:
704 record["reason"] = "its findings summary is not one keel rendered and cannot be read"
705 return record
706 # Fail closed on unknown severities (#1570): only an explicit `minor:` or `nit:` item
707 # is non-blocking; anything else (no `:`, `blocker:`, empty or unknown token) holds.
708 labelled = [
709 (item.partition(":")[0].strip().lower() if ":" in item else "", item) for item in summary
710 ]
711 blocking = [item for label, item in labelled if label in _COMMENT_BLOCKING_LABELS]
712 if blocking:
713 record["reason"] = f"its findings summary carries a blocking finding ({blocking[0]})"
714 return record
715 unrecognised = [item for label, item in labelled if label not in _COMMENT_NON_BLOCKING_LABELS]
716 if unrecognised:
717 record["reason"] = (
718 "its findings summary carries an item with an unrecognised severity "
719 f"({unrecognised[0]})"
720 )
721 return record
722 tally = _review_tally(
723 [*pr_comments, *pr_reviews],
724 head_sha=head_sha,
725 covered_heads=covered_heads,
726 enforced=enforced,
727 pr_title=pr_title,
728 )
729 record["approvals"] = len(tally.accepted)
730 if tally.not_approved:
731 record["reason"] = f"{len(tally.not_approved)} review verdict(s) at the head do not approve"
732 return record
733 if len(tally.accepted) < required:
734 record["reason"] = (
735 f"{len(tally.accepted)} of {required} required approving review verdicts at the head"
736 )
737 return record
738 record["applied"] = True
739 record["minors"] = [item for label, item in labelled if label != _COMMENT_ADVISORY_LABEL]
740 return record
743def refusal_reason(verification: dict[str, Any]) -> str:
744 """Why a verification that is not a pass refuses the merge, naming everything (#1420).
746 Every missing item, then every blocking finding's message. A finding can fail the
747 verification with nothing missing — ``attribution-label`` on a pull request whose
748 review verdicts are all there — and a reason built from ``missing`` alone then read
749 ``missing evidence: `` with nothing after it, which names no cause at all. Never empty:
750 a verification that names neither says its status.
751 """
752 missing = [str(item) for item in verification.get("missing") or ()]
753 blocking = [
754 f"{finding.get('id')}: {finding.get('message')}"
755 for finding in verification.get("findings") or ()
756 if isinstance(finding, dict) and finding.get("severity") == BLOCKING_FINDING_SEVERITY
757 ]
758 parts: list[str] = []
759 if missing:
760 parts.append(f"missing evidence: {', '.join(missing)}")
761 if blocking:
762 parts.append(f"blocking finding(s): {'; '.join(blocking)}")
763 return "; ".join(parts) or f"evidence verification is {verification.get('status')}"
766def _require_distinct_vendors(review_contract: dict[str, Any]) -> bool:
767 reviewers = review_contract.get("reviewers")
768 return bool(reviewers.get("require_distinct_vendors")) if isinstance(reviewers, dict) else False
771def review_panel(review_contract: dict[str, Any]) -> str:
772 """Who the reviewers are on this contract: ``reviewers`` or ``jury`` (#1015).
774 A missing or malformed ``reviewers`` block reads as the host bench, which is
775 the stricter of the two answers everywhere this is asked.
776 """
777 reviewers = review_contract.get("reviewers")
778 panel = reviewers.get("panel") if isinstance(reviewers, dict) else None
779 return panel if isinstance(panel, str) and panel else "reviewers"
782def _minimum_jury_vendors(review_contract: dict[str, Any]) -> int:
783 """``jury.minimum_vendors`` from the contract, with the schema floor as fallback."""
784 jury = review_contract.get("jury")
785 minimum = jury.get("minimum_vendors") if isinstance(jury, dict) else None
786 if isinstance(minimum, int) and not isinstance(minimum, bool) and minimum > 0:
787 return minimum
788 return DEFAULT_MINIMUM_JURY_VENDORS
791def _verdict_substance_findings(
792 items: list[dict[str, Any]],
793 *,
794 head_sha: str | None,
795 covered_heads: Collection[str] = (),
796 enforced: bool,
797 pr_title: str,
798) -> list[dict[str, Any]]:
799 """One finding per verdict refused for naming nothing concrete (#926).
801 Reported rather than dropped. A verdict silently not counted surfaces as
802 "missing required evidence: review-verdict-2", which sends the operator
803 looking for a comment that is sitting right there — the reason has to say
804 the verdict was read and found to be a receipt.
806 ``minor`` when the gate is not enforced, mirroring the other content
807 findings: an advisory run should say what it saw without failing.
808 """
809 _, rejected = _review_evidence_keys_and_rejections(
810 items, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced, pr_title=pr_title
811 )
812 return [
813 {
814 "id": "review-verdict-insubstantial",
815 "severity": "major" if enforced else "minor",
816 "kind": "review",
817 "message": f"{key}: {reason}.",
818 }
819 for key, reason in sorted(rejected)
820 ]
823def _malformed_marker_findings(
824 items: list[dict[str, Any]],
825 *,
826 enforced: bool,
827) -> list[dict[str, Any]]:
828 """One finding per trusted comment whose header names more than one marker.
830 Such a header does not say which artifact the comment is, so
831 :func:`marker_in_header` refuses to classify it and the comment counts toward
832 nothing. Excluding it silently would reproduce the failure #926 named — a
833 comment sitting right there on the PR, reported as missing evidence — so the
834 exclusion is stated instead of inferred.
836 ``minor``, never blocking: the comment is malformed, not fraudulent, and the
837 requirement it failed to satisfy already fails on its own.
838 """
839 findings: list[dict[str, Any]] = []
840 for item in items:
841 if not _is_trusted_source(item, enforced=enforced):
842 continue
843 markers = header_markers(_body(item))
844 if len(markers) < 2:
845 continue
846 findings.append(
847 {
848 "id": MALFORMED_MARKER_FINDING,
849 "severity": "minor",
850 "kind": "evidence",
851 "message": (
852 "Comment header carries more than one keel marker "
853 f"({', '.join(markers)}); it is excluded from evidence."
854 ),
855 }
856 )
857 return findings
860def _distinct_vendor_finding(
861 review_contract: dict[str, Any],
862 *,
863 items: tuple[EvidenceItem, ...],
864 deferred: set[str],
865 pr_comments: list[dict[str, Any]],
866 pr_reviews: list[dict[str, Any]],
867 head_sha: str | None,
868 covered_heads: Collection[str] = (),
869 enforced: bool,
870) -> dict[str, Any] | None:
871 """Return a blocking finding when the optional vendor-distinctness check fails.
873 Off by default: ``None`` unless ``reviewers.require_distinct_vendors`` is set
874 on the contract. Skipped when review evidence is deferred so the knob never
875 overrides an explicit deferral.
876 """
877 if not _require_distinct_vendors(review_contract):
878 return None
879 if "review" in deferred or "all" in deferred:
880 return None
881 required = sum(1 for item in items if item.kind == "review" and item.id not in deferred)
882 if required <= 0:
883 return None
884 provenance = _review_vendor_provenance(
885 [*pr_comments, *pr_reviews],
886 head_sha=head_sha,
887 covered_heads=covered_heads,
888 enforced=enforced,
889 )
890 if review_panel(review_contract) == JURY_PANEL:
891 result = panel_vendor_check(
892 list(provenance.values()),
893 required_count=required,
894 minimum_vendors=_minimum_jury_vendors(review_contract),
895 )
896 else:
897 result = distinct_vendor_check(list(provenance.values()), required_count=required)
898 if result["ok"]:
899 return None
900 return {
901 "id": "review-vendor-distinctness",
902 "severity": "major",
903 "kind": "review",
904 "message": f"require_distinct_vendors: {result['reason']}.",
905 }
908_CLOSURE_MISMATCH_REASON = "closure comment does not match the ship_run ledger record"
911def _result_reason(item: EvidenceItem, mismatch: set[str]) -> str:
912 if item.id == "closure-comment-pr" and "pr" in mismatch:
913 return _CLOSURE_MISMATCH_REASON
914 if item.id == "closure-comment-issue" and "issue" in mismatch:
915 return _CLOSURE_MISMATCH_REASON
916 return f"missing required evidence: {item.id}"
919def _closure_mismatch_scopes(
920 *,
921 pr_comments: list[dict[str, Any]],
922 issue_comments: list[dict[str, Any]],
923 enforced: bool,
924 ledger_record: dict[str, Any] | None,
925) -> set[str]:
926 """Return scopes ({"pr"}/{"issue"}) where a marker closure mismatched the ledger.
928 A scope is reported only when a trusted marker-bearing closure exists but none
929 of them match the record — so a stale comment alongside a correct re-post does
930 not produce a misleading mismatch reason.
931 """
932 if ledger_record is None:
933 return set()
934 scopes: set[str] = set()
935 for scope, comments in (("pr", pr_comments), ("issue", issue_comments)):
936 markered = [
937 comment
938 for comment in comments
939 if _is_trusted_source(comment, enforced=enforced)
940 and _has_closure_marker(_body(comment))
941 ]
942 if markered and not any(
943 closure_body_matches_record(_body(comment), ledger_record) for comment in markered
944 ):
945 scopes.add(scope)
946 return scopes
949def _evidence_counts(
950 *,
951 pr_comments: list[dict[str, Any]],
952 issue_comments: list[dict[str, Any]],
953 pr_reviews: list[dict[str, Any]],
954 head_sha: str | None = None,
955 covered_heads: Collection[str] = (),
956 enforced: bool = True,
957 ledger_record: dict[str, Any] | None = None,
958) -> dict[str, int]:
959 review_keys = _review_evidence_keys(
960 [*pr_comments, *pr_reviews],
961 head_sha=head_sha,
962 covered_heads=covered_heads,
963 enforced=enforced,
964 )
965 return {
966 "closure_pr": sum(
967 _is_closure_comment(comment, enforced=enforced, record=ledger_record)
968 for comment in pr_comments
969 ),
970 "closure_issue": sum(
971 _is_closure_comment(comment, enforced=enforced, record=ledger_record)
972 for comment in issue_comments
973 ),
974 "review_verdict": len(review_keys),
975 # The jury requirement is met by the *standing* jury verdict at the head, and only
976 # when its consensus approves (#1429); a rejection is reported by
977 # `_jury_not_approved_finding` rather than counted.
978 "jury_verdict": int(
979 jury_verdict_approves(
980 _body(
981 _standing_jury_verdict(
982 pr_comments,
983 head_sha=head_sha,
984 covered_heads=covered_heads,
985 enforced=enforced,
986 )
987 or {}
988 )
989 )
990 ),
991 }
994def _is_present(item: EvidenceItem, counts: dict[str, int]) -> bool:
995 if item.id == "closure-comment-pr":
996 return counts["closure_pr"] >= 1
997 if item.id == "closure-comment-issue":
998 return counts["closure_issue"] >= 1
999 if item.kind == "review":
1000 index = int(item.id.rsplit("-", 1)[1])
1001 return counts["review_verdict"] >= index
1002 if item.id == "jury-verdict":
1003 return counts["jury_verdict"] >= 1
1004 return False
1007def _body(item: dict[str, Any]) -> str:
1008 body = item.get("body")
1009 return body if isinstance(body, str) else ""
1012def _header_line(body: str) -> str:
1013 """``body``'s header line: its first *non-empty* line, stripped.
1015 Leading blank lines are skipped rather than treated as the end, for the same
1016 reason :func:`_fields` skips them — a GitHub comment body routinely begins
1017 with a newline. Everything after that line is prose.
1018 """
1019 for raw_line in (body or "").splitlines():
1020 line = raw_line.strip()
1021 if not line:
1022 continue
1023 return line
1024 return ""
1027def _unwrap_html_comment(line: str) -> str:
1028 """Strip one literal ``<!-- … -->`` wrapper from ``line``, or return it unchanged.
1030 Deliberately not a regex and deliberately not an HTML parser: the only
1031 wrapper keel has to recognise is the one
1032 :func:`keel.closure.render_closure_comment` writes. Anything else — an
1033 unterminated ``<!--``, a ``--!>`` close, a second wrapper on the same line —
1034 is left intact, so the marker check below sees the delimiters as tokens and
1035 refuses to classify the comment. Failing to recognise a hand-rolled wrapper
1036 costs a comment its classification; guessing at one would let a body render
1037 as an invisible comment while counting as evidence.
1038 """
1039 if (
1040 line.startswith(_HTML_COMMENT_OPEN)
1041 and line.endswith(_HTML_COMMENT_CLOSE)
1042 and len(line) >= len(_HTML_COMMENT_OPEN) + len(_HTML_COMMENT_CLOSE)
1043 ):
1044 return line[len(_HTML_COMMENT_OPEN) : -len(_HTML_COMMENT_CLOSE)].strip()
1045 return line
1048def header_markers(body: str) -> tuple[str, ...]:
1049 """Return the distinct :data:`CLASSIFICATION_MARKERS` ``body``'s header carries.
1051 The header line, once unwrapped, must consist of markers and nothing else —
1052 that is exactly what every renderer emits, and it is what separates a marker
1053 line from a sentence that happens to name one. So this is empty for an
1054 ordinary comment (including one whose first line *mentions* a marker in
1055 prose, and one wearing a wrapper keel does not write), one entry for a
1056 well-formed artifact, and two or more for a malformed one, which
1057 :func:`marker_in_header` refuses to classify and
1058 :func:`_malformed_marker_findings` reports.
1059 """
1060 tokens = _unwrap_html_comment(_header_line(body)).split()
1061 if not tokens or not _CLASSIFICATION_MARKERS_SET.issuperset(tokens):
1062 return ()
1063 return tuple(marker for marker in CLASSIFICATION_MARKERS if marker in tokens)
1066def marker_in_header(body: str) -> str | None:
1067 """Return the single keel marker ``body`` is anchored to, or ``None`` (#1026).
1069 **The header block is the only place a marker classifies a comment.** A marker
1070 further down is prose — a reviewer writing "I checked the jury-verdict marker
1071 handling" is quoting a string, not filing a jury verdict. Testing
1072 ``MARKER in body`` could not tell the two apart: two `keel.review-verdict.v1`
1073 comments whose scope mentioned ``keel.jury-verdict.v1`` were counted as
1074 ``jury_verdict: 2, review_verdict: 0``, and the review that happened was
1075 invisible to the gate.
1077 ``None`` for a comment that carries no marker *and* for one whose header
1078 carries several: a header naming two artifacts does not say which one it is,
1079 so it is excluded rather than counted for either.
1080 """
1081 markers = header_markers(body)
1082 return markers[0] if len(markers) == 1 else None
1085def _has_closure_marker(body: str) -> bool:
1086 return marker_in_header(body) == closure.CLOSURE_SCHEMA_VERSION
1089#: The idempotency marker ``keel post-comment`` appends to a posted body so a re-post
1090#: can find and edit its own comment. It is transport bookkeeping, not content, so it is
1091#: stripped before a closure body is compared to its canonical render.
1092#:
1093#: Matched in the *exact* form the transport emits — a run id, then the close, then end
1094#: of line. A permissive ``.*?`` would let a trusted author smuggle arbitrary text past
1095#: the verbatim comparison: an HTML comment ends at its first ``-->``, so anything after
1096#: that renders visibly on the page while the whole line still normalizes away.
1097RUN_ID_MARKER_RE = re.compile(r"^\s*<!--\s*keel\.run-id:\s*[\w.:@/+-]+\s*-->\s*$")
1100def _normalize_closure_body(body: str) -> str:
1101 """Normalize a closure body for content comparison.
1103 Robust to harmless formatting drift but sensitive to real content changes:
1104 trailing whitespace is stripped per line, runs of blank lines collapse to a
1105 single blank line, and leading/trailing blank lines are dropped.
1107 The transport's ``keel.run-id`` marker line is dropped too. Without that, closure
1108 fidelity and post-comment idempotency were mutually exclusive: the marker is what
1109 lets a re-post edit its own comment instead of duplicating, and its presence made
1110 the body differ from the canonical render.
1111 """
1112 lines = [line.rstrip() for line in body.splitlines() if not RUN_ID_MARKER_RE.match(line)]
1113 normalized: list[str] = []
1114 for line in lines:
1115 if not line and (not normalized or not normalized[-1]):
1116 continue
1117 normalized.append(line)
1118 while normalized and not normalized[-1]:
1119 normalized.pop()
1120 return "\n".join(normalized)
1123def closure_body_matches_record(body: str, record: dict[str, Any]) -> bool:
1124 """Return whether ``body`` matches the canonical render of ``record``."""
1125 expected = closure.render_closure_comment(record)
1126 return _normalize_closure_body(body) == _normalize_closure_body(expected)
1129def _is_closure_comment(
1130 item: dict[str, Any],
1131 *,
1132 enforced: bool = True,
1133 record: dict[str, Any] | None = None,
1134) -> bool:
1135 if not _is_trusted_source(item, enforced=enforced):
1136 return False
1137 if not _has_closure_marker(_body(item)):
1138 return False
1139 if record is None:
1140 return True
1141 return closure_body_matches_record(_body(item), record)
1144def _run_context_findings(
1145 *,
1146 pr_comments: list[dict[str, Any]],
1147 issue_comments: list[dict[str, Any]],
1148 enforced: bool,
1149 ledger_record: dict[str, Any] | None = None,
1150) -> list[dict[str, Any]]:
1151 comments = [*pr_comments, *issue_comments]
1152 findings: list[dict[str, Any]] = []
1153 for item in comments:
1154 if not _is_closure_comment(item, enforced=enforced, record=ledger_record):
1155 continue
1156 body = _body(item)
1157 if _has_empty_run_context(body):
1158 findings.append(
1159 {
1160 "id": "run-context-empty",
1161 "severity": "major" if enforced else "minor",
1162 "kind": "closure",
1163 "message": "Closure comment Run context is fully degraded.",
1164 }
1165 )
1166 return findings
1169def _has_empty_run_context(body: str) -> bool:
1170 if "### Run context" not in body:
1171 return False
1172 fields = _run_context_fields(body)
1173 return fields == {
1174 "host agent": "unknown",
1175 "transport": "unknown",
1176 "profile": "unknown",
1177 "jury": "off",
1178 "consent": "unknown (scopes: none)",
1179 }
1182def _run_context_fields(body: str) -> dict[str, str]:
1183 fields: dict[str, str] = {}
1184 in_block = False
1185 for line in body.splitlines():
1186 if line.strip() == "### Run context":
1187 in_block = True
1188 continue
1189 if in_block and line.startswith("### "):
1190 break
1191 if not in_block:
1192 continue
1193 match = re.match(r"^-\s+\*\*(?P<key>[^*]+):\*\*\s+(?P<value>.+?)\s*$", line)
1194 if match:
1195 fields[match.group("key").strip().lower()] = match.group("value").strip().lower()
1196 return fields
1199def _is_trusted_source(item: dict[str, Any], *, enforced: bool = True) -> bool:
1200 """Return whether GitHub marks this evidence source as trusted.
1202 Live GitHub comment/review payloads include ``author_association``. Enforced
1203 evidence fails closed when that field is absent because offline fixtures are
1204 agent-writable and must not manufacture trust. Untrusted explicit
1205 associations fail closed even if the author type is ``Bot``.
1206 """
1207 association = item.get("author_association")
1208 if association is None:
1209 return not enforced
1210 if isinstance(association, str) and association.upper() in TRUSTED_AUTHOR_ASSOCIATIONS:
1211 return True
1212 return False
1215def _is_ship_assessment(body: str) -> bool:
1216 """Whether ``body`` is a ship assessment comment, decided by its header (#1035).
1218 Header-anchored for the same reason markers are (#1026): this is consulted as an
1219 *exclusion* by :func:`_is_review_verdict_body` and :func:`_is_jury_verdict`, so a
1220 whole-body substring test let a reviewer disarm their own verdict by quoting the
1221 heading while describing what they reviewed ("the ``### \U0001f6a2 keel ship``
1222 comment claims the gates passed, but…"). The verdict was then silently uncounted
1223 and ``evidence-verify`` reported it missing from a PR it was sitting on.
1225 The heading is a Markdown heading rather than a versioned ``keel.*.v1`` marker, so
1226 it cannot join :data:`CLASSIFICATION_MARKERS`; it gets the same anchoring instead.
1227 A real assessment leads with the heading (the workflow writes it first) or with the
1228 CLI's own banner, so nothing that armed the gate through a genuine assessment
1229 comment stops arming it.
1230 """
1231 header = _header_line(body)
1232 return header.startswith(SHIP_ASSESSMENT_HEADING) or header.startswith(SHIP_ASSESSMENT_BANNER)
1235def count_review_verdicts(
1236 pr_comments: list[dict[str, Any]] | None = None,
1237 pr_reviews: list[dict[str, Any]] | None = None,
1238 *,
1239 head_sha: str | None = None,
1240 covered_heads: Collection[str] = (),
1241 enforced: bool = True,
1242 pr_title: str = "",
1243) -> int:
1244 """Count distinct trusted review-verdict reviewers for a PR.
1246 This is the same evidence-side counting the verify report uses for the
1247 ``review`` items: it collapses idempotent re-posts by the same reviewer to
1248 one verdict and only counts trusted, head-bound verdicts whose reviewer's
1249 latest ``Verdict:`` line approves (#1426) — a reviewer who requested changes
1250 reviewed, but did not pass, the change. Reused by capture reconcile to
1251 cross-check the ledger's recorded reviewer count.
1252 """
1253 keys = _review_evidence_keys(
1254 [*(pr_comments or []), *(pr_reviews or [])],
1255 head_sha=head_sha,
1256 covered_heads=covered_heads,
1257 enforced=enforced,
1258 pr_title=pr_title,
1259 )
1260 return len(keys)
1263def _review_evidence_keys(
1264 items: list[dict[str, Any]],
1265 *,
1266 head_sha: str | None = None,
1267 covered_heads: Collection[str] = (),
1268 enforced: bool = True,
1269 pr_title: str = "",
1270) -> set[str]:
1271 keys, _ = _review_evidence_keys_and_rejections(
1272 items, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced, pr_title=pr_title
1273 )
1274 return keys
1277def verdict_reviewers(
1278 items: list[dict[str, Any]],
1279 *,
1280 head_sha: str | None = None,
1281 covered_heads: Collection[str] = (),
1282 enforced: bool = True,
1283) -> tuple[str, ...]:
1284 """The reviewers whose verdicts count for ``head_sha``, by name, sorted (#1422).
1286 The verdicts :func:`verify` counts toward ``review-verdict-N`` — trusted, head-pinned,
1287 with substance, and approving at the reviewer's latest word (#1426), so a reviewer who
1288 requested changes is not listed as having passed it — named by their ``reviewer:``
1289 field, else by the commenter's login. A
1290 verdict keyed only by its body names nobody and is left out. This is what a closure
1291 comment written after the merge says reviewed the change, read off the pull request
1292 rather than recalled.
1293 """
1294 keys = _review_evidence_keys(
1295 items, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced
1296 )
1297 names = []
1298 for key in keys:
1299 kind, _, name = key.partition(":")
1300 if kind in ("reviewer", "user"):
1301 names.append(name)
1302 return tuple(sorted(names))
1305@dataclass(frozen=True)
1306class _ReviewTally:
1307 """What the review verdicts at a head add up to, per reviewer (#1426).
1309 ``accepted`` are the reviewer keys whose verdict counts toward ``review-verdict-N``;
1310 ``vendors`` maps each of them to its declared vendor (or ``None``). ``insubstantial``
1311 are the ``(key, reason)`` pairs refused for naming nothing (#926), and
1312 ``not_approved`` the ``(key, token, head)`` triples of reviewers whose standing
1313 verdict does not approve (``token`` is ``None`` when it has no readable
1314 ``Verdict:`` line). Both refusals are reported, never dropped.
1315 """
1317 accepted: frozenset[str]
1318 vendors: dict[str, str | None]
1319 insubstantial: tuple[tuple[str, str], ...]
1320 not_approved: tuple[tuple[str, str | None, str], ...]
1323def _posted_at(item: dict[str, Any]) -> str:
1324 """When ``item`` was posted, as GitHub's ISO-8601 string, or ``""`` when unknown.
1326 An issue comment carries ``created_at`` and a pull-request review ``submitted_at``.
1327 ``updated_at`` is deliberately not read: editing an old approval (a typo fix, an
1328 idempotent re-post of the same run's comment) must not make it outrank a rejection
1329 posted after it. ISO-8601 UTC strings order lexically, so no clock is parsed.
1330 """
1331 for field in ("created_at", "submitted_at"):
1332 value = item.get(field)
1333 if isinstance(value, str) and value:
1334 return value
1335 return ""
1338def review_verdict_token(body: str) -> str | None:
1339 """The upper-cased first word of ``body``'s first ``Verdict:`` line, or ``None`` (#1426).
1341 The line is the one :func:`keel.artifacts.render_review_verdict` writes
1342 (``Verdict: <verdict>``); it is found anywhere in the comment, header block included,
1343 but only at the start of a line. The first word is read case-insensitively, with
1344 wrapper punctuation skipped and ``-`` spelled ``_``, so ``APPROVE — minor nits``,
1345 ``**lgtm**`` and ``request-changes`` read ``APPROVE``, ``LGTM`` and
1346 ``REQUEST_CHANGES``. ``None`` when the comment has no such line or the line has no
1347 word, which :func:`verdict_approves` treats as not an approval.
1348 """
1349 return _line_token(body, _VERDICT_LINE_RE)
1352def _line_token(body: str, line_re: re.Pattern[str]) -> str | None:
1353 """The first word of the first line ``line_re`` matches, read as a verdict token."""
1354 for line in body.splitlines():
1355 match = line_re.match(line)
1356 if match is None:
1357 continue
1358 token = _VERDICT_TOKEN_RE.match(match.group("value").strip())
1359 return token.group("token").upper().replace("-", "_") if token else None
1360 return None
1363def verdict_approves(body: str) -> bool:
1364 """Whether a review verdict's ``Verdict:`` line approves (:data:`APPROVING_VERDICTS`)."""
1365 return review_verdict_token(body) in APPROVING_VERDICTS
1368def jury_verdict_token(body: str) -> str | None:
1369 """The consensus token on a jury verdict's ``AI Jury verdict:`` line, or ``None`` (#1429).
1371 Read exactly as :func:`review_verdict_token` reads a review's ``Verdict:`` line — first
1372 word, any case, wrapper punctuation skipped, ``-`` spelled ``_`` — off the line
1373 :func:`keel.artifacts.render_jury_verdict` writes, ``AI Jury verdict: <verdict>.``, so
1374 its trailing full stop is not part of the token. ``None`` when there is no such line or
1375 it carries no word.
1376 """
1377 return _line_token(body, _JURY_VERDICT_LINE_RE)
1380def jury_verdict_approves(body: str) -> bool:
1381 """Whether a jury verdict's consensus approves (:data:`APPROVING_VERDICTS`, #1429).
1383 The same set a review verdict is read against: the chair's ``APPROVE`` / ``READY`` reach
1384 the comment as ``LGTM`` through :func:`keel.jury.map_verdict`. ``REQUEST_CHANGES`` (and
1385 ``NEEDS_INFO``, which maps onto it), ``COMMENT``, ``ABSTAIN`` — the token
1386 :func:`keel.jury.jury_verdict` writes when the panel produced no chair synthesis — and
1387 ``NO_QUORUM`` are not approvals, and neither is an unknown word or a missing line.
1388 """
1389 return jury_verdict_token(body) in APPROVING_VERDICTS
1392def _not_approved_message(key: str, token: str | None, head: str) -> str:
1393 """How a refusal names a reviewer whose standing verdict does not approve (#1426)."""
1394 name = _reviewer_name(key)
1395 if token in REQUEST_CHANGES_VERDICTS:
1396 return f"{name} requests changes at {head}."
1397 why = f"verdict {token}" if token else "no readable Verdict line"
1398 return f"{name} does not approve at {head} ({why})."
1401def _verdict_head(item: dict[str, Any], body: str) -> str:
1402 """The head a verdict answers for: its ``head:`` field, else the review's commit."""
1403 recorded = _fields(body).get("head")
1404 if recorded:
1405 return recorded
1406 commit_id = item.get("commit_id")
1407 return commit_id if isinstance(commit_id, str) and commit_id else "an unrecorded head"
1410def _reviewer_name(key: str) -> str:
1411 """A reviewer key as a refusal names it: the reviewer, or the verdict's digest."""
1412 kind, _, name = key.partition(":")
1413 return f"an unnamed reviewer (verdict {name[:12]})" if kind == "body" else name
1416def _review_tally(
1417 items: list[dict[str, Any]],
1418 *,
1419 head_sha: str | None = None,
1420 covered_heads: Collection[str] = (),
1421 enforced: bool = True,
1422 pr_title: str = "",
1423) -> _ReviewTally:
1424 """Tally the trusted review verdicts that answer for ``head_sha`` (#1426).
1426 **A reviewer's latest verdict is their review.** Verdicts are ordered by when they
1427 were posted (:func:`_posted_at`, then their position), grouped by reviewer key, and
1428 each reviewer is judged by the last one — at the head or at a head it descends from
1429 by capture commits alone (``covered_heads``, #1203):
1431 * **It does not approve** (:func:`verdict_approves`): the reviewer is refused, with
1432 their stance and the head it was given at. A rejection is evidence; dropping it
1433 silently would let another reviewer's approval outvote it. So a reviewer who
1434 approved and then requested changes holds the merge.
1435 * **It approves**: the reviewer counts when any verdict in their run of approvals
1436 since their last non-approving one has substance (:func:`verdict_substance`) —
1437 a thin verdict followed by a real one is accepted, because the later comment is
1438 the review (#926). A reviewer who requested changes and then approved counts; a
1439 thin approval cannot overturn their own substantive rejection.
1441 A verdict pinned to any other head is not read at all: a rejection of an older head
1442 is answered by the commits that moved it, and the reviewer's next verdict.
1443 """
1444 ordered = sorted(enumerate(items), key=lambda pair: (_posted_at(pair[1]), pair[0]))
1445 by_key: dict[str, list[tuple[dict[str, Any], str]]] = {}
1446 for _, item in ordered:
1447 if not _is_trusted_source(item, enforced=enforced):
1448 continue
1449 body = _body(item)
1450 if not _is_review_verdict_body(body):
1451 continue
1452 if not _matches_head(item, body, head_sha, covered_heads):
1453 continue
1454 by_key.setdefault(_reviewer_key(item, body), []).append((item, body))
1455 accepted: set[str] = set()
1456 vendors: dict[str, str | None] = {}
1457 insubstantial: list[tuple[str, str]] = []
1458 not_approved: list[tuple[str, str | None, str]] = []
1459 for key, verdicts in by_key.items():
1460 latest_item, latest_body = verdicts[-1]
1461 token = review_verdict_token(latest_body)
1462 if token not in APPROVING_VERDICTS:
1463 not_approved.append((key, token, _verdict_head(latest_item, latest_body)))
1464 continue
1465 approvals: list[str] = []
1466 for _, body in reversed(verdicts):
1467 if not verdict_approves(body):
1468 break
1469 approvals.append(body)
1470 substance = [verdict_substance(body, pr_title=pr_title) for body in approvals]
1471 if not any(ok for ok, _ in substance):
1472 insubstantial.extend((key, reason) for _, reason in reversed(substance))
1473 continue
1474 accepted.add(key)
1475 vendor = _fields(verdicts[0][1]).get("vendor")
1476 vendors[key] = vendor.lower() if vendor else None
1477 return _ReviewTally(
1478 accepted=frozenset(accepted),
1479 vendors=vendors,
1480 insubstantial=tuple(insubstantial),
1481 not_approved=tuple(not_approved),
1482 )
1485def _review_evidence_keys_and_rejections(
1486 items: list[dict[str, Any]],
1487 *,
1488 head_sha: str | None = None,
1489 covered_heads: Collection[str] = (),
1490 enforced: bool = True,
1491 pr_title: str = "",
1492) -> tuple[set[str], list[tuple[str, str]]]:
1493 """Accepted reviewer keys, and the (key, reason) pairs refused for substance.
1495 Rejections are returned rather than dropped so the gate can *hold with a
1496 reason* — a verdict silently not counted would surface as "missing
1497 review-verdict-2", which sends the operator looking for a comment that is
1498 right there (#926). A reviewer whose standing verdict does not approve is in
1499 neither: :func:`_verdict_not_approved_findings` reports them (#1426).
1500 """
1501 tally = _review_tally(
1502 items, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced, pr_title=pr_title
1503 )
1504 return set(tally.accepted), list(tally.insubstantial)
1507def _review_vendor_provenance(
1508 items: list[dict[str, Any]],
1509 *,
1510 head_sha: str | None = None,
1511 covered_heads: Collection[str] = (),
1512 enforced: bool = True,
1513) -> dict[str, str | None]:
1514 """Map each accepted review-verdict reviewer-key to its declared vendor.
1516 The value is the lower-cased ``vendor:`` provenance for that verdict, or
1517 ``None`` when the verdict carries no vendor field. Keys are exactly
1518 :func:`_review_evidence_keys`, so duplicate reviewer-keys collapse to one
1519 entry (idempotent re-posts do not inflate the vendor set), and a reviewer
1520 whose verdict does not count lends the panel no vendor either (#1426).
1521 """
1522 return dict(
1523 _review_tally(
1524 items, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced
1525 ).vendors
1526 )
1529def _verdict_not_approved_findings(
1530 items: list[dict[str, Any]],
1531 *,
1532 head_sha: str | None,
1533 covered_heads: Collection[str] = (),
1534 enforced: bool,
1535) -> list[dict[str, Any]]:
1536 """One finding per reviewer whose standing verdict at the head does not approve (#1426).
1538 ``major`` when the gate is enforced, so a rejection **holds** the merge with its
1539 reviewer and head named — ``review-verdict-not-approved: alice requests changes at
1540 <head>`` in :func:`refusal_reason` — instead of merely not counting, where another
1541 reviewer's approval could make up the number. ``minor`` otherwise, mirroring the
1542 other content findings.
1543 """
1544 tally = _review_tally(items, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced)
1545 return [
1546 {
1547 "id": VERDICT_NOT_APPROVED_FINDING,
1548 "severity": "major" if enforced else "minor",
1549 "kind": "review",
1550 "message": _not_approved_message(key, token, head),
1551 }
1552 for key, token, head in sorted(tally.not_approved, key=lambda entry: entry[0])
1553 ]
1556def distinct_vendor_check(
1557 vendors: Sequence[str | None],
1558 *,
1559 required_count: int,
1560) -> dict[str, Any]:
1561 """Pure vendor-distinctness check over review-verdict provenance.
1563 ``vendors`` is one entry per accepted review verdict: the declared vendor, or
1564 ``None`` when the verdict carries no vendor provenance. The check passes only
1565 when at least ``required_count`` verdicts each declare a vendor and those
1566 vendors are all distinct. It fails when a required verdict is missing vendor
1567 provenance, or when two required verdicts share a vendor.
1569 Returns ``{ok, reason, duplicated, missing_provenance}``. No I/O — fully
1570 unit-testable. A non-positive ``required_count`` always passes (nothing to
1571 require).
1572 """
1573 if required_count <= 0:
1574 return {"ok": True, "reason": None, "duplicated": [], "missing_provenance": 0}
1575 present = [vendor for vendor in vendors if vendor]
1576 missing = len(vendors) - len(present)
1577 seen: set[str] = set()
1578 duplicated: list[str] = []
1579 for vendor in present:
1580 if vendor in seen and vendor not in duplicated:
1581 duplicated.append(vendor)
1582 seen.add(vendor)
1583 if len(present) < required_count:
1584 return {
1585 "ok": False,
1586 "reason": "missing vendor provenance on required review verdict(s)",
1587 "duplicated": duplicated,
1588 "missing_provenance": missing,
1589 }
1590 if duplicated:
1591 return {
1592 "ok": False,
1593 "reason": f"review verdicts share a vendor: {', '.join(sorted(duplicated))}",
1594 "duplicated": sorted(duplicated),
1595 "missing_provenance": missing,
1596 }
1597 return {"ok": True, "reason": None, "duplicated": [], "missing_provenance": missing}
1600def panel_vendor_check(
1601 vendors: Sequence[str | None],
1602 *,
1603 required_count: int,
1604 minimum_vendors: int,
1605) -> dict[str, Any]:
1606 """Cross-vendor check for a **jury panel**, whose size the panel sets (#1015).
1608 :func:`distinct_vendor_check` asks for one distinct vendor per required
1609 verdict, which is the right question for a bench keel staffs: keel chose the
1610 seats, so keel can insist each one is a different vendor. It is the wrong
1611 question for a panel, where the *panel* chose the seats and three ballots from
1612 two vendors is a legitimate cross-vendor review — the same shape
1613 :data:`keel.ship.MINIMUM_JURY_VENDORS` already accepts as a gating jury.
1615 So the panel is held to the jury's own rule instead: every required ballot
1616 must declare a vendor, and the ballots together must span at least
1617 ``minimum_vendors`` distinct ones. A whole panel from one vendor is one
1618 opinion N times and fails, which is precisely what the strict check would
1619 have caught — the relaxation is only in *how many* distinct vendors are
1620 demanded, never in whether provenance is required at all.
1622 Returns the same ``{ok, reason, duplicated, missing_provenance}`` shape as
1623 :func:`distinct_vendor_check`, so a caller renders one finding either way.
1624 """
1625 if required_count <= 0:
1626 return {"ok": True, "reason": None, "duplicated": [], "missing_provenance": 0}
1627 present = [vendor for vendor in vendors if vendor]
1628 missing = len(vendors) - len(present)
1629 distinct = sorted(set(present))
1630 duplicated = sorted({vendor for vendor in present if present.count(vendor) > 1})
1631 if len(present) < required_count:
1632 return {
1633 "ok": False,
1634 "reason": "missing vendor provenance on required review verdict(s)",
1635 "duplicated": duplicated,
1636 "missing_provenance": missing,
1637 }
1638 if len(distinct) < minimum_vendors:
1639 return {
1640 "ok": False,
1641 "reason": (
1642 f"jury panel of {len(present)} ballot(s) spans "
1643 f"{len(distinct)} distinct vendor(s), below the minimum of {minimum_vendors}"
1644 ),
1645 "duplicated": duplicated,
1646 "missing_provenance": missing,
1647 }
1648 return {"ok": True, "reason": None, "duplicated": duplicated, "missing_provenance": missing}
1651def _label_values(labels: Sequence[str] | None, prefix: str) -> list[str]:
1652 """Lower-cased values of every ``<prefix><value>`` label, blanks dropped."""
1653 values: list[str] = []
1654 for label in labels or ():
1655 if not isinstance(label, str) or not label.startswith(prefix):
1656 continue
1657 value = label[len(prefix) :].strip().lower()
1658 if value:
1659 values.append(value)
1660 return values
1663def agent_label_vendors(labels: Sequence[str] | None) -> list[str]:
1664 """Return the lower-cased vendor slugs from every ``agent:<vendor>`` label.
1666 A blank vendor (a bare ``agent:`` label) is ignored. Order is preserved and
1667 duplicates are kept so callers can reason about the raw label set; this is a
1668 pure helper with no I/O.
1669 """
1670 return _label_values(labels, AGENT_LABEL_PREFIX)
1673def model_label_bases(labels: Sequence[str] | None) -> list[str]:
1674 """Return the lower-cased base slugs from every ``model:<base>`` label.
1676 The mirror of :func:`agent_label_vendors` for the second half of keel's
1677 attribution vocabulary. Same conventions: blanks dropped, order and duplicates
1678 preserved, no I/O.
1679 """
1680 return _label_values(labels, MODEL_LABEL_PREFIX)
1683def ledger_implementer(ledger_record: dict[str, Any] | None) -> str | None:
1684 """Return the raw ``actors.implementer`` string from a ship_run record, or ``None``.
1686 The full ``vendor`` / ``vendor:model`` value, not just the vendor half — the
1687 vocabulary check needs the model too. Blank/absent reads as ``None``. Pure.
1688 """
1689 if not isinstance(ledger_record, dict):
1690 return None
1691 actors = ledger_record.get("actors")
1692 implementer = actors.get("implementer") if isinstance(actors, dict) else None
1693 if not isinstance(implementer, str) or not implementer.strip():
1694 return None
1695 return implementer.strip()
1698def ledger_implementer_vendor(ledger_record: dict[str, Any] | None) -> str | None:
1699 """Return the implementer's vendor slug from a ship_run ``ledger_record``.
1701 The ledger stores the effective implementer as a codename or ``vendor:model``
1702 string under ``actors.implementer``; the vendor is the part before the first
1703 ``:``. Returns ``None`` when no record, no implementer, or a blank implementer
1704 is recorded so the cross-check can degrade to presence-only. Pure — no I/O.
1705 """
1706 implementer = ledger_implementer(ledger_record)
1707 if implementer is None:
1708 return None
1709 vendor, _ = agents.split_delegate(implementer)
1710 vendor = vendor.strip().lower()
1711 return vendor or None
1714def attribution_check(
1715 labels: Sequence[str] | None,
1716 *,
1717 implementer_vendor: str | None = None,
1718) -> dict[str, Any]:
1719 """Pure attribution-label check over a PR's labels and the ledger implementer.
1721 Two layers, both fail-closed only on a real contradiction:
1723 * **Presence** — at least one non-blank ``agent:<vendor>`` label must exist.
1724 Missing one is a ``missing-label`` finding.
1725 * **Cross-check** — when ``implementer_vendor`` is known (a ship_run record
1726 recorded an implementer), one of the PR's ``agent:*`` vendors must match it.
1727 A mismatch is a ``vendor-mismatch`` finding. When ``implementer_vendor`` is
1728 ``None`` (no record / no implementer) only the presence layer runs, so PRs
1729 that predate attribution recording are not broken.
1731 Returns ``{ok, reason, label_vendors, implementer_vendor}``. No I/O.
1732 """
1733 label_vendors = agent_label_vendors(labels)
1734 implementer = implementer_vendor.strip().lower() if implementer_vendor else None
1735 if not label_vendors:
1736 return {
1737 "ok": False,
1738 "reason": "missing-label",
1739 "label_vendors": label_vendors,
1740 "implementer_vendor": implementer,
1741 }
1742 if implementer is not None and implementer not in label_vendors:
1743 return {
1744 "ok": False,
1745 "reason": "vendor-mismatch",
1746 "label_vendors": label_vendors,
1747 "implementer_vendor": implementer,
1748 }
1749 return {
1750 "ok": True,
1751 "reason": None,
1752 "label_vendors": label_vendors,
1753 "implementer_vendor": implementer,
1754 }
1757def attribution_fix_command(
1758 pr: int | str,
1759 *,
1760 need_agent: bool,
1761 need_model: bool,
1762 implementer: str | None = None,
1763) -> str:
1764 """The exact ``gh pr edit`` command that supplies the missing attribution labels (#1504).
1766 Only the missing halves are added, so running it never stacks a second ``agent:*`` on a
1767 PR that already has one. When the implementer is known (``vendor`` or ``vendor:model``)
1768 the real labels from :func:`keel.agents.attribution_from_implementer` are written;
1769 otherwise a quoted ``'agent:<vendor>'`` / ``'model:<base>'`` placeholder stands in
1770 (``keel attribution`` prints the real values). Pure.
1771 """
1772 known = agents.attribution_from_implementer(implementer) or {}
1773 parts = ["gh", "pr", "edit", str(pr)]
1774 # Quoted: an unquoted `agent:<vendor>` is a shell redirection and the line would not run.
1775 if need_agent:
1776 label = known.get("agent_label") or f"{AGENT_LABEL_PREFIX}<vendor>"
1777 parts += ["--add-label", shlex.quote(label)]
1778 if need_model:
1779 label = known.get("model_label") or f"{MODEL_LABEL_PREFIX}<base>"
1780 parts += ["--add-label", shlex.quote(label)]
1781 return " ".join(parts)
1784def early_attribution_check(
1785 labels: Sequence[str] | None,
1786 *,
1787 pr: int | str,
1788 implementer: str | None = None,
1789) -> dict[str, Any]:
1790 """Attribution check run by ``keel review`` / ``keel ship`` before CI is awaited (#1504).
1792 The same presence rule :func:`attribution_check` applies at ``evidence-verify`` /
1793 ``keel merge`` (an ``agent:<vendor>`` label), moved to where the operator can still
1794 cheaply fix it. The vendor cross-check is deliberately **not** repeated: it needs the
1795 ledger record, which only the merge gate holds.
1797 A missing ``agent:`` label is an ``error`` (the caller refuses). A missing ``model:``
1798 label is only a ``warning``: ``keel attribution`` itself writes no ``model:`` label when
1799 the implementer ran without a recorded model, so refusing would break PRs keel
1800 produced. Returns ``{ok, errors, warnings, fix_command}``; ``fix_command`` is ``None``
1801 when nothing is missing. No I/O.
1802 """
1803 need_agent = not agent_label_vendors(labels)
1804 need_model = not model_label_bases(labels)
1805 errors: list[str] = []
1806 warnings: list[str] = []
1807 if need_agent:
1808 errors.append("PR is missing a mandatory agent:<vendor> attribution label.")
1809 if need_model:
1810 warnings.append("PR has no model:<base> attribution label.")
1811 fix = (
1812 attribution_fix_command(
1813 pr, need_agent=need_agent, need_model=need_model, implementer=implementer
1814 )
1815 if need_agent or need_model
1816 else None
1817 )
1818 return {"ok": not errors, "errors": errors, "warnings": warnings, "fix_command": fix}
1821def attribution_vocabulary_check(
1822 labels: Sequence[str] | None,
1823 *,
1824 implementer: str | None,
1825) -> dict[str, Any]:
1826 """Check a PR's attribution labels against keel's own vocabulary (#1013).
1828 :func:`attribution_check` compares the label's *vendor* with the ledger's
1829 *vendor*. That is a comparison of two hand-written strings: when the host wrote
1830 ``agent:gemini`` on the PR **and** ``gemini:gemini-3.8-flash-high`` into the
1831 ledger, the two agreed and the gate passed — while keel's own vocabulary for that
1832 run is ``agent:agy`` / ``model:gemini-3``. This check closes that hole by deriving
1833 the expected labels from :func:`keel.agents.attribution` instead of comparing the
1834 prose to itself.
1836 Only labels the PR actually carries are judged: a missing ``agent:`` label is
1837 :func:`attribution_check`'s ``missing-label`` finding and is not repeated here,
1838 and a ledger implementer with no model (``claude``) yields no expected
1839 ``model:`` label, so ``model:`` labels are left alone in that case.
1841 Returns ``{ok, checked, reason, expected, actual, implementer}``. ``checked`` is
1842 ``False`` when there was nothing to compare against — no ledger record, no
1843 recorded implementer — so a caller can tell "agrees" from "could not tell".
1844 Pure — no I/O.
1845 """
1846 expected = agents.attribution_from_implementer(implementer)
1847 actual = {
1848 "agent_labels": agent_label_vendors(labels),
1849 "model_labels": model_label_bases(labels),
1850 }
1851 if expected is None:
1852 return {
1853 "ok": True,
1854 "checked": False,
1855 "reason": "no-implementer",
1856 "expected": None,
1857 "actual": actual,
1858 "implementer": None,
1859 }
1860 recorded = implementer.strip() if isinstance(implementer, str) else None
1861 result = {
1862 "ok": True,
1863 "checked": True,
1864 "reason": None,
1865 "expected": dict(expected),
1866 "actual": actual,
1867 "implementer": recorded,
1868 }
1869 expected_agent = expected["agent_label"][len(AGENT_LABEL_PREFIX) :]
1870 expected_model = expected["model_label"]
1871 if actual["agent_labels"] and expected_agent not in actual["agent_labels"]:
1872 result["ok"] = False
1873 result["reason"] = "agent-label"
1874 return result
1875 if expected_model is not None:
1876 base = expected_model[len(MODEL_LABEL_PREFIX) :]
1877 if actual["model_labels"] and base not in actual["model_labels"]:
1878 result["ok"] = False
1879 result["reason"] = "model-label"
1880 return result
1883def _attribution_vocabulary_finding(
1884 *,
1885 pr_labels: Sequence[str] | None,
1886 enforced: bool,
1887 ledger_record: dict[str, Any] | None,
1888) -> dict[str, Any] | None:
1889 """Return the blocking ``attribution-vocabulary`` finding, or ``None``.
1891 Skips — never fails — when the gate is inactive, when labels were not fetched, or
1892 when no ledger record named an implementer: the check needs a recorded implementer
1893 to derive the expected labels from, and refusing a PR because keel could not read
1894 its own ledger would be a fail-closed rule with nothing behind it.
1895 """
1896 if not enforced or pr_labels is None:
1897 return None
1898 result = attribution_vocabulary_check(
1899 pr_labels,
1900 implementer=ledger_implementer(ledger_record),
1901 )
1902 if result["ok"]:
1903 return None
1904 expected = result["expected"]
1905 labels = ", ".join(
1906 label for label in (expected["agent_label"], expected["model_label"]) if label
1907 )
1908 observed = ", ".join(
1909 [
1910 *(f"{AGENT_LABEL_PREFIX}{value}" for value in result["actual"]["agent_labels"]),
1911 *(f"{MODEL_LABEL_PREFIX}{value}" for value in result["actual"]["model_labels"]),
1912 ]
1913 )
1914 return {
1915 "id": "attribution-vocabulary",
1916 "severity": "major",
1917 "kind": "attribution",
1918 "message": (
1919 f"PR attribution labels ({observed}) are not keel's vocabulary for ledger "
1920 f"implementer {result['implementer']!r}. Expected: {labels}. "
1921 "Obtain labels from `keel attribution` instead of composing them by hand."
1922 ),
1923 }
1926def _unarmed_finding(
1927 *,
1928 enforced: bool,
1929 dry_run: bool,
1930 require_armed: bool,
1931 waived: bool,
1932) -> dict[str, Any] | None:
1933 """Return a blocking finding when the gate was never armed, else ``None``.
1935 Opt-in via ``require_armed`` so existing callers keep today's behavior. An
1936 unarmed gate derives no requirements, so without this the report passes
1937 having verified nothing — indistinguishable from a genuine pass. Skipped
1938 under ``dry_run``, where producing no evidence is the expected outcome, and
1939 when ``waived``: an operator disarming the gate on purpose is the sanctioned
1940 way out, and the whole point is to separate that from arming by accident.
1941 """
1942 if not require_armed or dry_run or enforced or waived:
1943 return None
1944 return {
1945 "id": "gate-unarmed",
1946 "severity": "major",
1947 "kind": "arming",
1948 "message": (
1949 "Evidence gate is not armed, so no requirements were checked. Arm it via ship "
1950 "provenance (the keel.ship-provenance.v1 comment a live run posts on its PR, a "
1951 "ship branch, a posted review verdict, the ship-run ledger, or the gate label), "
1952 "or disarm deliberately with the operator waiver label."
1953 ),
1954 }
1957def _attribution_finding(
1958 *,
1959 pr_labels: Sequence[str] | None,
1960 enforced: bool,
1961 ledger_record: dict[str, Any] | None,
1962) -> dict[str, Any] | None:
1963 """Return a blocking attribution finding when the gate is active, else ``None``.
1965 Only runs when the evidence gate is active (``enforced``) *and* PR labels were
1966 actually fetched (``pr_labels is not None``): the presence check is cheap and
1967 default-on, while the vendor cross-check engages only when the ledger recorded
1968 an implementer vendor. Degrades gracefully — labels not available skips the
1969 check entirely, no record means presence-only, and a gate-inactive run skips
1970 the check (back-compat with callers that never pass labels).
1971 """
1972 if not enforced or pr_labels is None:
1973 return None
1974 implementer_vendor = ledger_implementer_vendor(ledger_record)
1975 result = attribution_check(pr_labels, implementer_vendor=implementer_vendor)
1976 if result["ok"]:
1977 return None
1978 if result["reason"] == "missing-label":
1979 message = "PR is missing a mandatory agent:<vendor> attribution label."
1980 else:
1981 message = (
1982 "PR agent:<vendor> attribution "
1983 f"({', '.join(result['label_vendors'])}) does not match the ship_run "
1984 f"ledger implementer vendor ({result['implementer_vendor']})."
1985 )
1986 return {
1987 "id": "attribution-label",
1988 "severity": "major",
1989 "kind": "attribution",
1990 "message": message,
1991 }
1994def _attribution_model_finding(
1995 *,
1996 pr_labels: Sequence[str] | None,
1997 enforced: bool,
1998) -> dict[str, Any] | None:
1999 """Return the advisory ``attribution-model-label`` finding, or ``None`` (#1504).
2001 ``minor``, never blocking: a missing ``model:<base>`` label was never flagged before,
2002 and keel's own attribution writes none for an implementer with no recorded model, so
2003 making it blocking would fail PRs that carry exactly what keel produced. Same gating as
2004 :func:`_attribution_finding`: only when the gate is enforced and labels were fetched.
2005 """
2006 if not enforced or pr_labels is None or model_label_bases(pr_labels):
2007 return None
2008 return {
2009 "id": "attribution-model-label",
2010 "severity": "minor",
2011 "kind": "attribution",
2012 "message": (
2013 "PR has no model:<base> attribution label; add one "
2014 "(`keel attribution --vendor <vendor> --model <model>` prints it)."
2015 ),
2016 }
2019#: A file named without its directory — ``evidence.py``, ``CHANGELOG.md``.
2020#:
2021#: The path anchors read *any* one-to-five-character extension, because a
2022#: directory has already proved the token is a path. With no directory nothing
2023#: is left to carry that proof: ``Node.js``, ``Next.js``, ``Vue.js`` and
2024#: ``D3.js`` are spelled exactly like ``evidence.py``, and in prose a product
2025#: is named far more often than a bare file is. Listing the extension cannot
2026#: separate them — ``.js`` is a real source extension, and dropping it would
2027#: refuse real reviews to refuse four product names — so this form does not
2028#: anchor a verdict by itself. It *corroborates*:
2029#: see :data:`_VERDICT_CORROBORATORS`.
2030_VERDICT_SOURCE_FILE = re.compile(
2031 r"\b[\w-]+\.(?:py|pyi|md|rst|txt|ya?ml|toml|json|cfg|ini|lock"
2032 r"|sh|js|mjs|cjs|ts|tsx|jsx|rb|go|rs|css|html?|svg|sql)\b"
2033)
2035#: ``module.symbol`` / ``Class.method`` — the dotted identifier a review writes
2036#: for something it is not calling (#1106), read only when the token carries a
2037#: mark prose does not use: an underscore, an internal capital, a run of
2038#: capitals, or a capitalised segment.
2039#:
2040#: **A dot is not evidence, and neither is a capital.** Requiring the mark
2041#: stops ``github.com`` and ``pypi.org``. It does not stop ``GitHub.com``,
2042#: ``GitLab.com``, ``OpenAI.com`` or ``SourceForge.net``, because a capitalised
2043#: first segment is precisely the mark ``Config.parse`` carries — the two are
2044#: one shape, and no sixth character class tells them apart. That is why this
2045#: form corroborates rather than anchors (:data:`_VERDICT_CORROBORATORS`): the
2046#: alternative was a list of hostnames to refuse, which cannot be finished,
2047#: because anyone can register the next one.
2048#:
2049#: Both sides of the dot still need two characters, so "e.g." and "i.e." are not
2050#: identifiers whatever else they carry.
2051_VERDICT_DOTTED_SYMBOL = re.compile(
2052 r"\b(?=[\w.]*(?:_|[a-z0-9][A-Z]|[A-Z]{2}|[A-Z][a-z]))"
2053 r"[A-Za-z_]\w+(?:\.[A-Za-z_]\w+)+"
2054)
2056#: A **bare** identifier: ``cache_key``, ``_prompt_mode``, ``__post_init__``.
2057#:
2058#: **The joining underscore is the whole rule** — two lowercase alphanumeric
2059#: segments with an underscore between them. Lowercase deliberately: `My_Thing`
2060#: is prose with a connector, and CamelCase is read only when dotted or
2061#: backticked, for the reason :data:`_VERDICT_DOTTED_SYMBOL` gives.
2062#:
2063#: Underscores that merely wrap a name are
2064#: decoration, so a lone ``_private`` or ``__dunder__`` is *not* read; it is
2065#: ``post_init`` inside ``__post_init__`` that matches.
2066#:
2067#: The joining underscore is a mark English prose and product names do not use,
2068#: and it is the only lexical difference between a symbol and a capitalised
2069#: noun: ``GitHub`` and ``GitLab`` are CamelCase in exactly the way
2070#: ``JuryConfig`` is, so reading CamelCase let "The GitHub and GitLab side of
2071#: this is unchanged" clear a floor of two while naming nothing in the change.
2072#: No pattern separates those two, and a list of product names to refuse would
2073#: need a new entry every time a reviewer mentions a new product. CamelCase
2074#: symbols are still read everywhere they are written with a dot or in
2075#: backticks, which is how a class is normally named.
2076_VERDICT_BARE_IDENTIFIER = re.compile(r"\b_*[a-z0-9]+(?:_[a-z0-9]+)+_*\b")
2078#: A concrete thing a review can point at, on its own — a ``path:line``, a
2079#: path, a backticked symbol, or a called identifier. Presence of *structure*,
2080#: never a judgement about whether the review was good — the same line
2081#: ai-jury's ``emitted_findings_block()`` draws.
2082#:
2083#: What these four have and :data:`_VERDICT_CORROBORATORS` do not is that the
2084#: **punctuation is the author pointing**. Backticks, a directory separator, a
2085#: ``:42`` and a ``()`` are marks a reviewer types deliberately at a thing;
2086#: none of them appear around a product name in ordinary prose. A bare dotted
2087#: token is just a token with a dot in it.
2088_VERDICT_ANCHORS = re.compile(
2089 r"[\w./-]+\.[A-Za-z0-9]{1,5}:\d+|" # path/to/file.py:42
2090 r"[\w-]+/[\w./-]+\.[A-Za-z0-9]{1,5}\b|" # src/keel/thing.py
2091 r"`[^`\n]{2,}`|" # `a_symbol`, `--a-flag`
2092 r"\b\w+\.\w+\(\)" # module.function()
2093)
2095#: The unbackticked forms — a bare filename, a bare dotted token, a bare
2096#: identifier — each of which is *also* how something that is not a symbol gets
2097#: written. Two of them anchor a verdict; one does not (#1106).
2098#:
2099#: **This is a shape, not a lexicon, and that is the point.** Three rounds of
2100#: widening the anchor set were each undone by a token that is not a symbol:
2101#: ``claude.ai`` in a tool footer, ``GitHub``/``GitLab`` as bare CamelCase,
2102#: then ``Node.js`` and ``GitHub.com``. Every one was answered by refining a
2103#: character class, and every refinement admitted the next such token, because
2104#: ``Node.js`` and ``evidence.py`` are the same shape and so are ``GitHub.com``
2105#: and ``Config.parse``. No character class ends that sequence and no blocklist
2106#: of products or hostnames can be finished. What separates a review from a
2107#: mention is not the spelling of one token but how many the verdict has: prose
2108#: mentions a product in passing; a review that walked the change names more
2109#: than one thing. A clause that says an act of review happened
2110#: (:data:`_VERDICT_CHECKED_CLAUSE`) is the one exception, and only for
2111#: "checked": see there for why the other verbs could not be given the same
2112#: latitude, and why judging their object by this same test made the branch
2113#: inert.
2114#:
2115#: Measured over every verdict posted across keel and ai-jury: corroboration
2116#: refuses all four ``*.js`` product names and all four capitalised hostnames,
2117#: costs 30 of the 141 verdicts #1106 recovered — 22 of those 30 being the
2118#: default template's "Scope reviewed:" line with a single token dropped into
2119#: it — and regresses nothing that passed before #1106. Dropping the two bare
2120#: forms outright instead, the fallback, refuses the same eight strings but
2121#: keeps only 98 of the 141 and loses a review naming ``evidence.py`` and
2122#: ``contracts.py``, which is a review.
2123_VERDICT_CORROBORATORS = (
2124 _VERDICT_SOURCE_FILE, # evidence.py, CHANGELOG.md
2125 _VERDICT_DOTTED_SYMBOL, # cache.cache_key, JuryConfig.__post_init__
2126 _VERDICT_BARE_IDENTIFIER, # collect_static_hints, _prompt_mode
2127)
2129#: How many distinct corroborating tokens stand in for an anchor. At one, the
2130#: corpus says the rule readmits 22 of the 75 ``Reviewed <title>: <affirmation>``
2131#: rubber stamps #926 is named for; at two it admits none of them.
2132_VERDICT_CORROBORATION_FLOOR = 2
2134#: What may sit between an act-of-review verb and its object: an optional colon,
2135#: and an optional break onto a bullet, because "Checked:\n- …" is the same
2136#: clause with a list under it and refusing it was punctuation pedantry (#1106).
2137#: Layout only — it says nothing about what the object has to be.
2138_VERDICT_CLAUSE_TAIL = r"[ \t]*:?[ \t]*(?:\r?\n[ \t]*[-*][ \t]*)?"
2140#: One sentence's worth of characters. A sentence ends at `.!?;` or an ellipsis
2141#: **followed by space or line end** — a period inside `evidence.py` is not a
2142#: sentence end, and the filename has to survive inside an object, so "Checked
2143#: evidence.py and contracts.py" is no longer truncated at the first dot —
2144#: the punctuation pedantry this change is about. Only
2145#: :data:`_VERDICT_CHECKED_CLAUSE` reads this; the second clause that did was
2146#: removed with the verb widening.
2147_VERDICT_SENTENCE = r"(?:[^.!?;\u2026\n]|[.!?;\u2026](?!\s|$))"
2149#: The escape hatch the issue insists on: a genuinely clean review must stay
2150#: expressible. "Checked X, Y and Z; found nothing" is a real review outcome and
2151#: must not be forced to invent an anchor. Its object stays free-form, as it has
2152#: been since #926: 35 verdicts in the corpus pass on this clause and nothing
2153#: else, saying things like "Checked the formula syntax, the version URL and the
2154#: checksum placeholder" — English objects, naming no symbol.
2155#:
2156#: The object now ends at a *sentence* rather than at any period, which is a
2157#: change from `[^.\n]{8,}`: it buys a filename, since `evidence.py` no longer
2158#: truncates to `evidence`, and it costs a trailing `;`, `!` or `?` clause —
2159#: "Checked config; found nothing of concern in the rest" passes on `main` and
2160#: is refused here. No verdict in the 1,421 measured is written that way, which
2161#: is why the corpus shows no regression; that is a weaker claim than "takes
2162#: nothing away" and is the one the evidence supports.
2163#:
2164#: **#1106 tried to widen this to traced/read/ran/inspected/verified and the
2165#: widening turned out inert.** Those verbs could not keep a free-form object —
2166#: "Read the whole diff and everything looks correct" is the #926 receipt with a
2167#: synonym at the front — so their object was made to name something. But
2168#: "names something" is the same test the whole prose already takes, and the
2169#: object is part of the prose, so a verdict that satisfied the clause had
2170#: always satisfied the anchor check first: across 1,421 verdicts the branch
2171#: decided **zero** of them. Dead code with a docstring explaining what it did
2172#: is worse than neither, so it is gone. Widening the vocabulary needs the
2173#: object to be judged by something other than the prose test, which #1106 did
2174#: not find.
2175_VERDICT_CHECKED_CLAUSE = re.compile(
2176 rf"\bchecked\b{_VERDICT_CLAUSE_TAIL}{_VERDICT_SENTENCE}{{8,}}", re.IGNORECASE
2177)
2179#: Below this share of novel words, the prose is the PR title said again. The
2180#: observed shape was `Reviewed <title>: <generic affirmation>` — 75 of 75
2181#: verdicts across 25 PRs (#926).
2182_VERDICT_NOVELTY_FLOOR = 0.35
2184_WORD = re.compile(r"[a-z0-9]+")
2187def _verdict_prose(body: str) -> str:
2188 """The verdict's own words: header block, marker line and HTML comments removed.
2190 The marker is matched as a *whole line*, never as a substring. It is rendered
2191 on its own bare line, so the header slice above has already dropped it; a
2192 substring test therefore only ever reached prose that *quotes* the marker,
2193 and deleted it. That cost the review of #1119 its entire scope — a
2194 1,700-character line naming four files was dropped because it named the
2195 marker it was documenting, and the verdict was then refused for naming
2196 nothing (#1120).
2198 This is the rule :func:`marker_in_header` already states earlier in this
2199 module: a marker below the header is prose, and ``MARKER in body`` cannot
2200 tell the two apart (#1026). This function was the one place that substring
2201 test survived.
2202 """
2203 lines = body.splitlines()
2204 start = 0
2205 for index, raw in enumerate(lines):
2206 if not raw.strip():
2207 start = index + 1
2208 break
2209 kept = [
2210 line
2211 for line in lines[start:]
2212 if line.strip()
2213 and not line.strip().startswith("<!--")
2214 and line.strip() != REVIEW_VERDICT_MARKER
2215 ]
2216 return "\n".join(kept)
2219def _verdict_corroborators(text: str) -> set[str]:
2220 """The distinct unbackticked tokens in ``text``, counting each one once.
2222 A written token is one piece of evidence however many patterns read it:
2223 ``cache.cache_key`` is matched by :data:`_VERDICT_DOTTED_SYMBOL` whole and by
2224 :data:`_VERDICT_BARE_IDENTIFIER` as its second half, and counting it twice
2225 would let one token clear a floor that exists to require two. Matches
2226 contained inside a longer match are therefore dropped, and what survives is
2227 deduplicated by text, so a reviewer who names ``evidence.py`` three times has
2228 still named one file.
2229 """
2230 spans = sorted(
2231 (
2232 (match.start(), match.end(), match.group(0))
2233 for pattern in _VERDICT_CORROBORATORS
2234 for match in pattern.finditer(text)
2235 ),
2236 key=lambda span: (span[0], -span[1]),
2237 )
2238 kept: list[tuple[int, int, str]] = []
2239 for span in spans:
2240 if any(span[0] >= start and span[1] <= end for start, end, _ in kept):
2241 continue
2242 kept.append(span)
2243 return {token for _, _, token in kept}
2246def _review_act_clause(prose: str) -> bool:
2247 """Whether ``prose`` says an act of review was performed on something."""
2248 return bool(_VERDICT_CHECKED_CLAUSE.search(prose))
2251def verdict_substance(body: str, *, pr_title: str = "") -> tuple[bool, str]:
2252 """Whether a verdict engages with the diff at all. ``(ok, reason)``.
2254 The evidence gate verified that verdicts *exist* with the right marker, head
2255 SHA and distinct reviewer ids — never that any of them looked at anything. A
2256 verdict engaging with nothing was indistinguishable from one that caught a
2257 blocker, and the record showed what that permits: 75 of 75 verdicts `pass`,
2258 all opening `Reviewed <PR title>: <affirmation>`, across 25 PRs that produced
2259 no review-driven commit between them (#926).
2261 Two mechanical requirements, both content-agnostic beyond structure:
2263 * **An anchor.** A ``path:line``, a path, a backticked symbol or a called
2264 identifier, whose punctuation is the author pointing — or, lacking one,
2265 two distinct unbackticked tokens, because ``Node.js`` and ``evidence.py``
2266 are one shape and only the count separates a mention from a review — or
2267 an act-of-review clause naming what was looked at, because a genuinely
2268 clean review must stay expressible and forcing it to invent a file
2269 reference would make the check worse than nothing.
2270 * **Novelty against the title.** Prose that is substantially the PR title
2271 restated is the observed shape, and it survives the anchor test whenever
2272 the title happens to contain a path.
2274 The two are independent on purpose, and that is what lets the anchor set be
2275 generous (#1106). An anchor asks whether the reviewer pointed at anything;
2276 the novelty floor asks whether the prose is the title said again. A verdict
2277 that names a symbol *and* is otherwise the title restated fails the second
2278 check, so widening the first cannot readmit the #926 shape by itself.
2280 This says nothing about whether a review was *good*. It cannot, and trying
2281 would make the gate a critic. It distinguishes a review from a receipt.
2282 """
2283 prose = _verdict_prose(body)
2284 if not prose.strip():
2285 return False, "verdict has no prose beyond its header"
2287 # ⚡ Bolt Optimization: Use combined compiled regex instead of generator overhead
2288 anchored = bool(_VERDICT_ANCHORS.search(prose)) or (
2289 len(_verdict_corroborators(prose)) >= _VERDICT_CORROBORATION_FLOOR
2290 )
2291 if not anchored and not _review_act_clause(prose):
2292 return False, (
2293 "verdict names nothing concrete — no file, line, symbol, no two "
2294 "of a filename/dotted name/identifier, and no 'checked …' clause, "
2295 "so it cannot be told apart from a receipt"
2296 )
2298 title_words = set(_WORD.findall(pr_title.lower()))
2299 prose_words = _WORD.findall(prose.lower())
2300 if title_words and prose_words:
2301 novel = [word for word in prose_words if word not in title_words]
2302 if len(novel) / len(prose_words) < _VERDICT_NOVELTY_FLOOR:
2303 return False, "verdict is substantially the pull request title restated"
2304 return True, ""
2307def _reviewer_key(item: dict[str, Any], body: str) -> str:
2308 fields = _fields(body)
2309 reviewer = fields.get("reviewer")
2310 if reviewer:
2311 return f"reviewer:{reviewer.lower()}"
2312 user = item.get("user")
2313 if isinstance(user, dict) and isinstance(user.get("login"), str) and user["login"]:
2314 return f"user:{user['login'].lower()}"
2315 digest = hashlib.sha256(body.encode("utf-8")).hexdigest()
2316 return f"body:{digest}"
2319def _matches_head(
2320 item: dict[str, Any],
2321 body: str,
2322 head_sha: str | None,
2323 covered_heads: Collection[str] = (),
2324) -> bool:
2325 """Does this comment answer for ``head_sha``? A blank head means *do not filter*.
2327 **Deliberately not :func:`keel.juryavail.is_pinnable_head`'s rule, and the difference
2328 is worth stating** (#1068). This one filters *evidence items* inside a gate that, with
2329 no head resolved, is head-agnostic from end to end — every review verdict counts, so
2330 holding jury verdicts alone to a head nobody knows would refuse a gate the rest of
2331 which is already unfiltered. Nothing reached through here removes a requirement:
2332 :func:`_review_evidence_keys` and :func:`_review_vendor_provenance` count verdicts
2333 towards one, and :func:`jury_panel_size` feeds ``max(declared, minimum_vendors)``, so a
2334 stale ``panelists:`` can only ever raise the bar.
2336 :func:`jury_participating_vendors` was the exception, and #1069 closed it rather than
2337 changing this predicate: its count *can* downgrade a gating jury to advisory (#1015), so
2338 it now asks :func:`keel.juryavail.is_pinnable_head` for itself before it reads anything.
2339 Its head rule is its own, for the reason a pin's is — it removes a requirement — and the
2340 two readers that do not remove one keep this reading. That is the whole distinction
2341 between the three panel-shaped readers layered here, and it is a property of what each
2342 one's answer can *do*, not of where it is read from.
2344 A *pin* is the case that cannot use this reading, because it does remove requirements —
2345 it takes ``review-verdict-1..3`` off the required set entirely. So the pin
2346 (:func:`keel.juryavail.pin`, read by :func:`keel.cli._shipped_jury_availability`)
2347 refuses a blank head before :func:`panel_verdict_posted` is asked at all, rather than
2348 this predicate changing under the surfaces that need the permissive one.
2349 """
2350 if not head_sha:
2351 return True
2352 # **A head the current one descends from by capture commits alone** answers for it
2353 # too (#1203). The learning is the pull request's last commit, written after review
2354 # and before the merge, so every verdict would otherwise be pinned to a head the
2355 # branch has already moved past. `covered_heads` is never filled from a comment or an
2356 # argument an agent supplies: its only producer is `capture.capture_only_descent`,
2357 # which holds each commit in between to one parent, the landing marker, and exactly
2358 # one path inside the configured sink — so what it admits is a lesson, never code.
2359 accepted = {head_sha, *covered_heads}
2360 fields = _fields(body)
2361 recorded = fields.get("head")
2362 if recorded:
2363 return recorded in accepted
2364 commit_id = item.get("commit_id")
2365 return isinstance(commit_id, str) and commit_id in accepted
2368def _fields(body: str) -> dict[str, str]:
2369 """Parse the header block at the top of a verdict comment, and only that.
2371 Scanning stops at the first line that is not part of a header block —
2372 whether or not a header has been seen yet. #868's second requirement said so
2373 and only the first shipped: the earlier version `continue`d past prose and
2374 blank lines until it found something header-shaped, so fields could be
2375 harvested from anywhere in a comment (#932):
2377 "Some prose line here.\\n\\nhead: 0000000\\nvendor: spoofed\\n"
2378 -> {'head': '0000000', 'vendor': 'spoofed'}
2380 Reachable through :func:`_reviewer_key`, which calls this with no marker
2381 requirement, so a comment whose *prose* contains ``reviewer: someone`` was
2382 keyed to that reviewer. Narrow — it needs a trusted author — and not a live
2383 hole, but it is the residual of the class #868 named, and stopping costs the
2384 real comment format nothing: a verdict's header is its first block.
2385 """
2386 fields: dict[str, str] = {}
2387 started = False
2389 for raw_line in (body or "").splitlines():
2390 line = raw_line.strip()
2391 if not line:
2392 # Leading blank lines are skipped, not treated as the end: a comment
2393 # body routinely begins with a newline, and breaking there would
2394 # reject legitimate verdicts. Once the block has started, a blank
2395 # line ends it — that is the boundary #868 asked for.
2396 if started:
2397 break
2398 continue
2399 started = True
2400 # A marker-only line is the artifact's own header, not a field: skip it and
2401 # keep reading. A line that merely *mentions* a marker is prose, and prose
2402 # ends the block — the #932 boundary this parser exists to hold.
2403 if (
2404 line.startswith("<!--") and line.endswith("-->")
2405 ) or _CLASSIFICATION_MARKERS_SET.issuperset(line.split()):
2406 continue
2407 match = _FIELD_RE.match(line)
2408 if match:
2409 key = match.group("key").lower()
2410 if key not in fields:
2411 fields[key] = match.group("value")
2412 elif not _HEADER_LINE_RE.match(line):
2413 break
2414 return fields
2417def _is_review_verdict_body(body: str) -> bool:
2418 """Whether ``body`` is a review verdict, decided by its header alone (#1026).
2420 The jury and closure exclusions are no longer separate substring tests:
2421 :func:`marker_in_header` yields at most one marker, so a body anchored to the
2422 jury or closure marker simply is not a review verdict, and a review verdict
2423 that *mentions* either one in its prose still is.
2424 """
2425 if _is_ship_assessment(body):
2426 return False
2427 return marker_in_header(body) == REVIEW_VERDICT_MARKER
2430def _has_trusted_review_marker(items: list[dict[str, Any]]) -> bool:
2431 return any(
2432 _is_trusted_source(item, enforced=True)
2433 and marker_in_header(_body(item)) == REVIEW_VERDICT_MARKER
2434 for item in items
2435 )
2438def jury_participating_vendors(
2439 pr_comments: list[dict[str, Any]] | None = None,
2440 pr_reviews: list[dict[str, Any]] | None = None,
2441 *,
2442 head_sha: str | None = None,
2443 covered_heads: Collection[str] = (),
2444 enforced: bool = True,
2445) -> int | None:
2446 """Return the panel size declared by a posted jury verdict, or ``None``.
2448 Reads the ``vendors: <N>`` field from a trusted, head-bound
2449 ``keel.jury-verdict.v1`` comment. This is how the participating-vendor count
2450 reaches a CI evidence check: the run ledger and the jury artifact both live
2451 under the gitignored ``.keel/state/``, so a hosted runner cannot read either,
2452 but PR comments are always visible.
2454 ``None`` means "not declared" — no jury verdict posted, or one that predates
2455 the field — and leaves the jury mode untouched rather than assuming a short
2456 panel. Only a verdict that actually states the count may relax the gate.
2458 When several verdicts qualify, the largest declared count wins: a re-post
2459 correcting an earlier partial run should not be capped by the stale one.
2461 **This one reader is held to an exact head, and its two siblings are not** (#1069).
2462 Alone among the three panel-shaped readers here, this count can *remove* a
2463 requirement: below ``jury.min_vendors`` it downgrades a gating jury to advisory
2464 (:func:`keel.ship.resolve_jury`), which drops ``jury-verdict`` from the required
2465 evidence entirely. So it asks :func:`keel.juryavail.is_pinnable_head` — the same
2466 blank-head predicate the panel pins ask — before it reads a comment at all, and a
2467 run that resolved no head declares nothing rather than inheriting the last verdict
2468 on the pull request. Without the guard, `keel evidence-verify` run offline with no
2469 ``--head-sha`` (its documented default) read a ``vendors: 1`` verdict posted against
2470 an earlier head as this head's and relaxed the gate: a requirement removed by
2471 evidence nobody re-checked. A *mismatched* head was already refused, by
2472 :func:`_matches_head` — that predicate is exact once a head is known, and permissive
2473 only when none is — so the guard closes the blank-head half and nothing else.
2475 :func:`jury_panel_size` and :func:`panel_verdict_posted` deliberately keep the
2476 permissive reading, because neither can relax anything: the first feeds
2477 ``max(declared, minimum_vendors)`` and can only raise the bar, and the second is
2478 already refused on a blank head by its caller, which owns the pin order.
2479 """
2480 if not juryavail.is_pinnable_head(head_sha):
2481 return None
2482 counts = [
2483 parsed
2484 for item in [*(pr_comments or []), *(pr_reviews or [])]
2485 if _is_jury_verdict(item, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced)
2486 if (parsed := _parse_vendor_count(_fields(_body(item)).get("vendors"))) is not None
2487 ]
2488 return max(counts) if counts else None
2491def jury_panel_size(
2492 pr_comments: list[dict[str, Any]] | None = None,
2493 pr_reviews: list[dict[str, Any]] | None = None,
2494 *,
2495 head_sha: str | None = None,
2496 covered_heads: Collection[str] = (),
2497 enforced: bool = True,
2498) -> int | None:
2499 """Return the panel size declared by a posted jury verdict, or ``None`` (#1015).
2501 Reads the ``panelists: <N>`` field off the same comment
2502 :func:`jury_participating_vendors` reads ``vendors:`` from, and for the same
2503 reason: when the jury **is** the review panel, the number of ballots is the
2504 reviewer count the evidence gate must require, and a hosted runner can read
2505 it from nowhere else.
2507 ``None`` means "not declared", which leaves the gate on the contract's floor
2508 rather than requiring nothing. The largest declared count wins, so a re-post
2509 that completes a partial panel raises the requirement instead of being capped
2510 by the stale verdict — the direction that fails closed.
2512 **It does not share that sibling's head rule, and the asymmetry is deliberate**
2513 (#1069). This count reaches :func:`keel.ship._jury_panel_size`, which answers
2514 ``max(declared, minimum_vendors)`` — so a stale or blank-head ``panelists:`` can
2515 only ever *raise* what the tier owes, never remove a requirement. Holding it to an
2516 exact head would refuse a bar-raising reading inside a gate whose other half, with
2517 no head resolved, is already unfiltered (:func:`_matches_head`). ``vendors:`` is the
2518 one that can relax, so ``vendors:`` is the one that is pinned.
2519 """
2520 counts = [
2521 parsed
2522 for item in [*(pr_comments or []), *(pr_reviews or [])]
2523 if _is_jury_verdict(item, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced)
2524 if (parsed := _parse_vendor_count(_fields(_body(item)).get("panelists"))) is not None
2525 ]
2526 return max(counts) if counts else None
2529def panel_verdict_posted(
2530 pr_comments: list[dict[str, Any]] | None = None,
2531 pr_reviews: list[dict[str, Any]] | None = None,
2532 *,
2533 head_sha: str | None = None,
2534 covered_heads: Collection[str] = (),
2535 enforced: bool = True,
2536) -> bool:
2537 """Is a head-pinned jury verdict already on this pull request? (#1066)
2539 Proof that the panel *sat*, from the one place a bare CI runner can read it: the run
2540 ledger and the jury artifact both live under the gitignored ``.keel/state/``, while PR
2541 comments are always visible. A verification surface uses it to pin the contract to what
2542 the ship measured rather than re-measuring the panel on its own machine. It is the
2543 *weaker* of the two pins and speaks only when the run left no ledger record for this
2544 head; :func:`keel.juryavail.pin` owns that order and is the one place it is written.
2546 Distinct from :func:`jury_panel_size`, which answers *how many* ballots and is ``None``
2547 for a verdict predating the ``panelists:`` field. Presence is the weaker question, and
2548 the one that must not depend on an optional field.
2550 **Call this only with a head you actually resolved.** Like every reader here it goes
2551 through :func:`_matches_head`, which reads a blank ``head_sha`` as "do not filter" —
2552 right for counting evidence, wrong for a pin, which is why the caller refuses a blank
2553 head first (:func:`keel.juryavail.is_pinnable_head`).
2555 :func:`jury_participating_vendors` asks that same predicate *itself* rather than
2556 leaving it to a caller (#1069), and the difference is about who the callers are: this
2557 one is read from exactly one place, :func:`keel.juryavail.pin`, which owns the pin
2558 order and so is the right place for the rule; the vendor count is read straight off
2559 ``keel evidence-verify``'s argv-derived head, where there is no single owner to put it
2560 in front of.
2561 """
2562 return any(
2563 _is_jury_verdict(item, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced)
2564 for item in [*(pr_comments or []), *(pr_reviews or [])]
2565 )
2568def shipped_panel_decision(
2569 pr_comments: list[dict[str, Any]] | None = None,
2570 *,
2571 head_sha: str | None = None,
2572 enforced: bool = True,
2573) -> str | None:
2574 """The panel decision **this run** recorded, read back off its closure comment (#1068).
2576 The middle pin, and the one that makes the strongest pin work anywhere. The run's own
2577 ``ship_run`` ledger record outranks a posted jury verdict — a comment records what
2578 somebody put on the pull request, the ledger records what the run *did* — but the
2579 ledger lives under the gitignored ``.keel/state/``, so on a hosted ``evidence-verify``
2580 or ``merge`` there is no record to read and that precedence held on the shipping
2581 workstation and nowhere else. A leftover or collaborator-posted ``keel.jury-verdict.v1``
2582 then answered for a run that had fallen back, and took ``review-verdict-1..3`` off the
2583 required set.
2585 The run's decision is already on the pull request: s11 posts the closure comment keel
2586 renders from that same ledger record, and since #1068 round 6 it carries
2587 :data:`keel.closure.JURY_PANEL_MARKER` beside the human ``Jury panel:`` line. So this
2588 reads the run's own statement from the one place that travels with the pull request.
2590 Three conditions, and each is the same rule its siblings hold to:
2592 * **Trusted author only** (:func:`_is_trusted_source`). keel posts the closure comment
2593 on the operator's behalf, which is exactly the authority a posted jury verdict has —
2594 no more. An untrusted author must not be able to relax the contract *in either
2595 direction*: neither to claim a fallback that drops the panel item, nor to claim the
2596 panel sat.
2597 * **An actual closure comment** (:func:`_has_closure_marker`), so the marker counts only
2598 inside the artifact that renders it — a reviewer quoting the marker while describing
2599 this change is prose, the #1026 rule every marker here is read under.
2600 * **Pinned to this head.** The marker names the head its record was written for and it
2601 must be the head under verification, because a pull request outlives its heads and a
2602 pin removes requirements.
2604 **The latest such comment is the answer, not the first** (#1068 round 7). One head can
2605 be shipped more than once — a re-run, a force-push back onto the same commit, a second
2606 ship on a different machine — and each ship posts its own closure comment. ``pr_comments``
2607 arrives in GitHub's order, oldest first, so this walks the whole list and keeps the last
2608 match: the newest statement wins, which is the same direction
2609 :func:`keel.ledger.latest_ship_run_for_pr` selects the ledger record in, and
2610 :func:`keel.juryavail.pin` ranks the two sources on the premise that they agree about it.
2611 Returning the first match meant an older ``decision=fallback`` outranked the panel-sat
2612 ship that followed it — and round 6 emitted no marker at all for a panel that sat, so the
2613 later run had nothing to outrank the older one *with*. :func:`keel.closure._jury_panel`
2614 now renders ``decision=available`` too, which is what makes last-wins well defined here.
2616 ``None`` for everything else — no comment, an older head, a marker keel did not write —
2617 and ``None`` means *this source is silent*, never a waiver: :func:`keel.juryavail.pin`
2618 then goes on to the posted verdict exactly as it did before.
2619 """
2620 if not head_sha:
2621 return None
2622 latest: str | None = None
2623 for item in pr_comments or []:
2624 if not _is_trusted_source(item, enforced=enforced):
2625 continue
2626 body = _body(item)
2627 if not _has_closure_marker(body):
2628 continue
2629 decision = _jury_panel_decision(body, head_sha)
2630 if decision is not None:
2631 latest = decision
2632 return latest
2635def _jury_panel_decision(body: str, head_sha: str) -> str | None:
2636 """The decision ``body``'s panel marker records for ``head_sha``, or ``None``.
2638 A token parser over one HTML-comment line, not a regex over Markdown: the line is
2639 :func:`keel.closure._jury_panel_marker`'s exact render, so it unwraps with the same
2640 :func:`_unwrap_html_comment` a header marker does and splits into
2641 ``<marker> head=<sha> decision=<value>``. A line that is not that shape is prose and is
2642 skipped, which is why the human sentence above it — which *names* neither field — can
2643 never be mistaken for the record.
2644 """
2645 for raw_line in (body or "").splitlines():
2646 tokens = _unwrap_html_comment(raw_line.strip()).split()
2647 if not tokens or tokens[0] != closure.JURY_PANEL_MARKER:
2648 continue
2649 fields: dict[str, str] = {}
2650 for token in tokens[1:]:
2651 key, _, value = token.partition("=")
2652 # First wins, the convention `_fields` already reads headers under, and a
2653 # token carrying no `=` becomes a valueless key that matches neither field.
2654 fields.setdefault(key, value)
2655 if fields.get("head") == head_sha:
2656 return fields.get("decision")
2657 return None
2660def _parse_vendor_count(raw: str | None) -> int | None:
2661 """Parse a declared vendor count, rejecting anything not a plain non-negative int."""
2662 if raw is None:
2663 return None
2664 try:
2665 parsed = int(raw)
2666 except ValueError:
2667 return None
2668 return parsed if parsed >= 0 else None
2671def _is_jury_verdict(
2672 item: dict[str, Any],
2673 *,
2674 head_sha: str | None = None,
2675 covered_heads: Collection[str] = (),
2676 enforced: bool = True,
2677) -> bool:
2678 """Whether ``item`` is a trusted jury verdict pinned to ``head_sha`` — presence only.
2680 **Deliberately blind to the consensus** (#1429). Its readers here are the panel-shape
2681 checks — :func:`jury_participating_vendors`, :func:`jury_panel_size` and
2682 :func:`panel_verdict_posted` — which size the panel and pin whether it sat. A panel
2683 that sat and rejected the change still sat, with that many vendors and ballots, so a
2684 rejecting verdict must keep answering them: reading the consensus there would turn a
2685 rejection into "no panel", and drop the ballots it owes. Whether the panel *approves*
2686 is a separate question, answered by :func:`_standing_jury_verdict` and
2687 :func:`jury_verdict_approves` for the ``jury-verdict`` requirement.
2688 """
2689 if not _is_trusted_source(item, enforced=enforced):
2690 return False
2691 body = _body(item)
2692 if _is_ship_assessment(body):
2693 return False
2694 return marker_in_header(body) == JURY_VERDICT_MARKER and _matches_head(
2695 item, body, head_sha, covered_heads
2696 )
2699def _standing_jury_verdict(
2700 items: list[dict[str, Any]],
2701 *,
2702 head_sha: str | None = None,
2703 covered_heads: Collection[str] = (),
2704 enforced: bool = True,
2705) -> dict[str, Any] | None:
2706 """The latest trusted jury verdict answering for ``head_sha``, or ``None`` (#1429).
2708 **The latest jury verdict is the panel's word**, ordered as :func:`_review_tally`
2709 orders review verdicts: by when it was posted (:func:`_posted_at` — ``created_at``,
2710 never ``updated_at``), then by position. A panel re-run on the same head that now
2711 approves supersedes the rejection before it, and one that now rejects supersedes the
2712 approval. A jury verdict pinned to an older head is not read: the head that moved
2713 answers it.
2714 """
2715 standing: dict[str, Any] | None = None
2716 for _, item in sorted(enumerate(items), key=lambda pair: (_posted_at(pair[1]), pair[0])):
2717 if _is_jury_verdict(
2718 item, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced
2719 ):
2720 standing = item
2721 return standing
2724def standing_jury_verdict(
2725 pr_comments: list[dict[str, Any]] | None,
2726 *,
2727 head_sha: str | None,
2728 covered_heads: Collection[str] = (),
2729 enforced: bool = True,
2730) -> dict[str, Any] | None:
2731 """The jury verdict comment that stands for ``head_sha``, or ``None`` (#1437).
2733 The merge gate's own reading (:func:`_standing_jury_verdict`): trusted author, the
2734 jury marker in the header, pinned to the head or a head it covers, latest by when it
2735 was posted. ``keel ship`` reuses the panel this names instead of convening another,
2736 so the two can never pick different panels. A blank head is refused here, unlike
2737 the evidence counters: reuse *replaces* a run, and a panel nobody pinned to a head
2738 must not stand in for one.
2739 """
2740 if not head_sha:
2741 return None
2742 return _standing_jury_verdict(
2743 pr_comments or [], head_sha=head_sha, covered_heads=covered_heads, enforced=enforced
2744 )
2747def verdict_head(item: dict[str, Any]) -> str:
2748 """The head a verdict comment answers for: its ``head:`` field, else its commit."""
2749 return _verdict_head(item, _body(item))
2752def _jury_not_approved_finding(
2753 items: list[dict[str, Any]],
2754 *,
2755 head_sha: str | None,
2756 covered_heads: Collection[str] = (),
2757 enforced: bool,
2758 blocking: bool,
2759 comment_rule: dict[str, Any] | None = None,
2760) -> dict[str, Any] | None:
2761 """The finding for a standing jury verdict whose consensus does not approve (#1429).
2763 ``comment_rule`` is :func:`comment_consensus`'s record when the opt-in rule was on
2764 and weighed this verdict without applying; its ``reason`` joins the message, so the
2765 operator is told which condition held the merge rather than only the consensus.
2767 ``major`` when ``blocking`` — the ``jury-verdict`` item is required and not deferred —
2768 so a rejecting panel **holds** the merge with its consensus and head named
2769 (``jury-verdict-not-approved: the jury's consensus at <head> is REQUEST_CHANGES, not an
2770 approval.`` in :func:`refusal_reason`). Merely leaving it uncounted would read as
2771 *missing* and send the operator looking for a comment that is on the pull request.
2772 ``minor`` otherwise: an advisory panel's rejection is said, never gated on.
2773 """
2774 standing = _standing_jury_verdict(
2775 items, head_sha=head_sha, covered_heads=covered_heads, enforced=enforced
2776 )
2777 if standing is None:
2778 return None
2779 body = _body(standing)
2780 token = jury_verdict_token(body)
2781 if token in APPROVING_VERDICTS:
2782 return None
2783 head = _verdict_head(standing, body)
2784 message = (
2785 f"the jury's consensus at {head} is {token}, not an approval."
2786 if token
2787 else f"the jury verdict at {head} has no readable AI Jury verdict line."
2788 )
2789 if comment_rule is not None:
2790 message = f"{message[:-1]} (comment_consensus not met: {comment_rule['reason']})."
2791 return {
2792 "id": JURY_NOT_APPROVED_FINDING,
2793 "severity": "major" if blocking else "minor",
2794 "kind": "jury",
2795 "message": message,
2796 }