Coverage for src/keel/jury.py: 100%
352 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
1"""The ``jury`` built-in gate — run the ai-jury CLI on the diff at s8.
3keel does **not** depend on ai-jury. If the ``jury`` CLI is on PATH, this gate runs it on
4the change's diff and maps its findings into keel :class:`~keel.findings.Finding`s. Without
5the ``jury`` binary the s8 run is a no-op (reported ``SKIPPED``, and blocking when ``jury`` is
6the only gate planned), but a tier-3 merge still requires a
7``jury-verdict`` unless the run passes ``--no-jury``; it relaxes to advisory only when a
8posted verdict (or ``--jury-vendors``) reports fewer than 2 vendors. That is the default
9policy: off a jury-panel tier, ``team.jury.mode: advisory`` or ``--jury-advisory`` never
10requires the verdict and ``team.jury.min_vendors`` may raise the 2; on a tier whose review is
11the jury panel, no flag or short panel relaxes it, and only a probe that finds the panel
12unstaffable turns that tier's jury off, under ``team.jury.on_unavailable: fallback`` (the
13default). That requirement is not decided here: :func:`keel.ship.resolve_jury` resolves the
14mode, and :func:`keel.evidence.required_items` demands the verdict from it. Parsing is pure and
15unit-tested; the subprocess is behind the injectable ``_run`` seam.
16"""
18from __future__ import annotations
20import contextlib
21import json
22import os
23import subprocess
24import tempfile
25from dataclasses import dataclass
26from pathlib import Path
27from typing import Any
29from . import artifacts, evidence
30from .findings import Finding
31from .model import DEFAULT_JURY_TIMEOUT_S
32from .runner import CommandResult, run_argv
34#: ai-jury severities → keel severities (unknown ⇒ ``minor``).
35_SEVERITY = {
36 "critical": "critical",
37 "blocker": "critical",
38 "major": "major",
39 "minor": "minor",
40 "nit": "nit",
41 "info": "nit",
42 "note": "nit",
43}
45MAX_DIFF_BYTES = 1_000_000
48def map_severity(severity: str) -> str:
49 """Map an ai-jury severity onto a keel severity (default ``minor``)."""
50 return _SEVERITY.get((severity or "").strip().lower(), "minor")
53def parse_report(data: dict | str) -> list[Finding] | None:
54 """Map an ai-jury JSON report into Findings, or ``None`` if it is not a report.
56 The ``None`` return is the point: it separates *"the panel reviewed the diff and
57 found nothing"* from *"this output is not a verdict at all"*, which
58 :func:`parse_findings` collapses into the same empty list. Only the caller that
59 decides whether a gate passed needs that distinction — see :func:`run_gate`.
61 Tolerates trailing non-JSON. :func:`keel.runner.run_argv` hands back
62 ``stdout + stderr`` concatenated, and ai-jury logs its progress to stderr, so a
63 real report is followed by ``[jury] …`` lines. A strict ``json.loads`` rejects the
64 whole thing and silently loses every finding.
65 """
66 if isinstance(data, str):
67 try:
68 data, _end = json.JSONDecoder().raw_decode(data.lstrip())
69 except json.JSONDecodeError:
70 return None
71 if not isinstance(data, dict) or "findings" not in data:
72 return None
73 return _findings_from(data)
76def parse_findings(data: dict | str) -> list[Finding]:
77 """Map an ai-jury JSON report (dict or raw string) into keel Findings.
79 Unparseable input yields ``[]``. Use :func:`parse_report` when the difference
80 between "no findings" and "no report" matters.
81 """
82 return parse_report(data) or []
85def _findings_from(data: dict) -> list[Finding]:
86 out: list[Finding] = []
87 for f in data.get("findings") or []:
88 path = f.get("file")
89 line = f.get("line")
90 line = line if isinstance(line, int) else None
91 out.append(
92 Finding(
93 severity=map_severity(f.get("severity", "")),
94 message=f.get("claim") or "(jury finding)",
95 source=f"jury:{f.get('reviewer') or 'consensus'}",
96 path=path,
97 line=line,
98 anchorable=bool(path) and line is not None,
99 )
100 )
101 return out
104def _kw(_run):
105 return {"_run": _run} if _run is not None else {}
108def available(*, cwd: str | None = None, _run=None) -> bool:
109 """True if the ``jury`` CLI is callable."""
110 return run_argv(["jury", "--version"], cwd=cwd, timeout=30, **_kw(_run)).ok
113def _incomplete_finding(result: CommandResult, *, timeout: int, severity: str = "nit") -> Finding:
114 """Record that the jury CLI ran but produced no verdict.
116 A timeout, or a nonzero exit whose output carries no parseable findings, means the
117 panel never reached a conclusion. That is emphatically **not** a clean pass — it is
118 the *absence* of a review — so in gating mode it fails closed exactly as an oversize
119 diff does. The timeout case is named apart from a crash so the operator can tell a
120 slow panel from a broken one.
121 """
122 if result.timed_out:
123 detail = (
124 f"timed out after {timeout}s; no verdict was produced. Raise "
125 "knobs.jury_timeout_s if the panel legitimately needs longer"
126 )
127 else:
128 detail = f"exited {result.code} without a parseable verdict; the panel did not complete"
129 return Finding(
130 severity=severity,
131 message=f"jury run incomplete: the jury CLI {detail}.",
132 source="jury:incomplete-run",
133 path=None,
134 line=None,
135 anchorable=False,
136 )
139def _unreadable_diff_finding(*, severity: str = "minor") -> Finding:
140 """Record that the diff itself could not be read, so no review was possible."""
141 return Finding(
142 severity=severity,
143 message=(
144 "jury could not run: the diff could not be read from git (is the base "
145 "branch fetched locally? a shallow or single-branch clone cannot "
146 "resolve base...HEAD). No review was performed."
147 ),
148 source="jury:unreadable-diff",
149 path=None,
150 line=None,
151 anchorable=False,
152 )
155def _oversize_finding(size: int, *, severity: str = "nit") -> Finding:
156 """Record that the jury gate skipped an oversize diff.
158 Advisory jury mode keeps the finding non-blocking (``nit``). Gating jury mode
159 escalates it to ``major`` so an oversize diff cannot bypass the blocking
160 cross-vendor review gate.
161 """
162 return Finding(
163 severity=severity,
164 message=(
165 f"jury skipped: diff is {size} bytes, over the {MAX_DIFF_BYTES}-byte "
166 "limit (ai-jury large-diff chunking not applied)"
167 ),
168 source="jury:skipped-oversize",
169 path=None,
170 line=None,
171 anchorable=False,
172 )
175#: The source of the finding a jury gate that **could not run** reports (#1369): no ``jury``
176#: CLI on the host, or an empty diff. It judged nothing, so it must never read as ``ok``.
177#: :func:`could_not_run` reads it back; :func:`keel.gates.lone_jury_cannot_judge` turns it
178#: into a blocking outcome when the jury is the only gate planned.
179NOT_RUN_SOURCE = "jury:not-run"
181#: Why the jury could not run, as the finding says it.
182NOT_RUN_NO_CLI = "the jury CLI is not available (`jury --version` failed; install ai-jury)"
183NOT_RUN_EMPTY_DIFF = "the diff against the base branch is empty"
186def _not_run_finding(reason: str) -> Finding:
187 """Record that the jury gate judged nothing, and why (#1369).
189 ``nit``: beside another gate that judges, a jury that could not run stays the
190 documented s8 no-op and does not hold the merge. It is reported rather than silent,
191 and it is what marks the outcome ``SKIPPED`` rather than ``ok``.
192 """
193 return Finding(
194 severity="nit",
195 message=f"jury did not run: {reason}; nothing was judged.",
196 source=NOT_RUN_SOURCE,
197 path=None,
198 line=None,
199 anchorable=False,
200 )
203#: The source of the finding a jury gate reports for the panel's **consensus** (#1436).
204CONSENSUS_SOURCE = "jury:consensus"
207def panel_consensus(data: dict | str) -> str | None:
208 """The panel's consensus in an ai-jury report, in keel's vocabulary, or ``None``.
210 The consensus is the **chair record's** ``verdict`` in the report's ``reviewers``
211 array (``role: chair``). ai-jury writes it from the chair's synthesis headline, or
212 from the panel vote when the run is configured with ``decision: vote``
213 (``ai_jury.ballots.chair_verdict``), so one field carries both. It is read through
214 :func:`map_verdict`, so ``APPROVE`` / ``READY`` arrive as ``LGTM``.
216 A ballot report with **no chair record** — the synthesis failed — has no consensus, and
217 reads ``ABSTAIN``, which is what :func:`jury_verdict` posts for it too. ``None`` means
218 the report carries **no ballots at all**: an ai-jury from before report schema 1.1,
219 which has no ``reviewers`` array, or one whose ballots are malformed.
220 """
221 try:
222 panel = parse_panel(data)
223 except JuryReportError:
224 return None
225 if panel is None:
226 return None
227 return panel.chair.verdict if panel.chair is not None else "ABSTAIN"
230def _non_approving_consensus_finding(
231 token: str | None,
232 *,
233 gating: bool,
234 posted: bool,
235 comment_rule: dict[str, Any] | None = None,
236) -> Finding:
237 """The finding emitted when a jury consensus does not approve (#1436, #1570).
239 When ``gating`` is True, the consensus is ``COMMENT``, and ``comment_rule``
240 (:func:`keel.evidence.comment_consensus`) is present:
242 * if ``comment_rule["applied"]`` is True, the finding degrades to ``minor`` and names
243 how many approving review verdicts satisfied the rule;
244 * otherwise it stays ``major`` and appends ``comment_rule["reason"]``, matching
245 :func:`keel.evidence._jury_not_approved_finding`.
246 """
247 prefix = "posted jury" if posted else "jury"
248 if not token:
249 return Finding(
250 severity="major" if gating else "minor",
251 message="posted jury verdict has no readable AI Jury verdict line.",
252 source=CONSENSUS_SOURCE,
253 path=None,
254 line=None,
255 anchorable=False,
256 )
257 if gating and token == evidence.COMMENT_CONSENSUS and comment_rule is not None:
258 if comment_rule.get("applied"):
259 approvals = comment_rule.get("approvals", 0)
260 required = comment_rule.get("required_approvals", approvals)
261 return Finding(
262 severity="minor",
263 message=(
264 f"{prefix} consensus is {token}; comment_consensus applied "
265 f"({approvals} of {required} required approving review verdicts at the head)."
266 ),
267 source=CONSENSUS_SOURCE,
268 path=None,
269 line=None,
270 anchorable=False,
271 )
272 return Finding(
273 severity="major",
274 message=(
275 f"{prefix} consensus is {token}, not an approval "
276 f"(comment_consensus not met: {comment_rule.get('reason')})."
277 ),
278 source=CONSENSUS_SOURCE,
279 path=None,
280 line=None,
281 anchorable=False,
282 )
283 return Finding(
284 severity="major" if gating else "minor",
285 message=f"{prefix} consensus is {token}, not an approval.",
286 source=CONSENSUS_SOURCE,
287 path=None,
288 line=None,
289 anchorable=False,
290 )
293def _consensus_finding(
294 data: dict | str,
295 *,
296 gating: bool,
297 comment_rule: dict[str, Any] | None = None,
298) -> Finding | None:
299 """The finding a jury gate reports when the panel's consensus does not approve (#1436).
301 Read against :data:`keel.evidence.APPROVING_VERDICTS` by the same reader the
302 evidence gate applies to the posted ``AI Jury verdict:`` line
303 (:func:`keel.evidence.jury_verdict_approves`), so the gates-pass and the evidence
304 gate cannot disagree about what the panel said. ``major`` in gating mode — the gate
305 fails — and ``minor`` in advisory mode, which reports a rejection and never gates on
306 one, as the evidence gate does. When the opt-in ``team.jury.comment_consensus`` rule
307 applies (:func:`keel.evidence.comment_consensus`, #1570), a ``COMMENT`` consensus is
308 reported as a non-blocking ``minor`` finding instead.
310 **A report with no consensus fails closed in gating mode, and is silent in advisory
311 mode.** A gating jury's verdict comment has to approve before ``keel merge`` will land
312 the change, and a report that states no consensus cannot truthfully produce one; a
313 gates-pass for it would certify a review that never concluded. Advisory mode keeps
314 today's behaviour: the severity rule alone, with nothing added.
315 """
316 consensus = panel_consensus(data)
317 if consensus is None:
318 if not gating:
319 return None
320 return Finding(
321 severity="major",
322 message=(
323 "jury report states no panel consensus: it carries no readable `reviewers` "
324 "ballots with a chair record (ai-jury before report schema 1.1, or a "
325 "malformed report). A gating jury must conclude; upgrade ai-jury."
326 ),
327 source=CONSENSUS_SOURCE,
328 path=None,
329 line=None,
330 anchorable=False,
331 )
332 line = f"AI Jury verdict: {consensus}."
333 if evidence.jury_verdict_approves(line):
334 return None
335 token = evidence.jury_verdict_token(line) or consensus
336 return _non_approving_consensus_finding(
337 token, gating=gating, posted=False, comment_rule=comment_rule
338 )
341#: The source of the findings a jury gate carries over from a reused panel (#1437).
342REUSED_SOURCE = "jury:reused"
344#: The source of the finding emitted when an isolated review checkout cannot be prepared (#1516).
345REVIEW_CHECKOUT_SOURCE = "jury:review-checkout"
348def reuse_posted_verdict(
349 body: str,
350 *,
351 gating: bool,
352 comment_rule: dict[str, Any] | None = None,
353) -> tuple[bool, list[Finding]] | None:
354 """The jury gate's outcome read off the panel already posted for the head (#1437).
356 ``body`` is the standing ``keel.jury-verdict.v1`` comment for the head
357 (:func:`keel.evidence.standing_jury_verdict`). The gate is judged by the two rules a
358 panel keel ran is judged by, so reusing one can never be kinder than running it:
360 * **the severity rule** — each ``<severity>: <message>`` item of the comment's findings
361 summary (:func:`keel.artifacts.jury_verdict_summary`, the verified findings
362 :func:`jury_verdict` posts) becomes a finding, and a critical or major one fails;
363 * **the consensus rule** (#1436, #1570) — an ``AI Jury verdict:`` line that does not
364 approve, or no readable line at all, is a ``major`` finding in gating mode (unless
365 ``comment_rule`` says ``team.jury.comment_consensus`` applied to a ``COMMENT``
366 consensus, in which case it is a ``minor`` finding) and a ``minor`` one in advisory
367 mode, the same as :func:`_consensus_finding` for a report.
369 ``(ok, findings)``, or ``None`` when the comment's summary cannot be read — a body keel
370 did not render. ``None`` means *do not reuse*: the caller convenes the panel as it
371 always did, so an unreadable comment can never stand in for a run.
372 """
373 summary = artifacts.jury_verdict_summary(body)
374 if summary is None:
375 return None
376 findings: list[Finding] = []
377 for item in summary:
378 label, sep, _message = item.partition(":")
379 findings.append(
380 Finding(
381 # An item with no `<severity>:` label is read as unknown, which
382 # map_severity already maps to `minor`.
383 severity=map_severity(label if sep else ""),
384 message=f"posted jury finding — {item}",
385 source=REUSED_SOURCE,
386 path=None,
387 line=None,
388 anchorable=False,
389 )
390 )
391 if not evidence.jury_verdict_approves(body):
392 token = evidence.jury_verdict_token(body)
393 findings.append(
394 _non_approving_consensus_finding(
395 token, gating=gating, posted=True, comment_rule=comment_rule
396 )
397 )
398 blocked = any(f.severity in ("critical", "major") for f in findings)
399 return (not blocked), findings
402def could_not_run(findings) -> bool:
403 """Did this jury gate result come back without running (no CLI, or an empty diff)?"""
404 return any(f.source == NOT_RUN_SOURCE for f in findings)
407#: The lines that frame a PR's base facts ahead of the diff the jury reads (#1570).
408BASE_FACTS_LEAD = (
409 "Base facts — stated by the pull request's author about code outside this diff, "
410 "which the change relies on:"
411)
412BASE_FACTS_CHECK = (
413 "Each base fact above is a claim, not a given. Check every one against this checkout "
414 "(the head of the change). A base fact that is false, or that the change depends on and "
415 "the checkout cannot confirm, is a finding against the change."
416)
419def with_base_facts(diff_text: str, base_facts: str | None) -> str:
420 """``diff_text`` with the PR's base facts ahead of it, or unchanged when there are none.
422 The jury reads only the diff, so a change that relies on code outside it leaves the
423 panel unable to check the reliance — and a panel that cannot check hedges. The facts
424 go where ``git format-patch`` puts a commit message, before the first ``diff --git``
425 header, which ai-jury keeps attached to the first file rather than dropping. They are
426 framed as claims to verify (:data:`BASE_FACTS_CHECK`): the author's word is input to
427 the review, never a substitute for it.
428 """
429 if not base_facts or not base_facts.strip() or not diff_text:
430 return diff_text
431 return f"{BASE_FACTS_LEAD}\n\n{base_facts.strip()}\n\n{BASE_FACTS_CHECK}\n\n{diff_text}"
434def run_gate(
435 diff_text: str,
436 *,
437 cwd: str | None = None,
438 mode: str = "advisory",
439 timeout: int = DEFAULT_JURY_TIMEOUT_S,
440 head_sha: str | None = None,
441 base_facts: str | None = None,
442 comment_rule: dict[str, Any] | None = None,
443 _run=None,
444) -> tuple[bool, list[Finding], bool]:
445 """Run ``jury`` on ``diff_text`` and map its findings.
447 ``base_facts`` is the PR's ``## Base facts`` section (``team.jury.base_facts``,
448 #1570); when given, the panel reads it ahead of the diff (:func:`with_base_facts`).
450 When ``head_sha`` is provided, executes ``jury`` inside an isolated detached checkout of
451 ``head_sha`` created via :func:`keel.swarm_runtime.review_checkout` (under ``cwd``) and
452 removed on exit; a checkout setup failure returns a ``major`` (gating) or ``minor``
453 (advisory) finding from :data:`REVIEW_CHECKOUT_SOURCE`.
455 Returns ``(ok, findings, timed_out)``. ``ok`` is False when a finding blocks
456 (critical/major) or when the run produced no verdict at all in gating mode — and, in
457 gating mode, when the panel's **consensus** does not approve or the report states
458 none (#1436, :func:`_consensus_finding`). Both rules hold at once: a verified major
459 still blocks a panel that approved, and a panel that requested changes over minors
460 alone, or abstained, no longer passes because no finding was severe.
461 No-op when there is no diff or the ``jury`` CLI is not installed — keel does not
462 depend on ai-jury, so an absent CLI is a legitimate no-op *for this run*, distinct
463 from a run that started and did not finish. A no-op is not a pass: it returns one
464 ``nit`` finding from :data:`NOT_RUN_SOURCE` saying why nothing was judged, so the
465 outcome reads ``SKIPPED`` and a plan with no other gate blocks (#1369). It waives
466 nothing downstream: a gating jury's ``jury-verdict`` is still required at merge (see
467 the module docstring).
469 Three ways a run can end without a review, all handled alike — gating fails closed
470 with a blocking ``major``, advisory surfaces a ``minor``:
472 * the diff is oversize and was never submitted,
473 * the CLI was killed by ``timeout``,
474 * the CLI returned no parseable verdict, whatever its exit code.
476 The last used to report ``(True, [])``: :func:`parse_findings` yields ``[]`` for
477 unparseable output, so ``blocked`` came out False and a hung, crashed, or
478 unreadable panel read as a clean pass. The test is deliberately *"did we parse a
479 verdict"* rather than *"was the exit code zero"* — ai-jury exits nonzero to signal
480 "request changes", which is a completed review whose findings must be honoured,
481 while an exit of zero carrying unreadable output is not a review at all.
482 """
483 if diff_text is None:
484 # The diff could not be read (git failed). That is not "nothing to review":
485 # passing here would silently remove the review gate from the merge decision,
486 # which is the same fail-open the verdict check below exists to prevent.
487 gating = mode == "gating"
488 return (
489 (not gating),
490 [_unreadable_diff_finding(severity="major" if gating else "minor")],
491 False,
492 )
493 if not diff_text:
494 return True, [_not_run_finding(NOT_RUN_EMPTY_DIFF)], False
495 diff_text = with_base_facts(diff_text, base_facts)
496 size = len(diff_text.encode("utf-8"))
497 if size > MAX_DIFF_BYTES:
498 if mode == "gating":
499 return False, [_oversize_finding(size, severity="major")], False
500 return True, [_oversize_finding(size)], False
501 if not available(cwd=cwd, _run=_run):
502 return True, [_not_run_finding(NOT_RUN_NO_CLI)], False
504 run_cwd = cwd
505 with contextlib.ExitStack() as stack:
506 if head_sha is not None:
507 from . import swarm_runtime
509 def _wrap_runner(cmd: list[str], path: Path) -> CommandResult:
510 res = _run(cmd, cwd=str(path), stdin=subprocess.DEVNULL)
511 if isinstance(res, CommandResult):
512 return res
513 out = getattr(res, "stdout", "") or ""
514 err = getattr(res, "stderr", "") or ""
515 code = getattr(res, "returncode", 0)
516 return CommandResult(
517 code == 0,
518 code,
519 out + err,
520 stdout=out,
521 stderr=err,
522 )
524 checkout_runner = _wrap_runner if _run is not None else None
525 try:
526 checkout = stack.enter_context(
527 swarm_runtime.review_checkout(
528 head_sha, name="jury", root=cwd or ".", runner=checkout_runner
529 )
530 )
531 except swarm_runtime.ReviewCheckoutError as exc:
532 gating = mode == "gating"
533 return (
534 (not gating),
535 [
536 Finding(
537 severity="major" if gating else "minor",
538 message=str(exc),
539 source=REVIEW_CHECKOUT_SOURCE,
540 )
541 ],
542 False,
543 )
544 run_cwd = str(checkout)
545 fd, path = tempfile.mkstemp(suffix=".diff")
546 try:
547 with os.fdopen(fd, "w", encoding="utf-8") as fh:
548 fh.write(diff_text)
549 result = run_argv(
550 ["jury", "--format", "json", "--diff-file", path],
551 cwd=run_cwd,
552 timeout=timeout,
553 **_kw(_run),
554 )
555 finally:
556 os.unlink(path)
557 # stdout alone: ai-jury logs its progress (`[jury] …`) to stderr, and reading the
558 # concatenation is what made every report unparseable (#624). `parse_report` still
559 # tolerates trailing non-JSON, for a vendor that also chats on stdout.
560 report = parse_report(result.stdout)
561 if report is None:
562 gating = mode == "gating"
563 incomplete = _incomplete_finding(
564 result, timeout=timeout, severity="major" if gating else "minor"
565 )
566 # timed_out rides along so the outcome renders as TIMEOUT rather than FAIL,
567 # the distinction #622 established for command gates.
568 return (not gating), [incomplete], result.timed_out
569 consensus = _consensus_finding(
570 result.stdout, gating=mode == "gating", comment_rule=comment_rule
571 )
572 if consensus is not None:
573 report = [*report, consensus]
574 blocked = any(f.severity in ("critical", "major") for f in report)
575 return (not blocked), report, False
578# --------------------------------------------------------------------------- #
579# Per-reviewer ballots (#1015) — the panel *as* the review, not beside it.
580# --------------------------------------------------------------------------- #
582#: The ``role`` ai-jury stamps on the chair's entry in the report's ``reviewers``
583#: array. The chair is the consensus record, not a panelist ballot, so it renders
584#: as the jury verdict rather than as one more review verdict.
585CHAIR_ROLE = "chair"
587#: ai-jury ballot tokens → keel verdict vocabulary. ai-jury emits one machine
588#: token per ballot (``REQUEST_CHANGES``, never ``REQUEST CHANGES``) in either the
589#: code or the ``--issue`` vocabulary; keel's verdicts are ``LGTM`` /
590#: ``REQUEST_CHANGES`` / ``COMMENT`` / ``ABSTAIN``. An unknown token is carried
591#: through verbatim rather than folded into ``LGTM``: inventing an approval for a
592#: stance keel does not recognise is the one mapping error that cannot be undone.
593_VERDICT = {
594 "APPROVE": "LGTM",
595 "READY": "LGTM",
596 "REQUEST_CHANGES": "REQUEST_CHANGES",
597 "NEEDS_INFO": "REQUEST_CHANGES",
598 "COMMENT": "COMMENT",
599 "UNCLEAR": "COMMENT",
600 "ABSTAIN": "ABSTAIN",
601 "NO_QUORUM": "ABSTAIN",
602}
604#: Files a ballot's scope line names before it starts counting instead.
605_SCOPE_FILES = 8
607#: The verification status ai-jury stamps on a consensus group the verification
608#: round upheld. Only these findings gate: an unsupported or unverified claim is
609#: reported, never merged against.
610VERIFIED_STATUS = "verified"
613class JuryReportError(ValueError):
614 """Raised when an ai-jury report cannot be read as a panel of ballots."""
617def map_verdict(verdict: str) -> str:
618 """Map an ai-jury ballot token onto keel's verdict vocabulary."""
619 token = (verdict or "").strip().upper().replace(" ", "_").replace("-", "_")
620 if not token:
621 return "ABSTAIN"
622 return _VERDICT.get(token, token)
625@dataclass(frozen=True)
626class Ballot:
627 """One panelist's own stance, with the provenance that makes it evidence."""
629 reviewer: str
630 verdict: str
631 vendor: str | None = None
632 model: str | None = None
633 verified_count: int = 0
634 round1_ok: bool = True
635 findings: tuple[dict[str, Any], ...] = ()
636 scope: str | None = None
637 testing: str | None = None
638 counts_as_review: bool | None = None
639 scope_substantive: bool | None = None
640 abstention_cause: str | None = None
642 def as_review(self) -> dict[str, Any]:
643 """This ballot in the ``keel review --reviews`` bundle shape."""
644 return {
645 "reviewer": self.reviewer,
646 "verdict": self.verdict,
647 "scope": ballot_scope(self),
648 "findings": [dict(finding) for finding in self.findings],
649 "testing": ballot_testing(self),
650 "vendor": self.vendor,
651 "model": self.model,
652 }
655def _review_ballots(ballots: tuple[Ballot, ...]) -> tuple[Ballot, ...]:
656 """Ballots that count as reviews — the one definition :class:`Panel` consumes."""
657 return tuple(ballot for ballot in ballots if ballot_is_review(ballot))
660@dataclass(frozen=True)
661class Panel:
662 """A parsed ai-jury panel: the panelist ballots and the chair's consensus."""
664 ballots: tuple[Ballot, ...] = ()
665 chair: Ballot | None = None
666 verified: tuple[dict[str, Any], ...] = ()
668 @property
669 def size(self) -> int:
670 """Reviews this panel produced — the reviewer count the evidence gate sizes.
672 Aligns with ai-jury's ``is_review``: an abstention, an empty ballot, or a
673 ``counts_as_review: false`` record is not a review and does not inflate
674 ``panelists`` / ``jury_panel_size``. :func:`parse_panel` already drops
675 those from :attr:`ballots`; this property re-applies the same predicate
676 so a hand-built panel cannot disagree with the posting path.
677 """
678 return len(_review_ballots(self.ballots))
680 @property
681 def vendors(self) -> tuple[str, ...]:
682 """Distinct declared vendors across the reviews, in panel order.
684 Lower-cased and de-duplicated exactly as :func:`keel.evidence.distinct_vendor_check`
685 reads the posted ``vendor:`` lines, so the count declared on the jury verdict and
686 the count the evidence gate recomputes from the verdicts cannot disagree.
687 Abstaining seats are not reviews and do not contribute a vendor.
688 """
689 seen: list[str] = []
690 for ballot in _review_ballots(self.ballots):
691 vendor = (ballot.vendor or "").strip().lower()
692 if vendor and vendor not in seen:
693 seen.append(vendor)
694 return tuple(seen)
696 def reviews(self) -> tuple[dict[str, Any], ...]:
697 """Review ballots in the ``--reviews`` bundle shape.
699 Non-reviews are omitted: posting them as head-pinned ``review-verdict-*``
700 evidence is the defect this mapping exists to close.
701 """
702 return tuple(ballot.as_review() for ballot in _review_ballots(self.ballots))
705def _finding_record(raw: Any) -> dict[str, Any] | None:
706 """One ai-jury finding in keel's finding shape (``file``→``path``, ``claim``→``message``)."""
707 if not isinstance(raw, dict):
708 return None
709 line = raw.get("line")
710 return {
711 "severity": map_severity(raw.get("severity", "")),
712 "path": raw.get("file") or None,
713 "line": line if isinstance(line, int) else None,
714 "message": raw.get("claim") or "(jury finding)",
715 }
718def _ballot_findings(raw: Any, findings: list[Any]) -> tuple[dict[str, Any], ...]:
719 """Resolve a ballot's ``findings`` index list against the report's findings array.
721 Out-of-range and non-integer indexes are dropped rather than raising: the
722 ballot's stance is the evidence, and a report whose indexes do not line up
723 must still produce a verdict that says so with the findings it *can* resolve.
724 """
725 if not isinstance(raw, list):
726 return ()
727 records: list[dict[str, Any]] = []
728 for index in raw:
729 if not isinstance(index, int) or isinstance(index, bool):
730 continue
731 if not 0 <= index < len(findings):
732 continue
733 record = _finding_record(findings[index])
734 if record is not None:
735 records.append(record)
736 return tuple(records)
739def _text_field(raw: dict[str, Any], key: str) -> str | None:
740 """A non-empty string field, or ``None`` when absent / blank / the wrong type."""
741 value = raw.get(key)
742 if isinstance(value, str) and value.strip():
743 return value.strip()
744 return None
747def _bool_field(raw: dict[str, Any], key: str) -> bool | None:
748 """A JSON boolean field, or ``None`` when absent or not a bool.
750 Integers are refused: ``1``/``0`` are not the schema ≥1.2 flags, and treating
751 them as booleans would let a malformed report opt a ballot into the review
752 count.
753 """
754 value = raw.get(key)
755 if isinstance(value, bool):
756 return value
757 return None
760#: Finding severities that make a panelist's stance a rejection rather than a remark.
761_BLOCKING_SEVERITIES = frozenset({"critical", "major"})
764def ballot_verdict(
765 verdict: str, findings: tuple[dict[str, Any], ...] = (), *, resolved: bool = True
766) -> str:
767 """A panelist's ballot in keel's vocabulary: :func:`map_verdict`, except ``COMMENT``.
769 ai-jury derives a code-review ballot from the worst finding the panelist raised that
770 the verifier did not reject: critical/major is ``REQUEST_CHANGES``, minor/nit only is
771 ``COMMENT``, none is ``APPROVE`` (``ai_jury.voting``). So a ``COMMENT`` ballot is "I
772 have only non-blocking remarks" — what a host reviewer posts as ``APPROVE`` with nits —
773 and it counts as approving (``LGTM``), its findings still listed. Without this, two
774 panelists with nits held a merge the chair approved (#1500). It stays ``COMMENT`` if
775 any finding it carries is critical or major. Only ballots: the panel's consensus is
776 read by :func:`map_verdict`, and a consensus of ``COMMENT`` still holds; ``UNCLEAR``
777 (the issue-mode remark) still holds too. ``resolved`` is ``False`` when some of the
778 ballot's finding indexes did not resolve: what they pointed at is unknown, so the
779 ``COMMENT`` is kept rather than read as non-blocking.
780 """
781 token = (verdict or "").strip().upper().replace(" ", "_").replace("-", "_")
782 if (
783 token == "COMMENT"
784 and resolved
785 and not any(
786 str(finding.get("severity", "")).strip().lower() in _BLOCKING_SEVERITIES
787 for finding in findings
788 )
789 ):
790 return "LGTM"
791 return map_verdict(verdict)
794def _ballot(raw: Any, findings: list[Any], *, position: int) -> Ballot:
795 if not isinstance(raw, dict):
796 raise JuryReportError(f"jury report reviewer #{position} must be a JSON object")
797 name = raw.get("name")
798 if not isinstance(name, str) or not name.strip():
799 raise JuryReportError(f"jury report reviewer #{position} requires a non-empty 'name'")
800 vendor = raw.get("vendor")
801 model = raw.get("model")
802 verified = raw.get("verified_count")
803 raw_indexes = raw.get("findings")
804 ballot_findings = _ballot_findings(raw_indexes, findings)
805 # Every index resolved (or none was given): only then is a COMMENT known to be minor.
806 resolved = raw_indexes is None or (
807 isinstance(raw_indexes, list) and len(ballot_findings) == len(raw_indexes)
808 )
809 # The chair record is the consensus, not a panelist: its COMMENT still holds.
810 is_chair = (raw.get("role") or "") == CHAIR_ROLE
811 return Ballot(
812 reviewer=name.strip(),
813 verdict=map_verdict(raw.get("verdict", ""))
814 if is_chair
815 else ballot_verdict(raw.get("verdict", ""), ballot_findings, resolved=resolved),
816 vendor=vendor.strip() if isinstance(vendor, str) and vendor.strip() else None,
817 model=model.strip() if isinstance(model, str) and model.strip() else None,
818 verified_count=verified
819 if isinstance(verified, int) and not isinstance(verified, bool)
820 else 0,
821 round1_ok=bool(raw.get("round1_ok", True)),
822 findings=ballot_findings,
823 scope=_text_field(raw, "scope"),
824 testing=_text_field(raw, "testing"),
825 counts_as_review=_bool_field(raw, "counts_as_review"),
826 scope_substantive=_bool_field(raw, "scope_substantive"),
827 abstention_cause=_text_field(raw, "abstention_cause"),
828 )
831def _verified_records(data: dict) -> tuple[dict[str, Any], ...]:
832 """Consensus-group representatives the verification round upheld.
834 These are the findings that gate. ai-jury verifies a consensus group and
835 stamps ``verification_status``; keel's own rule — critical/major block —
836 applies to the *upheld* ones only, so a claim the panel could not support
837 never holds a merge.
838 """
839 records: list[dict[str, Any]] = []
840 for group in data.get("consensus") or []:
841 if not isinstance(group, dict):
842 continue
843 if (group.get("verification_status") or "") != VERIFIED_STATUS:
844 continue
845 record = _finding_record(group.get("representative"))
846 if record is not None:
847 reviewers = group.get("reviewers")
848 record["reviewers"] = (
849 [name for name in reviewers if isinstance(name, str)]
850 if isinstance(reviewers, list)
851 else []
852 )
853 records.append(record)
854 return tuple(records)
857def parse_panel(data: dict | str) -> Panel | None:
858 """Parse an ai-jury JSON report into a :class:`Panel`, or ``None``.
860 ``None`` means *this is not a report carrying per-reviewer ballots* — an
861 unparseable document, or a pre-schema-1.1 report with no ``reviewers`` array.
862 The caller turns that into an actionable error (upgrade ai-jury, or supply a
863 ``--reviews`` bundle); it is deliberately not an exception, because "not a
864 ballot report" is the same question :func:`parse_report` answers for findings.
866 A report that *does* carry ballots but carries them malformed raises
867 :class:`JuryReportError`: dropping a panelist would silently post fewer
868 verdicts than the panel produced, which is the one failure this whole path
869 exists to prevent.
871 Only ballots that :func:`ballot_is_review` accepts enter :attr:`Panel.ballots`.
872 An ``ABSTAIN``, a ``counts_as_review: false`` record, or an older empty
873 ballot is parsed and then dropped, so it cannot inflate ``panel.size`` or
874 become a posted ``review-verdict-*``.
875 """
876 if isinstance(data, str):
877 try:
878 data, _end = json.JSONDecoder().raw_decode(data.lstrip())
879 except json.JSONDecodeError:
880 return None
881 if not isinstance(data, dict):
882 return None
883 raw_reviewers = data.get("reviewers")
884 if not isinstance(raw_reviewers, list):
885 return None
886 findings = list(data.get("findings") or [])
887 ballots: list[Ballot] = []
888 chair: Ballot | None = None
889 for position, raw in enumerate(raw_reviewers, start=1):
890 ballot = _ballot(raw, findings, position=position)
891 if isinstance(raw, dict) and (raw.get("role") or "") == CHAIR_ROLE:
892 chair = ballot
893 continue
894 if ballot_is_review(ballot):
895 ballots.append(ballot)
896 return Panel(ballots=tuple(ballots), chair=chair, verified=_verified_records(data))
899def _finding_paths(ballot: Ballot) -> list[str]:
900 """Distinct file paths this ballot's own findings named, in first-seen order."""
901 files: list[str] = []
902 for finding in ballot.findings:
903 path = finding.get("path")
904 if isinstance(path, str) and path.strip() and path not in files:
905 files.append(path.strip())
906 return files
909def ballot_is_review(ballot: Ballot) -> bool:
910 """Whether this panelist ballot counts as a review (ai-jury ``is_review``).
912 A review is a panelist whose scope is substantive and whose verdict is not
913 ``ABSTAIN``. The chair is split off before this predicate runs.
915 The flags a schema ≥1.2 report declares — ``counts_as_review`` and
916 ``scope_substantive`` — can only ever **remove** a ballot here, never admit
917 one that carries nothing. ai-jury derives ``counts_as_review`` from
918 ``scope_substantive``, which is itself derived from the ``scope`` prose, so a
919 record claiming ``counts_as_review: true`` with no ``scope`` and no finding
920 is not a clean review it produced; it is internally inconsistent, and
921 admitting it means keel writing the substance the report failed to supply.
922 That is the escape hatch #1150 is about, so the ambiguous record fails closed
923 like every other one.
925 What is left is the fact the scope line reads: a ballot counts when the
926 report gave prose to post or a path to name. Older reports carry no flags and
927 are decided by that same fact, so a schema-1.1 empty ``APPROVE`` is dropped
928 rather than dressed up.
929 """
930 if ballot.verdict == "ABSTAIN":
931 return False
932 if ballot.counts_as_review is False or ballot.scope_substantive is False:
933 return False
934 return bool(ballot.scope) or bool(_finding_paths(ballot))
937def _abstention_scope(ballot: Ballot) -> str:
938 """An explicitly anchorless scope: no ``Checked …``, no path, no backtick."""
939 cause = (ballot.abstention_cause or "").replace("_", " ")
940 if cause:
941 return f"ai-jury panelist {ballot.reviewer} did not review ({cause})."
942 return f"ai-jury panelist {ballot.reviewer} did not review."
945def ballot_scope(ballot: Ballot) -> str:
946 """The scope line keel renders for a panelist ballot.
948 A ballot that is not a review never gets a substance-passing opener: keel
949 used to always start with ``Checked the changed-file diff…``, which is its
950 own :func:`keel.evidence.verdict_substance` escape hatch, so an ``ABSTAIN``
951 still passed the gate by construction.
953 Every remaining branch renders something the report actually supplied. A
954 schema ≥1.2 ballot carries its own ``scope`` and that prose is posted as
955 written — including the clean review that read the diff and found nothing,
956 which ai-jury describes itself rather than leaving keel to. Otherwise the
957 ballot named paths, and the ``checked …`` line is built from them. There is
958 no third case: :func:`ballot_is_review` admits a ballot only when one of
959 those two is true, so this function never has to invent a scope for a ballot
960 it was told to post.
961 """
962 if not ballot_is_review(ballot):
963 return _abstention_scope(ballot)
964 if ballot.scope:
965 return ballot.scope
966 # Non-empty: `ballot_is_review` accepted this ballot, and with no `scope`
967 # prose the only way it could have is by naming a path.
968 files = _finding_paths(ballot)
969 opening = f"Checked the changed-file diff as ai-jury panelist {ballot.reviewer}"
970 listed = ", ".join(files[:_SCOPE_FILES])
971 more = len(files) - _SCOPE_FILES
972 suffix = f" (+{more} more)" if more > 0 else ""
973 return f"{opening}; named {len(files)} file(s): {listed}{suffix}."
976def ballot_testing(ballot: Ballot) -> str:
977 """The testing line keel renders for a panelist ballot.
979 Schema ≥1.2 reports carry their own ``testing`` prose; that is preferred
980 when present. Otherwise the panel's verification round *is* the ballot's
981 testing note: it is the only check ai-jury performs on a reviewer's claims,
982 and a ballot whose claims were never upheld must say so rather than borrow
983 the PR's own testing section.
984 """
985 if ballot.testing:
986 return ballot.testing
987 if ballot.verified_count > 0:
988 note = (
989 f"ai-jury verification upheld {ballot.verified_count} consensus "
990 "group(s) this panelist joined."
991 )
992 else:
993 note = "ai-jury verification upheld no consensus group from this panelist."
994 if not ballot.round1_ok:
995 return f"The panelist's adapter reported a failed run; its output was still read. {note}"
996 return note
999def verified_findings(panel: Panel) -> list[Finding]:
1000 """Verified consensus findings as keel :class:`~keel.findings.Finding`s.
1002 This is the s9 input: ``critical``/``major`` block, ``minor`` is a gated
1003 suggestion, ``nit`` is advisory — the same mapping a host reviewer's findings
1004 get, which is the whole point of the panel being the review rather than a
1005 second opinion beside it.
1007 Findings verified by consensus are carried first. Any minor finding raised
1008 by a panelist that was not part of a verified consensus group is carried
1009 as a single-reviewer minor tagged with its source (#1546), so a panelist's
1010 non-blocking remark reaches the s9 fix synthesis instead of being dropped.
1011 """
1012 out: list[Finding] = []
1013 seen: set[tuple[str | None, int | None, str]] = set()
1014 for record in panel.verified:
1015 reviewers = record.get("reviewers") or []
1016 source = f"jury:{reviewers[0]}" if reviewers else "jury:consensus"
1017 path = record.get("path")
1018 line = record.get("line")
1019 message = record.get("message") or "(jury finding)"
1020 seen.add((path, line, message))
1021 out.append(
1022 Finding(
1023 severity=record["severity"],
1024 message=message,
1025 source=source,
1026 path=path,
1027 line=line,
1028 anchorable=bool(path) and isinstance(line, int),
1029 )
1030 )
1031 for ballot in _review_ballots(panel.ballots):
1032 for finding in ballot.findings:
1033 if finding.get("severity") != "minor":
1034 continue
1035 path = finding.get("path")
1036 line = finding.get("line")
1037 message = finding.get("message") or "(jury finding)"
1038 key = (path, line, message)
1039 if key in seen:
1040 continue
1041 seen.add(key)
1042 out.append(
1043 Finding(
1044 severity="minor",
1045 message=message,
1046 source=f"jury:{ballot.reviewer}",
1047 path=path,
1048 line=line,
1049 anchorable=bool(path) and isinstance(line, int),
1050 )
1051 )
1052 return out
1055def jury_verdict(panel: Panel) -> dict[str, Any]:
1056 """The ``render_jury_verdict`` arguments for a parsed panel.
1058 The chair's ballot is the consensus record — that is what the jury verdict
1059 comment has always been — and the panel's own size and vendor count ride
1060 along on it, because the posted verdict is the only channel by which either
1061 reaches a hosted evidence check (see :func:`keel.artifacts.render_jury_verdict`).
1062 """
1063 chair = panel.chair
1064 summary = [f"{record['severity']}: {record['message']}" for record in panel.verified]
1065 return {
1066 "verdict": chair.verdict if chair is not None else "ABSTAIN",
1067 "participants": [
1068 f"{ballot.reviewer} ({ballot.vendor})" if ballot.vendor else ballot.reviewer
1069 for ballot in _review_ballots(panel.ballots)
1070 ],
1071 "participating_vendors": len(panel.vendors),
1072 "panelists": panel.size,
1073 "findings_summary": summary,
1074 "remaining_risks": None if summary else "none identified",
1075 }