Coverage for src/keel/jury.py: 100%

352 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 19:17 +0000

1"""The ``jury`` built-in gate — run the ai-jury CLI on the diff at s8. 

2 

3keel does **not** depend on ai-jury. If the ``jury`` CLI is on PATH, this gate runs it on 

4the change's diff and maps its findings into keel :class:`~keel.findings.Finding`s. Without 

5the ``jury`` binary the s8 run is a no-op (reported ``SKIPPED``, and blocking when ``jury`` is 

6the only gate planned), but a tier-3 merge still requires a 

7``jury-verdict`` unless the run passes ``--no-jury``; it relaxes to advisory only when a 

8posted verdict (or ``--jury-vendors``) reports fewer than 2 vendors. That is the default 

9policy: off a jury-panel tier, ``team.jury.mode: advisory`` or ``--jury-advisory`` never 

10requires the verdict and ``team.jury.min_vendors`` may raise the 2; on a tier whose review is 

11the jury panel, no flag or short panel relaxes it, and only a probe that finds the panel 

12unstaffable turns that tier's jury off, under ``team.jury.on_unavailable: fallback`` (the 

13default). That requirement is not decided here: :func:`keel.ship.resolve_jury` resolves the 

14mode, and :func:`keel.evidence.required_items` demands the verdict from it. Parsing is pure and 

15unit-tested; the subprocess is behind the injectable ``_run`` seam. 

16""" 

17 

18from __future__ import annotations 

19 

20import contextlib 

21import json 

22import os 

23import subprocess 

24import tempfile 

25from dataclasses import dataclass 

26from pathlib import Path 

27from typing import Any 

28 

29from . import artifacts, evidence 

30from .findings import Finding 

31from .model import DEFAULT_JURY_TIMEOUT_S 

32from .runner import CommandResult, run_argv 

33 

34#: ai-jury severities → keel severities (unknown ⇒ ``minor``). 

35_SEVERITY = { 

36 "critical": "critical", 

37 "blocker": "critical", 

38 "major": "major", 

39 "minor": "minor", 

40 "nit": "nit", 

41 "info": "nit", 

42 "note": "nit", 

43} 

44 

45MAX_DIFF_BYTES = 1_000_000 

46 

47 

48def map_severity(severity: str) -> str: 

49 """Map an ai-jury severity onto a keel severity (default ``minor``).""" 

50 return _SEVERITY.get((severity or "").strip().lower(), "minor") 

51 

52 

53def parse_report(data: dict | str) -> list[Finding] | None: 

54 """Map an ai-jury JSON report into Findings, or ``None`` if it is not a report. 

55 

56 The ``None`` return is the point: it separates *"the panel reviewed the diff and 

57 found nothing"* from *"this output is not a verdict at all"*, which 

58 :func:`parse_findings` collapses into the same empty list. Only the caller that 

59 decides whether a gate passed needs that distinction — see :func:`run_gate`. 

60 

61 Tolerates trailing non-JSON. :func:`keel.runner.run_argv` hands back 

62 ``stdout + stderr`` concatenated, and ai-jury logs its progress to stderr, so a 

63 real report is followed by ``[jury] …`` lines. A strict ``json.loads`` rejects the 

64 whole thing and silently loses every finding. 

65 """ 

66 if isinstance(data, str): 

67 try: 

68 data, _end = json.JSONDecoder().raw_decode(data.lstrip()) 

69 except json.JSONDecodeError: 

70 return None 

71 if not isinstance(data, dict) or "findings" not in data: 

72 return None 

73 return _findings_from(data) 

74 

75 

76def parse_findings(data: dict | str) -> list[Finding]: 

77 """Map an ai-jury JSON report (dict or raw string) into keel Findings. 

78 

79 Unparseable input yields ``[]``. Use :func:`parse_report` when the difference 

80 between "no findings" and "no report" matters. 

81 """ 

82 return parse_report(data) or [] 

83 

84 

85def _findings_from(data: dict) -> list[Finding]: 

86 out: list[Finding] = [] 

87 for f in data.get("findings") or []: 

88 path = f.get("file") 

89 line = f.get("line") 

90 line = line if isinstance(line, int) else None 

91 out.append( 

92 Finding( 

93 severity=map_severity(f.get("severity", "")), 

94 message=f.get("claim") or "(jury finding)", 

95 source=f"jury:{f.get('reviewer') or 'consensus'}", 

96 path=path, 

97 line=line, 

98 anchorable=bool(path) and line is not None, 

99 ) 

100 ) 

101 return out 

102 

103 

104def _kw(_run): 

105 return {"_run": _run} if _run is not None else {} 

106 

107 

108def available(*, cwd: str | None = None, _run=None) -> bool: 

109 """True if the ``jury`` CLI is callable.""" 

110 return run_argv(["jury", "--version"], cwd=cwd, timeout=30, **_kw(_run)).ok 

111 

112 

113def _incomplete_finding(result: CommandResult, *, timeout: int, severity: str = "nit") -> Finding: 

114 """Record that the jury CLI ran but produced no verdict. 

115 

116 A timeout, or a nonzero exit whose output carries no parseable findings, means the 

117 panel never reached a conclusion. That is emphatically **not** a clean pass — it is 

118 the *absence* of a review — so in gating mode it fails closed exactly as an oversize 

119 diff does. The timeout case is named apart from a crash so the operator can tell a 

120 slow panel from a broken one. 

121 """ 

122 if result.timed_out: 

123 detail = ( 

124 f"timed out after {timeout}s; no verdict was produced. Raise " 

125 "knobs.jury_timeout_s if the panel legitimately needs longer" 

126 ) 

127 else: 

128 detail = f"exited {result.code} without a parseable verdict; the panel did not complete" 

129 return Finding( 

130 severity=severity, 

131 message=f"jury run incomplete: the jury CLI {detail}.", 

132 source="jury:incomplete-run", 

133 path=None, 

134 line=None, 

135 anchorable=False, 

136 ) 

137 

138 

139def _unreadable_diff_finding(*, severity: str = "minor") -> Finding: 

140 """Record that the diff itself could not be read, so no review was possible.""" 

141 return Finding( 

142 severity=severity, 

143 message=( 

144 "jury could not run: the diff could not be read from git (is the base " 

145 "branch fetched locally? a shallow or single-branch clone cannot " 

146 "resolve base...HEAD). No review was performed." 

147 ), 

148 source="jury:unreadable-diff", 

149 path=None, 

150 line=None, 

151 anchorable=False, 

152 ) 

153 

154 

155def _oversize_finding(size: int, *, severity: str = "nit") -> Finding: 

156 """Record that the jury gate skipped an oversize diff. 

157 

158 Advisory jury mode keeps the finding non-blocking (``nit``). Gating jury mode 

159 escalates it to ``major`` so an oversize diff cannot bypass the blocking 

160 cross-vendor review gate. 

161 """ 

162 return Finding( 

163 severity=severity, 

164 message=( 

165 f"jury skipped: diff is {size} bytes, over the {MAX_DIFF_BYTES}-byte " 

166 "limit (ai-jury large-diff chunking not applied)" 

167 ), 

168 source="jury:skipped-oversize", 

169 path=None, 

170 line=None, 

171 anchorable=False, 

172 ) 

173 

174 

175#: The source of the finding a jury gate that **could not run** reports (#1369): no ``jury`` 

176#: CLI on the host, or an empty diff. It judged nothing, so it must never read as ``ok``. 

177#: :func:`could_not_run` reads it back; :func:`keel.gates.lone_jury_cannot_judge` turns it 

178#: into a blocking outcome when the jury is the only gate planned. 

179NOT_RUN_SOURCE = "jury:not-run" 

180 

181#: Why the jury could not run, as the finding says it. 

182NOT_RUN_NO_CLI = "the jury CLI is not available (`jury --version` failed; install ai-jury)" 

183NOT_RUN_EMPTY_DIFF = "the diff against the base branch is empty" 

184 

185 

186def _not_run_finding(reason: str) -> Finding: 

187 """Record that the jury gate judged nothing, and why (#1369). 

188 

189 ``nit``: beside another gate that judges, a jury that could not run stays the 

190 documented s8 no-op and does not hold the merge. It is reported rather than silent, 

191 and it is what marks the outcome ``SKIPPED`` rather than ``ok``. 

192 """ 

193 return Finding( 

194 severity="nit", 

195 message=f"jury did not run: {reason}; nothing was judged.", 

196 source=NOT_RUN_SOURCE, 

197 path=None, 

198 line=None, 

199 anchorable=False, 

200 ) 

201 

202 

203#: The source of the finding a jury gate reports for the panel's **consensus** (#1436). 

204CONSENSUS_SOURCE = "jury:consensus" 

205 

206 

207def panel_consensus(data: dict | str) -> str | None: 

208 """The panel's consensus in an ai-jury report, in keel's vocabulary, or ``None``. 

209 

210 The consensus is the **chair record's** ``verdict`` in the report's ``reviewers`` 

211 array (``role: chair``). ai-jury writes it from the chair's synthesis headline, or 

212 from the panel vote when the run is configured with ``decision: vote`` 

213 (``ai_jury.ballots.chair_verdict``), so one field carries both. It is read through 

214 :func:`map_verdict`, so ``APPROVE`` / ``READY`` arrive as ``LGTM``. 

215 

216 A ballot report with **no chair record** — the synthesis failed — has no consensus, and 

217 reads ``ABSTAIN``, which is what :func:`jury_verdict` posts for it too. ``None`` means 

218 the report carries **no ballots at all**: an ai-jury from before report schema 1.1, 

219 which has no ``reviewers`` array, or one whose ballots are malformed. 

220 """ 

221 try: 

222 panel = parse_panel(data) 

223 except JuryReportError: 

224 return None 

225 if panel is None: 

226 return None 

227 return panel.chair.verdict if panel.chair is not None else "ABSTAIN" 

228 

229 

230def _non_approving_consensus_finding( 

231 token: str | None, 

232 *, 

233 gating: bool, 

234 posted: bool, 

235 comment_rule: dict[str, Any] | None = None, 

236) -> Finding: 

237 """The finding emitted when a jury consensus does not approve (#1436, #1570). 

238 

239 When ``gating`` is True, the consensus is ``COMMENT``, and ``comment_rule`` 

240 (:func:`keel.evidence.comment_consensus`) is present: 

241 

242 * if ``comment_rule["applied"]`` is True, the finding degrades to ``minor`` and names 

243 how many approving review verdicts satisfied the rule; 

244 * otherwise it stays ``major`` and appends ``comment_rule["reason"]``, matching 

245 :func:`keel.evidence._jury_not_approved_finding`. 

246 """ 

247 prefix = "posted jury" if posted else "jury" 

248 if not token: 

249 return Finding( 

250 severity="major" if gating else "minor", 

251 message="posted jury verdict has no readable AI Jury verdict line.", 

252 source=CONSENSUS_SOURCE, 

253 path=None, 

254 line=None, 

255 anchorable=False, 

256 ) 

257 if gating and token == evidence.COMMENT_CONSENSUS and comment_rule is not None: 

258 if comment_rule.get("applied"): 

259 approvals = comment_rule.get("approvals", 0) 

260 required = comment_rule.get("required_approvals", approvals) 

261 return Finding( 

262 severity="minor", 

263 message=( 

264 f"{prefix} consensus is {token}; comment_consensus applied " 

265 f"({approvals} of {required} required approving review verdicts at the head)." 

266 ), 

267 source=CONSENSUS_SOURCE, 

268 path=None, 

269 line=None, 

270 anchorable=False, 

271 ) 

272 return Finding( 

273 severity="major", 

274 message=( 

275 f"{prefix} consensus is {token}, not an approval " 

276 f"(comment_consensus not met: {comment_rule.get('reason')})." 

277 ), 

278 source=CONSENSUS_SOURCE, 

279 path=None, 

280 line=None, 

281 anchorable=False, 

282 ) 

283 return Finding( 

284 severity="major" if gating else "minor", 

285 message=f"{prefix} consensus is {token}, not an approval.", 

286 source=CONSENSUS_SOURCE, 

287 path=None, 

288 line=None, 

289 anchorable=False, 

290 ) 

291 

292 

293def _consensus_finding( 

294 data: dict | str, 

295 *, 

296 gating: bool, 

297 comment_rule: dict[str, Any] | None = None, 

298) -> Finding | None: 

299 """The finding a jury gate reports when the panel's consensus does not approve (#1436). 

300 

301 Read against :data:`keel.evidence.APPROVING_VERDICTS` by the same reader the 

302 evidence gate applies to the posted ``AI Jury verdict:`` line 

303 (:func:`keel.evidence.jury_verdict_approves`), so the gates-pass and the evidence 

304 gate cannot disagree about what the panel said. ``major`` in gating mode — the gate 

305 fails — and ``minor`` in advisory mode, which reports a rejection and never gates on 

306 one, as the evidence gate does. When the opt-in ``team.jury.comment_consensus`` rule 

307 applies (:func:`keel.evidence.comment_consensus`, #1570), a ``COMMENT`` consensus is 

308 reported as a non-blocking ``minor`` finding instead. 

309 

310 **A report with no consensus fails closed in gating mode, and is silent in advisory 

311 mode.** A gating jury's verdict comment has to approve before ``keel merge`` will land 

312 the change, and a report that states no consensus cannot truthfully produce one; a 

313 gates-pass for it would certify a review that never concluded. Advisory mode keeps 

314 today's behaviour: the severity rule alone, with nothing added. 

315 """ 

316 consensus = panel_consensus(data) 

317 if consensus is None: 

318 if not gating: 

319 return None 

320 return Finding( 

321 severity="major", 

322 message=( 

323 "jury report states no panel consensus: it carries no readable `reviewers` " 

324 "ballots with a chair record (ai-jury before report schema 1.1, or a " 

325 "malformed report). A gating jury must conclude; upgrade ai-jury." 

326 ), 

327 source=CONSENSUS_SOURCE, 

328 path=None, 

329 line=None, 

330 anchorable=False, 

331 ) 

332 line = f"AI Jury verdict: {consensus}." 

333 if evidence.jury_verdict_approves(line): 

334 return None 

335 token = evidence.jury_verdict_token(line) or consensus 

336 return _non_approving_consensus_finding( 

337 token, gating=gating, posted=False, comment_rule=comment_rule 

338 ) 

339 

340 

341#: The source of the findings a jury gate carries over from a reused panel (#1437). 

342REUSED_SOURCE = "jury:reused" 

343 

344#: The source of the finding emitted when an isolated review checkout cannot be prepared (#1516). 

345REVIEW_CHECKOUT_SOURCE = "jury:review-checkout" 

346 

347 

348def reuse_posted_verdict( 

349 body: str, 

350 *, 

351 gating: bool, 

352 comment_rule: dict[str, Any] | None = None, 

353) -> tuple[bool, list[Finding]] | None: 

354 """The jury gate's outcome read off the panel already posted for the head (#1437). 

355 

356 ``body`` is the standing ``keel.jury-verdict.v1`` comment for the head 

357 (:func:`keel.evidence.standing_jury_verdict`). The gate is judged by the two rules a 

358 panel keel ran is judged by, so reusing one can never be kinder than running it: 

359 

360 * **the severity rule** — each ``<severity>: <message>`` item of the comment's findings 

361 summary (:func:`keel.artifacts.jury_verdict_summary`, the verified findings 

362 :func:`jury_verdict` posts) becomes a finding, and a critical or major one fails; 

363 * **the consensus rule** (#1436, #1570) — an ``AI Jury verdict:`` line that does not 

364 approve, or no readable line at all, is a ``major`` finding in gating mode (unless 

365 ``comment_rule`` says ``team.jury.comment_consensus`` applied to a ``COMMENT`` 

366 consensus, in which case it is a ``minor`` finding) and a ``minor`` one in advisory 

367 mode, the same as :func:`_consensus_finding` for a report. 

368 

369 ``(ok, findings)``, or ``None`` when the comment's summary cannot be read — a body keel 

370 did not render. ``None`` means *do not reuse*: the caller convenes the panel as it 

371 always did, so an unreadable comment can never stand in for a run. 

372 """ 

373 summary = artifacts.jury_verdict_summary(body) 

374 if summary is None: 

375 return None 

376 findings: list[Finding] = [] 

377 for item in summary: 

378 label, sep, _message = item.partition(":") 

379 findings.append( 

380 Finding( 

381 # An item with no `<severity>:` label is read as unknown, which 

382 # map_severity already maps to `minor`. 

383 severity=map_severity(label if sep else ""), 

384 message=f"posted jury finding — {item}", 

385 source=REUSED_SOURCE, 

386 path=None, 

387 line=None, 

388 anchorable=False, 

389 ) 

390 ) 

391 if not evidence.jury_verdict_approves(body): 

392 token = evidence.jury_verdict_token(body) 

393 findings.append( 

394 _non_approving_consensus_finding( 

395 token, gating=gating, posted=True, comment_rule=comment_rule 

396 ) 

397 ) 

398 blocked = any(f.severity in ("critical", "major") for f in findings) 

399 return (not blocked), findings 

400 

401 

402def could_not_run(findings) -> bool: 

403 """Did this jury gate result come back without running (no CLI, or an empty diff)?""" 

404 return any(f.source == NOT_RUN_SOURCE for f in findings) 

405 

406 

407#: The lines that frame a PR's base facts ahead of the diff the jury reads (#1570). 

408BASE_FACTS_LEAD = ( 

409 "Base facts — stated by the pull request's author about code outside this diff, " 

410 "which the change relies on:" 

411) 

412BASE_FACTS_CHECK = ( 

413 "Each base fact above is a claim, not a given. Check every one against this checkout " 

414 "(the head of the change). A base fact that is false, or that the change depends on and " 

415 "the checkout cannot confirm, is a finding against the change." 

416) 

417 

418 

419def with_base_facts(diff_text: str, base_facts: str | None) -> str: 

420 """``diff_text`` with the PR's base facts ahead of it, or unchanged when there are none. 

421 

422 The jury reads only the diff, so a change that relies on code outside it leaves the 

423 panel unable to check the reliance — and a panel that cannot check hedges. The facts 

424 go where ``git format-patch`` puts a commit message, before the first ``diff --git`` 

425 header, which ai-jury keeps attached to the first file rather than dropping. They are 

426 framed as claims to verify (:data:`BASE_FACTS_CHECK`): the author's word is input to 

427 the review, never a substitute for it. 

428 """ 

429 if not base_facts or not base_facts.strip() or not diff_text: 

430 return diff_text 

431 return f"{BASE_FACTS_LEAD}\n\n{base_facts.strip()}\n\n{BASE_FACTS_CHECK}\n\n{diff_text}" 

432 

433 

434def run_gate( 

435 diff_text: str, 

436 *, 

437 cwd: str | None = None, 

438 mode: str = "advisory", 

439 timeout: int = DEFAULT_JURY_TIMEOUT_S, 

440 head_sha: str | None = None, 

441 base_facts: str | None = None, 

442 comment_rule: dict[str, Any] | None = None, 

443 _run=None, 

444) -> tuple[bool, list[Finding], bool]: 

445 """Run ``jury`` on ``diff_text`` and map its findings. 

446 

447 ``base_facts`` is the PR's ``## Base facts`` section (``team.jury.base_facts``, 

448 #1570); when given, the panel reads it ahead of the diff (:func:`with_base_facts`). 

449 

450 When ``head_sha`` is provided, executes ``jury`` inside an isolated detached checkout of 

451 ``head_sha`` created via :func:`keel.swarm_runtime.review_checkout` (under ``cwd``) and 

452 removed on exit; a checkout setup failure returns a ``major`` (gating) or ``minor`` 

453 (advisory) finding from :data:`REVIEW_CHECKOUT_SOURCE`. 

454 

455 Returns ``(ok, findings, timed_out)``. ``ok`` is False when a finding blocks 

456 (critical/major) or when the run produced no verdict at all in gating mode — and, in 

457 gating mode, when the panel's **consensus** does not approve or the report states 

458 none (#1436, :func:`_consensus_finding`). Both rules hold at once: a verified major 

459 still blocks a panel that approved, and a panel that requested changes over minors 

460 alone, or abstained, no longer passes because no finding was severe. 

461 No-op when there is no diff or the ``jury`` CLI is not installed — keel does not 

462 depend on ai-jury, so an absent CLI is a legitimate no-op *for this run*, distinct 

463 from a run that started and did not finish. A no-op is not a pass: it returns one 

464 ``nit`` finding from :data:`NOT_RUN_SOURCE` saying why nothing was judged, so the 

465 outcome reads ``SKIPPED`` and a plan with no other gate blocks (#1369). It waives 

466 nothing downstream: a gating jury's ``jury-verdict`` is still required at merge (see 

467 the module docstring). 

468 

469 Three ways a run can end without a review, all handled alike — gating fails closed 

470 with a blocking ``major``, advisory surfaces a ``minor``: 

471 

472 * the diff is oversize and was never submitted, 

473 * the CLI was killed by ``timeout``, 

474 * the CLI returned no parseable verdict, whatever its exit code. 

475 

476 The last used to report ``(True, [])``: :func:`parse_findings` yields ``[]`` for 

477 unparseable output, so ``blocked`` came out False and a hung, crashed, or 

478 unreadable panel read as a clean pass. The test is deliberately *"did we parse a 

479 verdict"* rather than *"was the exit code zero"* — ai-jury exits nonzero to signal 

480 "request changes", which is a completed review whose findings must be honoured, 

481 while an exit of zero carrying unreadable output is not a review at all. 

482 """ 

483 if diff_text is None: 

484 # The diff could not be read (git failed). That is not "nothing to review": 

485 # passing here would silently remove the review gate from the merge decision, 

486 # which is the same fail-open the verdict check below exists to prevent. 

487 gating = mode == "gating" 

488 return ( 

489 (not gating), 

490 [_unreadable_diff_finding(severity="major" if gating else "minor")], 

491 False, 

492 ) 

493 if not diff_text: 

494 return True, [_not_run_finding(NOT_RUN_EMPTY_DIFF)], False 

495 diff_text = with_base_facts(diff_text, base_facts) 

496 size = len(diff_text.encode("utf-8")) 

497 if size > MAX_DIFF_BYTES: 

498 if mode == "gating": 

499 return False, [_oversize_finding(size, severity="major")], False 

500 return True, [_oversize_finding(size)], False 

501 if not available(cwd=cwd, _run=_run): 

502 return True, [_not_run_finding(NOT_RUN_NO_CLI)], False 

503 

504 run_cwd = cwd 

505 with contextlib.ExitStack() as stack: 

506 if head_sha is not None: 

507 from . import swarm_runtime 

508 

509 def _wrap_runner(cmd: list[str], path: Path) -> CommandResult: 

510 res = _run(cmd, cwd=str(path), stdin=subprocess.DEVNULL) 

511 if isinstance(res, CommandResult): 

512 return res 

513 out = getattr(res, "stdout", "") or "" 

514 err = getattr(res, "stderr", "") or "" 

515 code = getattr(res, "returncode", 0) 

516 return CommandResult( 

517 code == 0, 

518 code, 

519 out + err, 

520 stdout=out, 

521 stderr=err, 

522 ) 

523 

524 checkout_runner = _wrap_runner if _run is not None else None 

525 try: 

526 checkout = stack.enter_context( 

527 swarm_runtime.review_checkout( 

528 head_sha, name="jury", root=cwd or ".", runner=checkout_runner 

529 ) 

530 ) 

531 except swarm_runtime.ReviewCheckoutError as exc: 

532 gating = mode == "gating" 

533 return ( 

534 (not gating), 

535 [ 

536 Finding( 

537 severity="major" if gating else "minor", 

538 message=str(exc), 

539 source=REVIEW_CHECKOUT_SOURCE, 

540 ) 

541 ], 

542 False, 

543 ) 

544 run_cwd = str(checkout) 

545 fd, path = tempfile.mkstemp(suffix=".diff") 

546 try: 

547 with os.fdopen(fd, "w", encoding="utf-8") as fh: 

548 fh.write(diff_text) 

549 result = run_argv( 

550 ["jury", "--format", "json", "--diff-file", path], 

551 cwd=run_cwd, 

552 timeout=timeout, 

553 **_kw(_run), 

554 ) 

555 finally: 

556 os.unlink(path) 

557 # stdout alone: ai-jury logs its progress (`[jury] …`) to stderr, and reading the 

558 # concatenation is what made every report unparseable (#624). `parse_report` still 

559 # tolerates trailing non-JSON, for a vendor that also chats on stdout. 

560 report = parse_report(result.stdout) 

561 if report is None: 

562 gating = mode == "gating" 

563 incomplete = _incomplete_finding( 

564 result, timeout=timeout, severity="major" if gating else "minor" 

565 ) 

566 # timed_out rides along so the outcome renders as TIMEOUT rather than FAIL, 

567 # the distinction #622 established for command gates. 

568 return (not gating), [incomplete], result.timed_out 

569 consensus = _consensus_finding( 

570 result.stdout, gating=mode == "gating", comment_rule=comment_rule 

571 ) 

572 if consensus is not None: 

573 report = [*report, consensus] 

574 blocked = any(f.severity in ("critical", "major") for f in report) 

575 return (not blocked), report, False 

576 

577 

578# --------------------------------------------------------------------------- # 

579# Per-reviewer ballots (#1015) — the panel *as* the review, not beside it. 

580# --------------------------------------------------------------------------- # 

581 

582#: The ``role`` ai-jury stamps on the chair's entry in the report's ``reviewers`` 

583#: array. The chair is the consensus record, not a panelist ballot, so it renders 

584#: as the jury verdict rather than as one more review verdict. 

585CHAIR_ROLE = "chair" 

586 

587#: ai-jury ballot tokens → keel verdict vocabulary. ai-jury emits one machine 

588#: token per ballot (``REQUEST_CHANGES``, never ``REQUEST CHANGES``) in either the 

589#: code or the ``--issue`` vocabulary; keel's verdicts are ``LGTM`` / 

590#: ``REQUEST_CHANGES`` / ``COMMENT`` / ``ABSTAIN``. An unknown token is carried 

591#: through verbatim rather than folded into ``LGTM``: inventing an approval for a 

592#: stance keel does not recognise is the one mapping error that cannot be undone. 

593_VERDICT = { 

594 "APPROVE": "LGTM", 

595 "READY": "LGTM", 

596 "REQUEST_CHANGES": "REQUEST_CHANGES", 

597 "NEEDS_INFO": "REQUEST_CHANGES", 

598 "COMMENT": "COMMENT", 

599 "UNCLEAR": "COMMENT", 

600 "ABSTAIN": "ABSTAIN", 

601 "NO_QUORUM": "ABSTAIN", 

602} 

603 

604#: Files a ballot's scope line names before it starts counting instead. 

605_SCOPE_FILES = 8 

606 

607#: The verification status ai-jury stamps on a consensus group the verification 

608#: round upheld. Only these findings gate: an unsupported or unverified claim is 

609#: reported, never merged against. 

610VERIFIED_STATUS = "verified" 

611 

612 

613class JuryReportError(ValueError): 

614 """Raised when an ai-jury report cannot be read as a panel of ballots.""" 

615 

616 

617def map_verdict(verdict: str) -> str: 

618 """Map an ai-jury ballot token onto keel's verdict vocabulary.""" 

619 token = (verdict or "").strip().upper().replace(" ", "_").replace("-", "_") 

620 if not token: 

621 return "ABSTAIN" 

622 return _VERDICT.get(token, token) 

623 

624 

625@dataclass(frozen=True) 

626class Ballot: 

627 """One panelist's own stance, with the provenance that makes it evidence.""" 

628 

629 reviewer: str 

630 verdict: str 

631 vendor: str | None = None 

632 model: str | None = None 

633 verified_count: int = 0 

634 round1_ok: bool = True 

635 findings: tuple[dict[str, Any], ...] = () 

636 scope: str | None = None 

637 testing: str | None = None 

638 counts_as_review: bool | None = None 

639 scope_substantive: bool | None = None 

640 abstention_cause: str | None = None 

641 

642 def as_review(self) -> dict[str, Any]: 

643 """This ballot in the ``keel review --reviews`` bundle shape.""" 

644 return { 

645 "reviewer": self.reviewer, 

646 "verdict": self.verdict, 

647 "scope": ballot_scope(self), 

648 "findings": [dict(finding) for finding in self.findings], 

649 "testing": ballot_testing(self), 

650 "vendor": self.vendor, 

651 "model": self.model, 

652 } 

653 

654 

655def _review_ballots(ballots: tuple[Ballot, ...]) -> tuple[Ballot, ...]: 

656 """Ballots that count as reviews — the one definition :class:`Panel` consumes.""" 

657 return tuple(ballot for ballot in ballots if ballot_is_review(ballot)) 

658 

659 

660@dataclass(frozen=True) 

661class Panel: 

662 """A parsed ai-jury panel: the panelist ballots and the chair's consensus.""" 

663 

664 ballots: tuple[Ballot, ...] = () 

665 chair: Ballot | None = None 

666 verified: tuple[dict[str, Any], ...] = () 

667 

668 @property 

669 def size(self) -> int: 

670 """Reviews this panel produced — the reviewer count the evidence gate sizes. 

671 

672 Aligns with ai-jury's ``is_review``: an abstention, an empty ballot, or a 

673 ``counts_as_review: false`` record is not a review and does not inflate 

674 ``panelists`` / ``jury_panel_size``. :func:`parse_panel` already drops 

675 those from :attr:`ballots`; this property re-applies the same predicate 

676 so a hand-built panel cannot disagree with the posting path. 

677 """ 

678 return len(_review_ballots(self.ballots)) 

679 

680 @property 

681 def vendors(self) -> tuple[str, ...]: 

682 """Distinct declared vendors across the reviews, in panel order. 

683 

684 Lower-cased and de-duplicated exactly as :func:`keel.evidence.distinct_vendor_check` 

685 reads the posted ``vendor:`` lines, so the count declared on the jury verdict and 

686 the count the evidence gate recomputes from the verdicts cannot disagree. 

687 Abstaining seats are not reviews and do not contribute a vendor. 

688 """ 

689 seen: list[str] = [] 

690 for ballot in _review_ballots(self.ballots): 

691 vendor = (ballot.vendor or "").strip().lower() 

692 if vendor and vendor not in seen: 

693 seen.append(vendor) 

694 return tuple(seen) 

695 

696 def reviews(self) -> tuple[dict[str, Any], ...]: 

697 """Review ballots in the ``--reviews`` bundle shape. 

698 

699 Non-reviews are omitted: posting them as head-pinned ``review-verdict-*`` 

700 evidence is the defect this mapping exists to close. 

701 """ 

702 return tuple(ballot.as_review() for ballot in _review_ballots(self.ballots)) 

703 

704 

705def _finding_record(raw: Any) -> dict[str, Any] | None: 

706 """One ai-jury finding in keel's finding shape (``file``→``path``, ``claim``→``message``).""" 

707 if not isinstance(raw, dict): 

708 return None 

709 line = raw.get("line") 

710 return { 

711 "severity": map_severity(raw.get("severity", "")), 

712 "path": raw.get("file") or None, 

713 "line": line if isinstance(line, int) else None, 

714 "message": raw.get("claim") or "(jury finding)", 

715 } 

716 

717 

718def _ballot_findings(raw: Any, findings: list[Any]) -> tuple[dict[str, Any], ...]: 

719 """Resolve a ballot's ``findings`` index list against the report's findings array. 

720 

721 Out-of-range and non-integer indexes are dropped rather than raising: the 

722 ballot's stance is the evidence, and a report whose indexes do not line up 

723 must still produce a verdict that says so with the findings it *can* resolve. 

724 """ 

725 if not isinstance(raw, list): 

726 return () 

727 records: list[dict[str, Any]] = [] 

728 for index in raw: 

729 if not isinstance(index, int) or isinstance(index, bool): 

730 continue 

731 if not 0 <= index < len(findings): 

732 continue 

733 record = _finding_record(findings[index]) 

734 if record is not None: 

735 records.append(record) 

736 return tuple(records) 

737 

738 

739def _text_field(raw: dict[str, Any], key: str) -> str | None: 

740 """A non-empty string field, or ``None`` when absent / blank / the wrong type.""" 

741 value = raw.get(key) 

742 if isinstance(value, str) and value.strip(): 

743 return value.strip() 

744 return None 

745 

746 

747def _bool_field(raw: dict[str, Any], key: str) -> bool | None: 

748 """A JSON boolean field, or ``None`` when absent or not a bool. 

749 

750 Integers are refused: ``1``/``0`` are not the schema ≥1.2 flags, and treating 

751 them as booleans would let a malformed report opt a ballot into the review 

752 count. 

753 """ 

754 value = raw.get(key) 

755 if isinstance(value, bool): 

756 return value 

757 return None 

758 

759 

760#: Finding severities that make a panelist's stance a rejection rather than a remark. 

761_BLOCKING_SEVERITIES = frozenset({"critical", "major"}) 

762 

763 

764def ballot_verdict( 

765 verdict: str, findings: tuple[dict[str, Any], ...] = (), *, resolved: bool = True 

766) -> str: 

767 """A panelist's ballot in keel's vocabulary: :func:`map_verdict`, except ``COMMENT``. 

768 

769 ai-jury derives a code-review ballot from the worst finding the panelist raised that 

770 the verifier did not reject: critical/major is ``REQUEST_CHANGES``, minor/nit only is 

771 ``COMMENT``, none is ``APPROVE`` (``ai_jury.voting``). So a ``COMMENT`` ballot is "I 

772 have only non-blocking remarks" — what a host reviewer posts as ``APPROVE`` with nits — 

773 and it counts as approving (``LGTM``), its findings still listed. Without this, two 

774 panelists with nits held a merge the chair approved (#1500). It stays ``COMMENT`` if 

775 any finding it carries is critical or major. Only ballots: the panel's consensus is 

776 read by :func:`map_verdict`, and a consensus of ``COMMENT`` still holds; ``UNCLEAR`` 

777 (the issue-mode remark) still holds too. ``resolved`` is ``False`` when some of the 

778 ballot's finding indexes did not resolve: what they pointed at is unknown, so the 

779 ``COMMENT`` is kept rather than read as non-blocking. 

780 """ 

781 token = (verdict or "").strip().upper().replace(" ", "_").replace("-", "_") 

782 if ( 

783 token == "COMMENT" 

784 and resolved 

785 and not any( 

786 str(finding.get("severity", "")).strip().lower() in _BLOCKING_SEVERITIES 

787 for finding in findings 

788 ) 

789 ): 

790 return "LGTM" 

791 return map_verdict(verdict) 

792 

793 

794def _ballot(raw: Any, findings: list[Any], *, position: int) -> Ballot: 

795 if not isinstance(raw, dict): 

796 raise JuryReportError(f"jury report reviewer #{position} must be a JSON object") 

797 name = raw.get("name") 

798 if not isinstance(name, str) or not name.strip(): 

799 raise JuryReportError(f"jury report reviewer #{position} requires a non-empty 'name'") 

800 vendor = raw.get("vendor") 

801 model = raw.get("model") 

802 verified = raw.get("verified_count") 

803 raw_indexes = raw.get("findings") 

804 ballot_findings = _ballot_findings(raw_indexes, findings) 

805 # Every index resolved (or none was given): only then is a COMMENT known to be minor. 

806 resolved = raw_indexes is None or ( 

807 isinstance(raw_indexes, list) and len(ballot_findings) == len(raw_indexes) 

808 ) 

809 # The chair record is the consensus, not a panelist: its COMMENT still holds. 

810 is_chair = (raw.get("role") or "") == CHAIR_ROLE 

811 return Ballot( 

812 reviewer=name.strip(), 

813 verdict=map_verdict(raw.get("verdict", "")) 

814 if is_chair 

815 else ballot_verdict(raw.get("verdict", ""), ballot_findings, resolved=resolved), 

816 vendor=vendor.strip() if isinstance(vendor, str) and vendor.strip() else None, 

817 model=model.strip() if isinstance(model, str) and model.strip() else None, 

818 verified_count=verified 

819 if isinstance(verified, int) and not isinstance(verified, bool) 

820 else 0, 

821 round1_ok=bool(raw.get("round1_ok", True)), 

822 findings=ballot_findings, 

823 scope=_text_field(raw, "scope"), 

824 testing=_text_field(raw, "testing"), 

825 counts_as_review=_bool_field(raw, "counts_as_review"), 

826 scope_substantive=_bool_field(raw, "scope_substantive"), 

827 abstention_cause=_text_field(raw, "abstention_cause"), 

828 ) 

829 

830 

831def _verified_records(data: dict) -> tuple[dict[str, Any], ...]: 

832 """Consensus-group representatives the verification round upheld. 

833 

834 These are the findings that gate. ai-jury verifies a consensus group and 

835 stamps ``verification_status``; keel's own rule — critical/major block — 

836 applies to the *upheld* ones only, so a claim the panel could not support 

837 never holds a merge. 

838 """ 

839 records: list[dict[str, Any]] = [] 

840 for group in data.get("consensus") or []: 

841 if not isinstance(group, dict): 

842 continue 

843 if (group.get("verification_status") or "") != VERIFIED_STATUS: 

844 continue 

845 record = _finding_record(group.get("representative")) 

846 if record is not None: 

847 reviewers = group.get("reviewers") 

848 record["reviewers"] = ( 

849 [name for name in reviewers if isinstance(name, str)] 

850 if isinstance(reviewers, list) 

851 else [] 

852 ) 

853 records.append(record) 

854 return tuple(records) 

855 

856 

857def parse_panel(data: dict | str) -> Panel | None: 

858 """Parse an ai-jury JSON report into a :class:`Panel`, or ``None``. 

859 

860 ``None`` means *this is not a report carrying per-reviewer ballots* — an 

861 unparseable document, or a pre-schema-1.1 report with no ``reviewers`` array. 

862 The caller turns that into an actionable error (upgrade ai-jury, or supply a 

863 ``--reviews`` bundle); it is deliberately not an exception, because "not a 

864 ballot report" is the same question :func:`parse_report` answers for findings. 

865 

866 A report that *does* carry ballots but carries them malformed raises 

867 :class:`JuryReportError`: dropping a panelist would silently post fewer 

868 verdicts than the panel produced, which is the one failure this whole path 

869 exists to prevent. 

870 

871 Only ballots that :func:`ballot_is_review` accepts enter :attr:`Panel.ballots`. 

872 An ``ABSTAIN``, a ``counts_as_review: false`` record, or an older empty 

873 ballot is parsed and then dropped, so it cannot inflate ``panel.size`` or 

874 become a posted ``review-verdict-*``. 

875 """ 

876 if isinstance(data, str): 

877 try: 

878 data, _end = json.JSONDecoder().raw_decode(data.lstrip()) 

879 except json.JSONDecodeError: 

880 return None 

881 if not isinstance(data, dict): 

882 return None 

883 raw_reviewers = data.get("reviewers") 

884 if not isinstance(raw_reviewers, list): 

885 return None 

886 findings = list(data.get("findings") or []) 

887 ballots: list[Ballot] = [] 

888 chair: Ballot | None = None 

889 for position, raw in enumerate(raw_reviewers, start=1): 

890 ballot = _ballot(raw, findings, position=position) 

891 if isinstance(raw, dict) and (raw.get("role") or "") == CHAIR_ROLE: 

892 chair = ballot 

893 continue 

894 if ballot_is_review(ballot): 

895 ballots.append(ballot) 

896 return Panel(ballots=tuple(ballots), chair=chair, verified=_verified_records(data)) 

897 

898 

899def _finding_paths(ballot: Ballot) -> list[str]: 

900 """Distinct file paths this ballot's own findings named, in first-seen order.""" 

901 files: list[str] = [] 

902 for finding in ballot.findings: 

903 path = finding.get("path") 

904 if isinstance(path, str) and path.strip() and path not in files: 

905 files.append(path.strip()) 

906 return files 

907 

908 

909def ballot_is_review(ballot: Ballot) -> bool: 

910 """Whether this panelist ballot counts as a review (ai-jury ``is_review``). 

911 

912 A review is a panelist whose scope is substantive and whose verdict is not 

913 ``ABSTAIN``. The chair is split off before this predicate runs. 

914 

915 The flags a schema ≥1.2 report declares — ``counts_as_review`` and 

916 ``scope_substantive`` — can only ever **remove** a ballot here, never admit 

917 one that carries nothing. ai-jury derives ``counts_as_review`` from 

918 ``scope_substantive``, which is itself derived from the ``scope`` prose, so a 

919 record claiming ``counts_as_review: true`` with no ``scope`` and no finding 

920 is not a clean review it produced; it is internally inconsistent, and 

921 admitting it means keel writing the substance the report failed to supply. 

922 That is the escape hatch #1150 is about, so the ambiguous record fails closed 

923 like every other one. 

924 

925 What is left is the fact the scope line reads: a ballot counts when the 

926 report gave prose to post or a path to name. Older reports carry no flags and 

927 are decided by that same fact, so a schema-1.1 empty ``APPROVE`` is dropped 

928 rather than dressed up. 

929 """ 

930 if ballot.verdict == "ABSTAIN": 

931 return False 

932 if ballot.counts_as_review is False or ballot.scope_substantive is False: 

933 return False 

934 return bool(ballot.scope) or bool(_finding_paths(ballot)) 

935 

936 

937def _abstention_scope(ballot: Ballot) -> str: 

938 """An explicitly anchorless scope: no ``Checked …``, no path, no backtick.""" 

939 cause = (ballot.abstention_cause or "").replace("_", " ") 

940 if cause: 

941 return f"ai-jury panelist {ballot.reviewer} did not review ({cause})." 

942 return f"ai-jury panelist {ballot.reviewer} did not review." 

943 

944 

945def ballot_scope(ballot: Ballot) -> str: 

946 """The scope line keel renders for a panelist ballot. 

947 

948 A ballot that is not a review never gets a substance-passing opener: keel 

949 used to always start with ``Checked the changed-file diff…``, which is its 

950 own :func:`keel.evidence.verdict_substance` escape hatch, so an ``ABSTAIN`` 

951 still passed the gate by construction. 

952 

953 Every remaining branch renders something the report actually supplied. A 

954 schema ≥1.2 ballot carries its own ``scope`` and that prose is posted as 

955 written — including the clean review that read the diff and found nothing, 

956 which ai-jury describes itself rather than leaving keel to. Otherwise the 

957 ballot named paths, and the ``checked …`` line is built from them. There is 

958 no third case: :func:`ballot_is_review` admits a ballot only when one of 

959 those two is true, so this function never has to invent a scope for a ballot 

960 it was told to post. 

961 """ 

962 if not ballot_is_review(ballot): 

963 return _abstention_scope(ballot) 

964 if ballot.scope: 

965 return ballot.scope 

966 # Non-empty: `ballot_is_review` accepted this ballot, and with no `scope` 

967 # prose the only way it could have is by naming a path. 

968 files = _finding_paths(ballot) 

969 opening = f"Checked the changed-file diff as ai-jury panelist {ballot.reviewer}" 

970 listed = ", ".join(files[:_SCOPE_FILES]) 

971 more = len(files) - _SCOPE_FILES 

972 suffix = f" (+{more} more)" if more > 0 else "" 

973 return f"{opening}; named {len(files)} file(s): {listed}{suffix}." 

974 

975 

976def ballot_testing(ballot: Ballot) -> str: 

977 """The testing line keel renders for a panelist ballot. 

978 

979 Schema ≥1.2 reports carry their own ``testing`` prose; that is preferred 

980 when present. Otherwise the panel's verification round *is* the ballot's 

981 testing note: it is the only check ai-jury performs on a reviewer's claims, 

982 and a ballot whose claims were never upheld must say so rather than borrow 

983 the PR's own testing section. 

984 """ 

985 if ballot.testing: 

986 return ballot.testing 

987 if ballot.verified_count > 0: 

988 note = ( 

989 f"ai-jury verification upheld {ballot.verified_count} consensus " 

990 "group(s) this panelist joined." 

991 ) 

992 else: 

993 note = "ai-jury verification upheld no consensus group from this panelist." 

994 if not ballot.round1_ok: 

995 return f"The panelist's adapter reported a failed run; its output was still read. {note}" 

996 return note 

997 

998 

999def verified_findings(panel: Panel) -> list[Finding]: 

1000 """Verified consensus findings as keel :class:`~keel.findings.Finding`s. 

1001 

1002 This is the s9 input: ``critical``/``major`` block, ``minor`` is a gated 

1003 suggestion, ``nit`` is advisory — the same mapping a host reviewer's findings 

1004 get, which is the whole point of the panel being the review rather than a 

1005 second opinion beside it. 

1006 

1007 Findings verified by consensus are carried first. Any minor finding raised 

1008 by a panelist that was not part of a verified consensus group is carried 

1009 as a single-reviewer minor tagged with its source (#1546), so a panelist's 

1010 non-blocking remark reaches the s9 fix synthesis instead of being dropped. 

1011 """ 

1012 out: list[Finding] = [] 

1013 seen: set[tuple[str | None, int | None, str]] = set() 

1014 for record in panel.verified: 

1015 reviewers = record.get("reviewers") or [] 

1016 source = f"jury:{reviewers[0]}" if reviewers else "jury:consensus" 

1017 path = record.get("path") 

1018 line = record.get("line") 

1019 message = record.get("message") or "(jury finding)" 

1020 seen.add((path, line, message)) 

1021 out.append( 

1022 Finding( 

1023 severity=record["severity"], 

1024 message=message, 

1025 source=source, 

1026 path=path, 

1027 line=line, 

1028 anchorable=bool(path) and isinstance(line, int), 

1029 ) 

1030 ) 

1031 for ballot in _review_ballots(panel.ballots): 

1032 for finding in ballot.findings: 

1033 if finding.get("severity") != "minor": 

1034 continue 

1035 path = finding.get("path") 

1036 line = finding.get("line") 

1037 message = finding.get("message") or "(jury finding)" 

1038 key = (path, line, message) 

1039 if key in seen: 

1040 continue 

1041 seen.add(key) 

1042 out.append( 

1043 Finding( 

1044 severity="minor", 

1045 message=message, 

1046 source=f"jury:{ballot.reviewer}", 

1047 path=path, 

1048 line=line, 

1049 anchorable=bool(path) and isinstance(line, int), 

1050 ) 

1051 ) 

1052 return out 

1053 

1054 

1055def jury_verdict(panel: Panel) -> dict[str, Any]: 

1056 """The ``render_jury_verdict`` arguments for a parsed panel. 

1057 

1058 The chair's ballot is the consensus record — that is what the jury verdict 

1059 comment has always been — and the panel's own size and vendor count ride 

1060 along on it, because the posted verdict is the only channel by which either 

1061 reaches a hosted evidence check (see :func:`keel.artifacts.render_jury_verdict`). 

1062 """ 

1063 chair = panel.chair 

1064 summary = [f"{record['severity']}: {record['message']}" for record in panel.verified] 

1065 return { 

1066 "verdict": chair.verdict if chair is not None else "ABSTAIN", 

1067 "participants": [ 

1068 f"{ballot.reviewer} ({ballot.vendor})" if ballot.vendor else ballot.reviewer 

1069 for ballot in _review_ballots(panel.ballots) 

1070 ], 

1071 "participating_vendors": len(panel.vendors), 

1072 "panelists": panel.size, 

1073 "findings_summary": summary, 

1074 "remaining_risks": None if summary else "none identified", 

1075 }