Coverage for src/keel/jury.py: 100%
69 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-18 12:05 +0000
1"""The ``jury`` built-in gate — run the ai-jury CLI on the diff (optional, fail-soft).
3keel does **not** depend on ai-jury. If the ``jury`` CLI is on PATH, this gate runs it on
4the change's diff and maps its findings into keel :class:`~keel.findings.Finding`s; if it is
5absent, the gate is a fail-soft no-op (the flow runs with or without jury). Parsing is pure
6and unit-tested; the subprocess is behind the injectable ``_run`` seam.
7"""
9from __future__ import annotations
11import json
12import os
13import tempfile
15from .findings import Finding
16from .model import DEFAULT_JURY_TIMEOUT_S
17from .runner import CommandResult, run_argv
19#: ai-jury severities → keel severities (unknown ⇒ ``minor``).
20_SEVERITY = {
21 "critical": "critical", "blocker": "critical",
22 "major": "major",
23 "minor": "minor",
24 "nit": "nit", "info": "nit", "note": "nit",
25}
27MAX_DIFF_BYTES = 1_000_000
30def map_severity(severity: str) -> str:
31 """Map an ai-jury severity onto a keel severity (default ``minor``)."""
32 return _SEVERITY.get((severity or "").strip().lower(), "minor")
35def parse_report(data: dict | str) -> list[Finding] | None:
36 """Map an ai-jury JSON report into Findings, or ``None`` if it is not a report.
38 The ``None`` return is the point: it separates *"the panel reviewed the diff and
39 found nothing"* from *"this output is not a verdict at all"*, which
40 :func:`parse_findings` collapses into the same empty list. Only the caller that
41 decides whether a gate passed needs that distinction — see :func:`run_gate`.
43 Tolerates trailing non-JSON. :func:`keel.runner.run_argv` hands back
44 ``stdout + stderr`` concatenated, and ai-jury logs its progress to stderr, so a
45 real report is followed by ``[jury] …`` lines. A strict ``json.loads`` rejects the
46 whole thing and silently loses every finding.
47 """
48 if isinstance(data, str):
49 try:
50 data, _end = json.JSONDecoder().raw_decode(data.lstrip())
51 except json.JSONDecodeError:
52 return None
53 if not isinstance(data, dict) or "findings" not in data:
54 return None
55 return _findings_from(data)
58def parse_findings(data: dict | str) -> list[Finding]:
59 """Map an ai-jury JSON report (dict or raw string) into keel Findings.
61 Unparseable input yields ``[]``. Use :func:`parse_report` when the difference
62 between "no findings" and "no report" matters.
63 """
64 return parse_report(data) or []
67def _findings_from(data: dict) -> list[Finding]:
68 out: list[Finding] = []
69 for f in data.get("findings") or []:
70 path = f.get("file")
71 line = f.get("line")
72 line = line if isinstance(line, int) else None
73 out.append(Finding(
74 severity=map_severity(f.get("severity", "")),
75 message=f.get("claim") or "(jury finding)",
76 source=f"jury:{f.get('reviewer') or 'consensus'}",
77 path=path,
78 line=line,
79 anchorable=bool(path) and line is not None,
80 ))
81 return out
84def _kw(_run):
85 return {"_run": _run} if _run is not None else {}
88def available(*, cwd: str | None = None, _run=None) -> bool:
89 """True if the ``jury`` CLI is callable."""
90 return run_argv(["jury", "--version"], cwd=cwd, timeout=30, **_kw(_run)).ok
93def _incomplete_finding(
94 result: CommandResult, *, timeout: int, severity: str = "nit"
95) -> Finding:
96 """Record that the jury CLI ran but produced no verdict.
98 A timeout, or a nonzero exit whose output carries no parseable findings, means the
99 panel never reached a conclusion. That is emphatically **not** a clean pass — it is
100 the *absence* of a review — so in gating mode it fails closed exactly as an oversize
101 diff does. The timeout case is named apart from a crash so the operator can tell a
102 slow panel from a broken one.
103 """
104 if result.timed_out:
105 detail = (f"timed out after {timeout}s; no verdict was produced. Raise "
106 "knobs.jury_timeout_s if the panel legitimately needs longer")
107 else:
108 detail = (f"exited {result.code} without a parseable verdict; the panel did not "
109 "complete")
110 return Finding(
111 severity=severity,
112 message=f"jury run incomplete: the jury CLI {detail}.",
113 source="jury:incomplete-run",
114 path=None,
115 line=None,
116 anchorable=False,
117 )
120def _unreadable_diff_finding(*, severity: str = "minor") -> Finding:
121 """Record that the diff itself could not be read, so no review was possible."""
122 return Finding(
123 severity=severity,
124 message=("jury could not run: the diff could not be read from git (is the base "
125 "branch fetched locally? a shallow or single-branch clone cannot "
126 "resolve base...HEAD). No review was performed."),
127 source="jury:unreadable-diff",
128 path=None,
129 line=None,
130 anchorable=False,
131 )
134def _oversize_finding(size: int, *, severity: str = "nit") -> Finding:
135 """Record that the jury gate skipped an oversize diff.
137 Advisory jury mode keeps the finding non-blocking (``nit``). Gating jury mode
138 escalates it to ``major`` so an oversize diff cannot bypass the blocking
139 cross-vendor review gate.
140 """
141 return Finding(
142 severity=severity,
143 message=(f"jury skipped: diff is {size} bytes, over the {MAX_DIFF_BYTES}-byte "
144 "limit (ai-jury large-diff chunking not applied)"),
145 source="jury:skipped-oversize",
146 path=None,
147 line=None,
148 anchorable=False,
149 )
152def run_gate(
153 diff_text: str,
154 *,
155 cwd: str | None = None,
156 mode: str = "advisory",
157 timeout: int = DEFAULT_JURY_TIMEOUT_S,
158 _run=None,
159) -> tuple[bool, list[Finding], bool]:
160 """Run ``jury`` on ``diff_text`` and map its findings.
162 Returns ``(ok, findings, timed_out)``. ``ok`` is False when a finding blocks
163 (critical/major) or when the run produced no verdict at all in gating mode.
164 Fail-soft no-op when there is no diff or the ``jury`` CLI is not installed — keel
165 does not depend on ai-jury, so an absent CLI is a legitimate no-op, distinct from
166 a run that started and did not finish.
168 Three ways a run can end without a review, all handled alike — gating fails closed
169 with a blocking ``major``, advisory surfaces a ``minor``:
171 * the diff is oversize and was never submitted,
172 * the CLI was killed by ``timeout``,
173 * the CLI returned no parseable verdict, whatever its exit code.
175 The last used to report ``(True, [])``: :func:`parse_findings` yields ``[]`` for
176 unparseable output, so ``blocked`` came out False and a hung, crashed, or
177 unreadable panel read as a clean pass. The test is deliberately *"did we parse a
178 verdict"* rather than *"was the exit code zero"* — ai-jury exits nonzero to signal
179 "request changes", which is a completed review whose findings must be honoured,
180 while an exit of zero carrying unreadable output is not a review at all.
181 """
182 if diff_text is None:
183 # The diff could not be read (git failed). That is not "nothing to review":
184 # passing here would silently remove the review gate from the merge decision,
185 # which is the same fail-open the verdict check below exists to prevent.
186 gating = mode == "gating"
187 return (not gating), [_unreadable_diff_finding(
188 severity="major" if gating else "minor")], False
189 if not diff_text:
190 return True, [], False
191 size = len(diff_text.encode("utf-8"))
192 if size > MAX_DIFF_BYTES:
193 if mode == "gating":
194 return False, [_oversize_finding(size, severity="major")], False
195 return True, [_oversize_finding(size)], False
196 if not available(cwd=cwd, _run=_run):
197 return True, [], False
198 fd, path = tempfile.mkstemp(suffix=".diff")
199 try:
200 with os.fdopen(fd, "w", encoding="utf-8") as fh:
201 fh.write(diff_text)
202 result = run_argv(["jury", "--format", "json", "--diff-file", path],
203 cwd=cwd, timeout=timeout, **_kw(_run))
204 finally:
205 os.unlink(path)
206 # stdout alone: ai-jury logs its progress (`[jury] …`) to stderr, and reading the
207 # concatenation is what made every report unparseable (#624). `parse_report` still
208 # tolerates trailing non-JSON, for a vendor that also chats on stdout.
209 report = parse_report(result.stdout)
210 if report is None:
211 gating = mode == "gating"
212 incomplete = _incomplete_finding(
213 result, timeout=timeout, severity="major" if gating else "minor")
214 # timed_out rides along so the outcome renders as TIMEOUT rather than FAIL,
215 # the distinction #622 established for command gates.
216 return (not gating), [incomplete], result.timed_out
217 blocked = any(f.severity in ("critical", "major") for f in report)
218 return (not blocked), report, False