Coverage for src/keel/revertcheck.py: 100%

531 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 19:17 +0000

1"""The opt-in ``revert-check`` gate (#1289): does any test notice when a change is undone? 

2 

3Coverage proves that a line *ran*, not that an assertion *depends* on it. With 

4``fail_under = 100`` enforced, "maintained 100 % coverage" is true of every merged pull 

5request before anyone writes it, and an audit of 14 closed fixes found three whose tests 

6still passed with the fix removed at the time their issue was closed (#877, and one arm each 

7of #871 and #879); later audits added tests that catch them, and when #1289 recorded the audit 

8on 2026-09-22 all 14 failed without their fix. This gate asks the question coverage cannot: for each 

9production change on the branch, revert **that change alone** in a scratch worktree, run 

10the project's tests, and require a test to **fail as an assertion**. 

11 

12It is mutation testing scoped to one mutation per change — the change itself. 

13 

14This module is the pure half, and deterministic: 

15 

16* :func:`resolve` — ``knobs.revert_check`` (+ ``build_gate_cmd``, ``gate_timeout_s``) -> 

17 :class:`Settings`; 

18* :func:`test_paths` — where the tests live, from ``policy_pack.test_groups.*.test_paths``; 

19* :func:`parse_diff` / :func:`plan_changes` — the branch's unified diff -> the production 

20 changes to revert, each with the patch that undoes it; 

21* :func:`read_output` / :func:`classify` — a test run's output -> *caught* (a test failed as 

22 an assertion), *errored*, *unnoticed*, *timed out* or *unreadable*; 

23* :func:`execute` — the bounded loop (``max_changes``, ``budget_s``), driven through an 

24 injected ``run`` callable and ``clock``, so the loop is tested offline; 

25* :func:`judge` — the gate's verdict and findings. 

26 

27The git and subprocess work — the scratch worktree, ``git apply -R``, the test command — 

28lives in :mod:`keel.cli` through :mod:`keel.git` and :mod:`keel.runner`; nothing here 

29touches the file system. 

30 

31**What it does not do.** The unit is a git hunk (or a file), not a *behaviour*: #871's 

32guarded and unguarded arms shared one hunk, so a per-hunk check passes while half that fix 

33is unpinned. And a passing check is necessary, not sufficient: #873's test fails without 

34its fix and still could not see the regression the fix shipped (#1268), because its fixture 

35made the fix and the bug agree. Both remain a reviewer's questions. 

36""" 

37 

38from __future__ import annotations 

39 

40import ast 

41import fnmatch 

42import posixpath 

43import re 

44from collections.abc import Callable, Mapping, Sequence 

45from dataclasses import dataclass 

46from typing import Any 

47 

48from .findings import Finding 

49 

50#: The gate id, as ``gates:`` lists it. 

51GATE_ID = "revert-check" 

52 

53#: ``knobs.revert_check.unit``: revert each hunk alone (the default), or each file. 

54UNIT_HUNK = "hunk" 

55UNIT_FILE = "file" 

56UNITS = (UNIT_HUNK, UNIT_FILE) 

57 

58#: At most this many changes are reverted per run unless ``max_changes`` says otherwise. 

59DEFAULT_MAX_CHANGES = 10 

60#: Wall-clock seconds the whole check — the baseline run and every revert — may spend. 

61DEFAULT_BUDGET_S = 1800 

62 

63#: What counts as production when ``knobs.revert_check.paths`` is not set: a changed file 

64#: with a source-code suffix that is not under the project's test paths. Docs, config, 

65#: lockfiles and data are left out — reverting a README cannot make a test fail, and 

66#: reporting it as unnoticed would be noise, not evidence. 

67SOURCE_SUFFIXES: tuple[str, ...] = ( 

68 ".py", 

69 ".pyi", 

70 ".js", 

71 ".jsx", 

72 ".mjs", 

73 ".cjs", 

74 ".ts", 

75 ".tsx", 

76 ".go", 

77 ".rs", 

78 ".java", 

79 ".kt", 

80 ".kts", 

81 ".scala", 

82 ".rb", 

83 ".php", 

84 ".cs", 

85 ".fs", 

86 ".c", 

87 ".h", 

88 ".cc", 

89 ".cpp", 

90 ".cxx", 

91 ".hpp", 

92 ".m", 

93 ".mm", 

94 ".swift", 

95 ".dart", 

96 ".ex", 

97 ".exs", 

98 ".erl", 

99 ".clj", 

100 ".lua", 

101 ".pl", 

102 ".pm", 

103 ".r", 

104 ".sh", 

105 ".bash", 

106) 

107 

108# Per-change results. 

109CAUGHT = "caught" # a test failed as an assertion 

110ERRORED = "errored" # tests failed, none as an assertion (an exception, an import error) 

111UNNOTICED = "unnoticed" # the test command passed with the change reverted 

112TIMED_OUT = "timed-out" 

113UNREADABLE = "unreadable" # the command failed and its output says no test failed 

114NOT_APPLIED = "not-applied" # the reverse patch did not apply on HEAD 

115 

116_BLOCK = "major" 

117 

118 

119@dataclass(frozen=True) 

120class Settings: 

121 """The resolved ``knobs.revert_check`` for one run.""" 

122 

123 #: The test command each revert runs; ``None`` when neither knob sets one. 

124 cmd: str | None 

125 #: The knob :attr:`cmd` came from (``knobs.build_gate_cmd`` when neither sets one). 

126 cmd_source: str 

127 #: Globs that select production files; empty means :data:`SOURCE_SUFFIXES`. 

128 paths: tuple[str, ...] 

129 unit: str 

130 max_changes: int 

131 budget_s: int 

132 #: The wall-clock limit of one test run (``knobs.gate_timeout_s``). 

133 run_timeout_s: int 

134 

135 

136def resolve( 

137 knob: Mapping[str, Any] | None, *, build_cmd: str | None, gate_timeout_s: int 

138) -> Settings: 

139 """``knobs.revert_check`` -> :class:`Settings`, defaults filled in. 

140 

141 The test command is ``knobs.revert_check.cmd`` when set, else ``knobs.build_gate_cmd``: 

142 a project whose suite is slow can point the check at the tests that matter without 

143 changing what its ``build`` gate runs. The schema owns the value shapes; anything it 

144 would refuse reads as the default here, because this resolver runs on every gate run. 

145 """ 

146 raw = knob if isinstance(knob, Mapping) else {} 

147 own = raw.get("cmd") 

148 if isinstance(own, str) and own.strip(): 

149 cmd: str | None = own 

150 source = "knobs.revert_check.cmd" 

151 else: 

152 cmd = build_cmd if isinstance(build_cmd, str) and build_cmd.strip() else None 

153 source = "knobs.build_gate_cmd" 

154 paths = raw.get("paths") 

155 unit = raw.get("unit") 

156 return Settings( 

157 cmd=cmd, 

158 cmd_source=source, 

159 paths=tuple(_strings(paths)), 

160 unit=unit if unit in UNITS else UNIT_HUNK, 

161 max_changes=_positive(raw.get("max_changes"), DEFAULT_MAX_CHANGES), 

162 budget_s=_positive(raw.get("budget_s"), DEFAULT_BUDGET_S), 

163 run_timeout_s=max(1, int(gate_timeout_s)), 

164 ) 

165 

166 

167def _positive(value: Any, default: int) -> int: 

168 """A positive integer knob, or ``default`` (``bool`` is not a number here).""" 

169 if isinstance(value, int) and not isinstance(value, bool) and value >= 1: 

170 return value 

171 return default 

172 

173 

174def _strings(raw: Any) -> list[str]: 

175 """Non-blank string entries of a list (anything else contributes nothing).""" 

176 if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes, bytearray)): 

177 return [] 

178 return [entry.strip() for entry in raw if isinstance(entry, str) and entry.strip()] 

179 

180 

181def test_paths(policy_pack: Mapping[str, Any] | None) -> tuple[str, ...]: 

182 """Where the project's tests live: every ``policy_pack.test_groups.*.test_paths`` glob. 

183 

184 Only the **declared** ``test_paths`` count. A group's ``paths`` are selectors and 

185 routinely include the implementation surface (keel's own ``unit`` group selects 

186 ``src/**``); read as test paths they would classify every production file as a test, 

187 and the gate would skip every change it exists to check. 

188 """ 

189 pack = policy_pack if isinstance(policy_pack, Mapping) else {} 

190 groups = pack.get("test_groups") 

191 if not isinstance(groups, Mapping): 

192 return () 

193 globs: list[str] = [] 

194 for _name, group in sorted(groups.items(), key=lambda item: str(item[0])): 

195 if isinstance(group, Mapping): 

196 globs.extend(_strings(group.get("test_paths"))) 

197 return tuple(dict.fromkeys(globs)) 

198 

199 

200def is_production(path: str, *, tests: Sequence[str], paths: Sequence[str]) -> bool: 

201 """Is ``path`` production code this gate reverts? 

202 

203 Never a test path. Then ``paths`` (``fnmatch`` globs, the matcher ``tier3_globs`` and 

204 ``test_paths`` use) when the project set it, else a source-code suffix. 

205 """ 

206 if any(fnmatch.fnmatch(path, glob) for glob in tests): 

207 return False 

208 if paths: 

209 return any(fnmatch.fnmatch(path, glob) for glob in paths) 

210 return path.lower().endswith(SOURCE_SUFFIXES) 

211 

212 

213# --- the diff ------------------------------------------------------------------------ 

214 

215_HUNK_RE = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@") 

216 

217 

218@dataclass(frozen=True) 

219class Hunk: 

220 """One ``@@`` hunk: its header line and body, verbatim.""" 

221 

222 header: str 

223 lines: tuple[str, ...] 

224 added: int 

225 removed: int 

226 #: Context lines. :func:`keel.git.revert_diff` asks for none, so any here means the 

227 #: diff is not the shape the plan needs (:func:`context_problem`). 

228 context: int = 0 

229 

230 @property 

231 def pure_addition(self) -> bool: 

232 """Only adds lines: reverting it removes code rather than restoring old code.""" 

233 return self.removed == 0 

234 

235 

236@dataclass(frozen=True) 

237class FileDiff: 

238 """One file's part of a unified diff.""" 

239 

240 path: str 

241 #: The lines from ``diff --git`` up to the first hunk — what ``git apply`` needs to 

242 #: know which file, and whether it is new or deleted. 

243 header: tuple[str, ...] 

244 hunks: tuple[Hunk, ...] 

245 status: str # added | deleted | modified 

246 

247 @property 

248 def mode(self) -> tuple[str, str] | None: 

249 """``(old, new)`` when the diff changes the file's mode (``old mode`` / ``new mode``).""" 

250 old = next((line[9:] for line in self.header if line.startswith("old mode ")), None) 

251 new = next((line[9:] for line in self.header if line.startswith("new mode ")), None) 

252 return (old, new) if old and new else None 

253 

254 @property 

255 def binary(self) -> bool: 

256 """git printed no text for it (``Binary files … differ``).""" 

257 return any(line.startswith(("Binary files ", "GIT binary patch")) for line in self.header) 

258 

259 @property 

260 def content_header(self) -> tuple[str, ...]: 

261 """The header without its mode lines: a content hunk must not carry the mode along.""" 

262 return tuple( 

263 line for line in self.header if not line.startswith(("old mode ", "new mode ")) 

264 ) 

265 

266 

267#: git's C-style escapes in a quoted path (``quote_c_style``), beside ``\\ooo`` octal bytes. 

268_C_ESCAPES = {"a": 7, "b": 8, "t": 9, "n": 10, "v": 11, "f": 12, "r": 13, '"': 34, "\\": 92} 

269 

270 

271def _unquote(name: str) -> str: 

272 """A path as git wrote it -> the path itself. 

273 

274 git wraps a name in double quotes when it holds a control character, a quote or a 

275 backslash, and escapes those inside (``"a/q\\"x.py"``); a byte is ``\\ooo`` octal. An 

276 unquoted name is returned as it is. 

277 """ 

278 if len(name) < 2 or not (name.startswith('"') and name.endswith('"')): 

279 return name 

280 body, out, i = name[1:-1], bytearray(), 0 

281 while i < len(body): 

282 char = body[i] 

283 octal = body[i + 1 : i + 4] 

284 if char == "\\" and body[i + 1 : i + 2] in _C_ESCAPES: 

285 out.append(_C_ESCAPES[body[i + 1]]) 

286 i += 2 

287 elif char == "\\" and len(octal) == 3 and all(c in "01234567" for c in octal): 

288 out.append(int(octal, 8) & 0xFF) 

289 i += 4 

290 else: 

291 out += char.encode("utf-8", "surrogateescape") 

292 i += 1 

293 return out.decode("utf-8", "surrogateescape") 

294 

295 

296def _closing_quote(text: str) -> int: 

297 """The index of the quote that closes the quoted name opening ``text``; ``-1`` if none.""" 

298 i = 1 

299 while i < len(text): 

300 if text[i] == "\\": 

301 i += 2 

302 continue 

303 if text[i] == '"': 

304 return i 

305 i += 1 

306 return -1 

307 

308 

309def _strip(name: str, prefix: str) -> str: 

310 return name[len(prefix) :] if name.startswith(prefix) else name 

311 

312 

313def _path_from(line: str, prefix: str) -> str | None: 

314 """The path on a ``--- a/x`` / ``+++ b/x`` line; ``None`` for ``/dev/null``. 

315 

316 git ends the line with a tab when the name contains a space, and quotes a name that 

317 holds a control character, a quote or a backslash (:func:`_unquote`); the path is read 

318 back so it matches the project's globs. 

319 """ 

320 name = line[4:].rstrip("\t") 

321 if name == "/dev/null": 

322 return None 

323 return _strip(_unquote(name), prefix) 

324 

325 

326def _header_path(line: str) -> str: 

327 """The path on a ``diff --git a/x b/x`` line — the only name a file with no ``---`` / 

328 ``+++`` lines (a mode change, a binary file) carries. 

329 

330 With renames off both names are the same file, so an unquoted pair splits in the middle 

331 (a name may hold ``" b/"``), and a quoted one ends at its closing quote. A header that 

332 reads neither way falls back to the text after the last ``" b/"``. 

333 """ 

334 rest = line[len("diff --git ") :] 

335 if rest.startswith('"'): 

336 end = _closing_quote(rest) 

337 if end > 0: 

338 return _strip(_unquote(rest[: end + 1]), "a/") 

339 half = (len(rest) - 5) // 2 

340 name = rest[2 : 2 + half] 

341 if rest.startswith("a/") and rest[2 + half :] == f" b/{name}": 

342 return name 

343 return rest.rsplit(" b/", 1)[-1] 

344 

345 

346def parse_diff(text: str) -> tuple[FileDiff, ...]: 

347 """Split ``git diff --no-renames --src-prefix=a/ --dst-prefix=b/`` output into files. 

348 

349 Hunk bodies are consumed by the counts in their ``@@`` header, not by their first 

350 character, so a blank context line (``diff.suppressBlankEmpty``) cannot end a hunk 

351 early. A file with no hunk — binary, or a mode change — is kept with ``hunks=()``, so 

352 the plan can say it was not checked rather than drop it. 

353 """ 

354 files: list[FileDiff] = [] 

355 lines = text.split("\n") 

356 i = 0 

357 while i < len(lines): 

358 if not lines[i].startswith("diff --git "): 

359 i += 1 

360 continue 

361 header = [lines[i]] 

362 fallback = _header_path(lines[i]) 

363 old = new = None 

364 i += 1 

365 while i < len(lines) and not lines[i].startswith(("@@", "diff --git ")): 

366 line = lines[i] 

367 if line.startswith("--- "): 

368 old = _path_from(line, "a/") 

369 elif line.startswith("+++ "): 

370 new = _path_from(line, "b/") 

371 header.append(line) 

372 i += 1 

373 hunks: list[Hunk] = [] 

374 while i < len(lines) and lines[i].startswith("@@"): 

375 hunk, i = _read_hunk(lines, i) 

376 hunks.append(hunk) 

377 text_header = "\n".join(header) 

378 if "\nnew file mode " in text_header: 

379 status = "added" 

380 elif "\ndeleted file mode " in text_header: 

381 status = "deleted" 

382 else: 

383 status = "modified" 

384 files.append(FileDiff(new or old or fallback, tuple(header), tuple(hunks), status)) 

385 return tuple(files) 

386 

387 

388def _read_hunk(lines: list[str], i: int) -> tuple[Hunk, int]: 

389 """Read the hunk whose header is ``lines[i]``; return it and the index after it.""" 

390 header = lines[i] 

391 match = _HUNK_RE.match(header) 

392 old_left = int(match.group(2) or "1") if match else 0 

393 new_left = int(match.group(4) or "1") if match else 0 

394 body: list[str] = [] 

395 added = removed = context = 0 

396 i += 1 

397 while i < len(lines) and (old_left > 0 or new_left > 0): 

398 line = lines[i] 

399 tag = line[:1] 

400 if tag == "+": 

401 added += 1 

402 new_left -= 1 

403 elif tag == "-": 

404 removed += 1 

405 old_left -= 1 

406 elif tag == "\\": 

407 pass # "\ No newline at end of file" belongs to the line before it 

408 else: # context, including an empty line git wrote for a blank one 

409 old_left -= 1 

410 new_left -= 1 

411 context += 1 

412 body.append(line) 

413 i += 1 

414 # A trailing "\ No newline at end of file" marker after the last counted line. 

415 while i < len(lines) and lines[i].startswith("\\"): 

416 body.append(lines[i]) 

417 i += 1 

418 return Hunk(header, tuple(body), added, removed, context), i 

419 

420 

421def context_problem(files: Sequence[FileDiff]) -> str | None: 

422 """Why the diff cannot be split into independent changes, or ``None`` when it can. 

423 

424 Every hunk must carry **no context line**. :func:`keel.git.revert_diff` pins 

425 ``--unified=0`` and ``--inter-hunk-context=0``, but ``GIT_DIFF_OPTS`` or a setting keel 

426 does not know could still widen a hunk — and a hunk that carries context is either 

427 two edits merged into one change, where one caught edit passes the other, or a patch 

428 whose context ``--unidiff-zero`` would misplace. Checked here, on what git printed. 

429 """ 

430 widened = [f.path for f in files if any(h.context for h in f.hunks)] 

431 if not widened: 

432 return None 

433 return ( 

434 f"the diff carries context lines (in {', '.join(widened[:3])}), so its hunks may merge " 

435 "independent edits; check GIT_DIFF_OPTS and the diff settings in your git config" 

436 ) 

437 

438 

439@dataclass(frozen=True) 

440class Change: 

441 """One production change to revert: a label for the report and its patch.""" 

442 

443 label: str 

444 path: str 

445 #: A unified diff that ``git apply -R`` undoes on HEAD. 

446 patch: str 

447 #: Every hunk in it only adds lines — see :func:`judge` for why that matters. 

448 pure_addition: bool 

449 

450 

451@dataclass(frozen=True) 

452class Plan: 

453 """What the gate will revert, and what it had to leave out.""" 

454 

455 changes: tuple[Change, ...] 

456 #: Production files with no textual hunk (a binary file, or an empty file added or 

457 #: deleted): not checked. A mode-only change is a change of its own, and is checked. 

458 unrevertable: tuple[str, ...] 

459 #: Every path the diff touched, for the finding that says none was production. 

460 touched: tuple[str, ...] 

461 

462 

463def plan_changes( 

464 files: Sequence[FileDiff], *, tests: Sequence[str], paths: Sequence[str], unit: str 

465) -> Plan: 

466 """The production changes to revert, ordered by path. 

467 

468 Ordered here, not by git: ``diff.orderFile`` reorders git's output, and the order 

469 decides which changes ``max_changes`` reaches. Each change is reverted alone, so each 

470 must carry only itself: 

471 

472 * **A mode change is its own change** (``old mode``/``new mode``). Copied into every 

473 content hunk's patch, it was undone with each of them, and one test of the 

474 executable bit "caught" every hunk of the file. 

475 * **Added Python code is split at its top-level definitions** (:func:`_blocks`) — a 

476 hunk that only adds lines, and a new file's content, whatever ``unit`` says for a 

477 new file. Undone whole, three added functions where a test calls one read as 

478 "noticed" for all three; undone one at a time, each must be noticed on its own. A 

479 new file that is one block stays one change, which deletes it. Other languages, and 

480 added lines that do not parse on their own, are not split. 

481 * Otherwise one change per hunk (``unit: hunk``) or per file (``unit: file``); a 

482 deleted file is one change, which restores it. 

483 """ 

484 changes: list[Change] = [] 

485 unrevertable: list[str] = [] 

486 for f in sorted(files, key=lambda f: f.path): 

487 if not is_production(f.path, tests=tests, paths=paths): 

488 continue 

489 if f.mode is not None and f.status == "modified": 

490 old, new = f.mode 

491 mode_patch = "\n".join((f.header[0], f"old mode {old}", f"new mode {new}")) + "\n" 

492 changes.append(Change(f"{f.path} (mode {old} -> {new})", f.path, mode_patch, False)) 

493 if f.binary or not f.hunks: 

494 if f.binary or f.mode is None: 

495 unrevertable.append(f.path) 

496 continue 

497 header = f.content_header 

498 if f.status == "added": 

499 blocks = _blocks(f.hunks[0], f.path) if len(f.hunks) == 1 else [] 

500 if len(blocks) > 1: 

501 plain = (header[0], _old_side(header), *(h for h in header if h[:4] == "+++ ")) 

502 changes.extend(_block_change(f.path, plain, block) for block in blocks) 

503 else: 

504 changes.append( 

505 Change(f"{f.path} (new file)", f.path, _patch(header, f.hunks), True) 

506 ) 

507 continue 

508 if unit == UNIT_FILE or f.status == "deleted": 

509 suffix = " (deleted file)" if f.status == "deleted" else "" 

510 changes.append( 

511 Change( 

512 f"{f.path}{suffix}", 

513 f.path, 

514 _patch(header, f.hunks), 

515 all(h.pure_addition for h in f.hunks), 

516 ) 

517 ) 

518 continue 

519 for hunk in f.hunks: 

520 blocks = _blocks(hunk, f.path) if hunk.pure_addition else [] 

521 if len(blocks) > 1: 

522 changes.extend(_block_change(f.path, header, block) for block in blocks) 

523 continue 

524 label = f"{f.path} {hunk.header.split(' @@', 1)[0]} @@" 

525 changes.append(Change(label, f.path, _patch(header, (hunk,)), hunk.pure_addition)) 

526 return Plan(tuple(changes), tuple(unrevertable), tuple(f.path for f in files)) 

527 

528 

529#: Files whose added code :func:`_blocks` can split: it parses them. 

530_PYTHON_SUFFIXES = (".py", ".pyi") 

531_DEFINITIONS = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) 

532 

533 

534def _old_side(header: Sequence[str]) -> str: 

535 """A new file's ``--- a/x`` line, from its ``+++ b/x`` line as git wrote it (quoting and 

536 all), so a patch that removes part of the file names the file that stays.""" 

537 new = next(h for h in header if h[:4] == "+++ ") 

538 name = new[4:] 

539 return "--- " + ('"a/' + name[3:] if name.startswith('"b/') else "a/" + name[2:]) 

540 

541 

542def _blocks(hunk: Hunk, path: str) -> list[tuple[int, tuple[str, ...]]]: 

543 """A pure-addition hunk's lines split at its top-level definitions, as ``(first line, lines)``. 

544 

545 Only Python is split, and only when the added lines parse on their own 

546 (:func:`_block_starts`): a boundary read from the text alone was wrong both ways — two 

547 functions with no blank line between them stayed one change, so a test of one vouched 

548 for the other, and a blank line inside a string literal split it, so each half's revert 

549 was a syntax error. ``[]`` (keep the hunk whole) otherwise, and for a header that cannot 

550 be read. Lines between blocks ride with the block before them (leading ones with the 

551 first); a ``\\ No newline`` marker stays with its line. 

552 """ 

553 match = _HUNK_RE.match(hunk.header) 

554 if not match or not path.endswith(_PYTHON_SUFFIXES): 

555 return [] 

556 starts = _block_starts([line[1:] for line in hunk.lines if line.startswith("+")]) 

557 if not starts: 

558 return [] 

559 first = int(match.group(3)) 

560 blocks: list[tuple[int, list[str]]] = [] 

561 offset = 0 

562 for line in hunk.lines: 

563 if line.startswith("+"): 

564 if not blocks or offset in starts: 

565 blocks.append((first + offset, [])) 

566 offset += 1 

567 blocks[-1][1].append(line) 

568 return [(start, tuple(lines)) for start, lines in blocks] 

569 

570 

571def _block_starts(added: Sequence[str]) -> set[int]: 

572 """The 0-based added lines where a new block starts; empty when there is one block. 

573 

574 The lines are dedented by their common leading whitespace and parsed. Each ``def`` and 

575 ``class`` (from its first decorator) is a block, and so is each run of other statements 

576 between them. Empty when the lines do not parse on their own — part of an expression, 

577 a string whose continuation sits left of the rest — or do not share one indent. 

578 """ 

579 texts = [text for text in added if text.strip()] 

580 if not texts: 

581 return set() 

582 width = min(len(text) - len(text.lstrip()) for text in texts) 

583 indent = texts[0][:width] 

584 if any(not text.startswith(indent) for text in texts): 

585 return set() 

586 source = "\n".join(text[width:] if text.strip() else "" for text in added) + "\n" 

587 try: 

588 tree = ast.parse(source) 

589 except (SyntaxError, ValueError, RecursionError): 

590 return set() 

591 starts: set[int] = set() 

592 previous: bool | None = None 

593 for node in tree.body: 

594 definition = isinstance(node, _DEFINITIONS) 

595 if previous is not None and (definition or previous): 

596 decorators = getattr(node, "decorator_list", []) 

597 starts.add(min([node.lineno, *(d.lineno for d in decorators)]) - 1) 

598 previous = definition 

599 return starts 

600 

601 

602def _block_change(path: str, header: Sequence[str], block: tuple[int, tuple[str, ...]]) -> Change: 

603 """One block of added lines as its own change: a patch that removes just those lines.""" 

604 first, lines = block 

605 count = sum(1 for line in lines if line.startswith("+")) 

606 hunk_header = f"@@ -{first - 1},0 +{first},{count} @@" 

607 patch = "\n".join((*header, hunk_header, *lines)) + "\n" 

608 return Change(f"{path} {hunk_header}", path, patch, True) 

609 

610 

611def _patch(header: Sequence[str], hunks: Sequence[Hunk]) -> str: 

612 lines = list(header) 

613 for hunk in hunks: 

614 lines.append(hunk.header) 

615 lines.extend(hunk.lines) 

616 return "\n".join(lines) + "\n" 

617 

618 

619# --- what a change looks like ------------------------------------------------------ 

620# 

621# Nothing here skips a run. Every production change in scope is reverted and tested: an 

622# earlier version skipped changes it judged "inert" (comments, docstrings, formatting), 

623# and each proof of inertness had a hole — a Go ``//go:embed`` directive, a Python 

624# ``# coding:`` cookie, a comment that moves ``__LINE__`` or ``f_lineno`` — because "no 

625# test can observe this change" is not provable in general (#1289 review rounds). 

626 

627#: Leading text that makes a line **look** like a comment in some language. A wording 

628#: heuristic only (:func:`looks_comment_only`): it never decides whether a change is run. 

629_COMMENT_LOOKS: tuple[str, ...] = ("#", "//", "/*", "*/", "* ", "--", ";", "%") 

630 

631 

632def _changed_lines(patch: str) -> list[str]: 

633 """The ``+``/``-`` lines of a change's hunks, without their marker.""" 

634 _header, _sep, body = patch.partition("\n@@") 

635 return [line[1:] for line in body.split("\n") if line[:1] in ("+", "-")] 

636 

637 

638def looks_comment_only(change: Change) -> bool: 

639 """Does every changed line look blank or like a comment? **Wording only.** 

640 

641 Used to explain a blocking "no test notices this change" finding, never to skip a 

642 run: a comment can still change behaviour. 

643 """ 

644 lines = _changed_lines(change.patch) 

645 return bool(lines) and all( 

646 not line.strip() or line.strip() == "*" or line.strip().startswith(_COMMENT_LOOKS) 

647 for line in lines 

648 ) 

649 

650 

651_NAMES = r"[A-Za-z_]\w*(?:\s+as\s+[A-Za-z_]\w*)?(?:\s*,\s*[A-Za-z_]\w*(?:\s+as\s+[A-Za-z_]\w*)?)*" 

652_DOTTED = r"[A-Za-z_][\w.]*(?:\s+as\s+[A-Za-z_]\w*)?" 

653#: One complete import statement on one line, and nothing else on it — no trailing 

654#: comment, no ``;``, no open parenthesis left for a continuation line. 

655_IMPORT_LINE = re.compile( 

656 rf"^\s*(?:import\s+{_DOTTED}(?:\s*,\s*{_DOTTED})*" 

657 rf"|from\s+(?:\.+[\w.]*|[A-Za-z_][\w.]*)\s+import\s+" 

658 rf"(?:\*|{_NAMES}|\(\s*{_NAMES}\s*,?\s*\)))\s*$" 

659) 

660 

661 

662def imports_only(change: Change) -> bool: 

663 """Is ``change`` a Python change whose every changed line is a whole import statement? 

664 

665 Read from the change's **own** lines, never from a comparison of the two files: an 

666 AST cannot see an encoding cookie, and ``# coding: latin-1`` beside an import once 

667 passed as "imports only" while it changed what a string literal decodes to. Blank 

668 lines are ignored; a comment, a cookie, a continuation line of a multi-line import or 

669 anything else makes the answer ``False`` — that change's errors then block, which is 

670 the safe side. It classifies a result *after* its run; it never skips one. 

671 """ 

672 if posixpath.splitext(change.path)[1].lower() not in (".py", ".pyi"): 

673 return False 

674 lines = [line for line in _changed_lines(change.patch) if line.strip()] 

675 return bool(lines) and all(_IMPORT_LINE.match(line) for line in lines) 

676 

677 

678#: The exceptions that say "a name the tests reach is gone" — all a revert of pure 

679#: added code (or of an imported name) can produce. ``AttributeError`` counts only for a 

680#: *module* or *class* attribute: ``'NoneType' object has no attribute`` is behaviour. 

681_MISSING_NAME = re.compile( 

682 r"^(?:NameError|UnboundLocalError|ImportError|ModuleNotFoundError)\b" 

683 r"|^AttributeError: (?:(?:partially initialized )?module '[^']*'|type object '[^']*') " 

684 r"has no attribute\b" 

685) 

686#: The start of a Python traceback; its exception line is the first unindented line after it. 

687_TRACEBACK = "Traceback (most recent call last):" 

688#: A pytest short-summary line's reason (`` - <Exception>: message``), any exception name. 

689_PYTEST_REASON = re.compile(r"^(?:FAILED|ERROR) .*? - ([A-Za-z_][\w.]*(?::.*)?)$", re.MULTILINE) 

690 

691 

692def _raised(output: str) -> list[str]: 

693 """Every exception the output reports, as ``Name: message`` with the module dropped. 

694 

695 Read from each traceback's final line — whatever the exception is called, so a 

696 custom ``Boom`` or a ``StopIteration`` is seen too — and from pytest's short summary. 

697 """ 

698 found: list[str] = [] 

699 lines = output.split("\n") 

700 for i, line in enumerate(lines): 

701 if line.strip() != _TRACEBACK: 

702 continue 

703 rest = (text for text in lines[i + 1 :] if text.strip() and not text[:1].isspace()) 

704 found.append(next(rest, "")) 

705 found.extend(match.group(1) for match in _PYTEST_REASON.finditer(output)) 

706 return [ 

707 name.rsplit(".", 1)[-1] + sep + message 

708 for name, sep, message in (text.partition(":") for text in found) 

709 ] 

710 

711 

712def missing_names_only(output: str) -> bool: 

713 """Does every exception the run reports say only that a name is missing? 

714 

715 The evidence a lenient reading needs: the error-only result of undoing an addition is 

716 a ``nit`` only when the errors are the ones removing code can cause. A ``KeyError``, a 

717 ``TypeError``, an attribute missing from an *instance*, any exception keel cannot 

718 name, and output with no traceback at all are behaviour, and block. 

719 

720 **Every** failure the run counts must be described: a summary of ``2 failed`` whose 

721 short summary names one ``NameError`` says nothing about the other, which could be an 

722 assertion or a crash (#1289 review round 9). So pytest's undescribed failures 

723 (:attr:`Tally.unclassified`) refuse it, and so does a count of failures and errors 

724 larger than the exceptions keel read. 

725 """ 

726 found = _raised(output) 

727 tally = read_output(output) 

728 return ( 

729 bool(found) 

730 and not tally.unclassified 

731 and len(found) >= tally.errors + tally.assertions 

732 and all(_MISSING_NAME.match(text) for text in found) 

733 ) 

734 

735 

736# --- reading a test run -------------------------------------------------------------- 

737 

738_UNITTEST_RAN = re.compile(r"^Ran (\d+) tests? in ", re.MULTILINE) 

739_UNITTEST_FAILED = re.compile(r"^FAILED \(([^)]*)\)\s*$", re.MULTILINE) 

740_PYTEST_SUMMARY = re.compile( 

741 r"^=*\s*((?:\d+ [a-z]+(?:, )?)+) in [\d.]+s\b.*$|^=*\s*no tests ran in [\d.]+s\b.*$", 

742 re.MULTILINE, 

743) 

744_PYTEST_COUNT = re.compile(r"(\d+) ([a-z]+)") 

745#: A pytest short-summary line. The node id never starts with ``(``, which is what keeps 

746#: unittest's own ``FAILED (failures=1)`` from being read as a failed pytest test. 

747#: 

748#: The node id may contain spaces — a parametrized id (``test_v[hello world]``) or a file 

749#: name — so it is read as: a bracketed id up to its ``]``, else one word, else the 

750#: shortest text before `` - ``; the reason is what follows `` - `` and may be absent. 

751_PYTEST_LINE = re.compile(r"^(FAILED|ERROR) (?!\()(\S*?\[.*?\]|\S+|.+?)(?: - (.*))?$", re.MULTILINE) 

752#: A unittest failure section's header: ``FAIL: test_x (pkg.T.test_x)`` is an assertion, 

753#: ``ERROR: …`` anything else. What the differential reading compares. 

754_UNITTEST_HEADER = re.compile(r"^(FAIL|ERROR): (.+?)\s*$", re.MULTILINE) 

755#: A pytest short-summary reason that is an assertion: a rewritten ``assert``, an 

756#: ``AssertionError`` (unittest-style assertions under pytest), or ``pytest.fail``. 

757#: ``Failed: Timeout`` is excluded: pytest-timeout reports a timed-out test that way, and 

758#: a hang is not an assertion. 

759_ASSERTION_REASON = re.compile(r"^(?:assert\b|AssertionError\b|Failed:(?! Timeout\b))") 

760 

761 

762@dataclass(frozen=True) 

763class Tally: 

764 """What a test run's output says happened.""" 

765 

766 #: The output carries a unittest or pytest summary at all. 

767 recognized: bool = False 

768 #: Tests that ran. 

769 ran: int = 0 

770 #: Tests that failed as an assertion. 

771 assertions: int = 0 

772 #: Tests that failed any other way: an exception, an import or collection error, a 

773 #: strict unexpected success. 

774 errors: int = 0 

775 #: pytest counted a failure its short summary does not describe (``-rN``, truncation). 

776 unclassified: int = 0 

777 #: The ids of the tests that failed as an assertion, where the runner names them. 

778 assertion_ids: frozenset[str] = frozenset() 

779 #: The ids of every failing test — assertion or not — where the runner names them. 

780 failure_ids: frozenset[str] = frozenset() 

781 

782 @property 

783 def failed(self) -> bool: 

784 """Does the output report any failing or erroring test at all?""" 

785 return bool(self.assertions or self.errors or self.unclassified or self.failure_ids) 

786 

787 

788def read_output(output: str) -> Tally: 

789 """Read unittest's and pytest's summaries out of a test command's combined output. 

790 

791 **unittest** separates the two cleanly: ``failures`` are the test's 

792 ``failureException`` (an ``AssertionError``), ``errors`` are anything else, a test 

793 module that failed to import included. 

794 

795 **pytest** does not: its ``failed`` count includes a ``NameError`` raised in the test 

796 body. The short test summary (``-rfE``, pytest's default since 6.0) names each 

797 failure's exception, and only an assertion reason counts; a failure the summary does 

798 not describe is *unclassified*, never an assertion. 

799 

800 Every summary in the output is summed, so a command running two suites is read whole. 

801 """ 

802 ran = assertions = errors = unclassified = 0 

803 recognized = False 

804 for match in _UNITTEST_RAN.finditer(output): 

805 recognized = True 

806 ran += int(match.group(1)) 

807 for match in _UNITTEST_FAILED.finditer(output): 

808 for part in match.group(1).split(","): 

809 key, _, value = part.strip().partition("=") 

810 count = int(value) if value.isdigit() else 0 

811 if key == "failures": 

812 assertions += count 

813 elif key in ("errors", "unexpected successes"): 

814 errors += count 

815 pytest_failed = 0 

816 for match in _PYTEST_SUMMARY.finditer(output): 

817 recognized = True 

818 for count, word in _PYTEST_COUNT.findall(match.group(1) or ""): 

819 if word in ("passed", "failed", "xfailed", "xpassed"): 

820 ran += int(count) 

821 if word == "failed": 

822 pytest_failed += int(count) 

823 elif word in ("error", "errors"): 

824 errors += int(count) 

825 described = 0 

826 asserted: set[str] = set() 

827 failing: set[str] = set() 

828 for kind, test_id in _UNITTEST_HEADER.findall(output): 

829 failing.add(test_id) 

830 if kind == "FAIL": 

831 asserted.add(test_id) 

832 for match in _PYTEST_LINE.finditer(output): 

833 failing.add(match.group(2)) 

834 if match.group(1) != "FAILED": 

835 continue # ERROR lines are already in the summary's error count 

836 described += 1 

837 if _ASSERTION_REASON.match(match.group(3) or ""): 

838 assertions += 1 

839 asserted.add(match.group(2)) 

840 else: 

841 errors += 1 

842 unclassified = max(0, pytest_failed - described) 

843 return Tally( 

844 recognized, 

845 ran, 

846 assertions, 

847 errors, 

848 unclassified, 

849 frozenset(asserted), 

850 frozenset(failing), 

851 ) 

852 

853 

854def classify( 

855 *, exit_ok: bool, timed_out: bool, output: str, baseline_failed: frozenset[str] = frozenset() 

856) -> tuple[str, str]: 

857 """One reverted run -> ``(result, why)``. Only :data:`CAUGHT` is a pass. 

858 

859 The exit code decides pass/fail; the output decides *how* it failed. A test that 

860 errored is not a test that asserted — reverting half a fix can raise ``NameError`` 

861 in every test that imports it, and that proves the import, not the behaviour. 

862 

863 **Differential.** ``baseline_failed`` names the tests that failed without the revert. 

864 Where the runner names the assertion failures, the change is caught only by one 

865 *not* among them, so a failure that was already there cannot certify it. 

866 (:func:`baseline_problem` already refuses a baseline with any failure; this holds the 

867 rule on its own too.) 

868 """ 

869 if timed_out: 

870 return TIMED_OUT, "the test command timed out with the change reverted" 

871 if exit_ok: 

872 return UNNOTICED, "the test command passed with the change reverted" 

873 tally = read_output(output) 

874 if tally.assertion_ids and tally.assertion_ids <= baseline_failed: 

875 return UNNOTICED, ( 

876 "the only tests that failed as an assertion also fail without the revert" 

877 ) 

878 if tally.assertions: 

879 return CAUGHT, f"{tally.assertions} test(s) failed as an assertion" 

880 if not tally.recognized: 

881 return UNREADABLE, ( 

882 "the test command failed, and its output carries no unittest or pytest summary " 

883 "to say whether a test failed" 

884 ) 

885 if tally.errors: 

886 return ERRORED, f"{tally.errors} test(s) errored and none failed as an assertion" 

887 if tally.unclassified: 

888 return UNREADABLE, ( 

889 f"pytest reported {tally.unclassified} failure(s) its short test summary does not " 

890 "describe, so an assertion cannot be told from an error (keep pytest's -rf)" 

891 ) 

892 return UNREADABLE, "the test command failed, but no test did" 

893 

894 

895def baseline_problem(*, exit_ok: bool, timed_out: bool, output: str) -> str | None: 

896 """Why the unreverted run cannot anchor the check, or ``None`` when it can. 

897 

898 A failure after a revert means something only if the same command passes — and is 

899 readable — on HEAD itself, in the same kind of clean checkout. 

900 """ 

901 if timed_out: 

902 return "the test command timed out on a clean checkout of HEAD" 

903 if not exit_ok: 

904 return ( 

905 "the test command fails on a clean checkout of HEAD, so a failure with a change " 

906 "reverted would prove nothing" 

907 ) 

908 tally = read_output(output) 

909 if not tally.recognized: 

910 return ( 

911 "the test command's output carries no unittest or pytest summary; revert-check " 

912 "reads those two to tell an assertion from an error" 

913 ) 

914 if tally.ran == 0: 

915 return "the test command ran no tests on a clean checkout of HEAD" 

916 if tally.failed: 

917 # `suite1; suite2` exits with suite2's status: a suite already failing an 

918 # assertion would otherwise lend that assertion to every revert. 

919 return ( 

920 "the test command exits 0 on a clean checkout of HEAD but its output reports " 

921 "failing tests, so a failure with a change reverted would prove nothing" 

922 ) 

923 return None 

924 

925 

926# --- the bounded loop ---------------------------------------------------------------- 

927 

928 

929@dataclass(frozen=True) 

930class RunResult: 

931 """One execution of the test command, as the I/O layer observed it.""" 

932 

933 exit_ok: bool = False 

934 timed_out: bool = False 

935 output: str = "" 

936 

937 

938@dataclass(frozen=True) 

939class Reverted: 

940 """One change undone in the scratch tree, as the I/O layer observed it.""" 

941 

942 #: The tree was reset and the reverse patch applied. 

943 applied: bool 

944 

945 

946@dataclass(frozen=True) 

947class ChangeResult: 

948 change: Change 

949 result: str 

950 why: str 

951 #: The change only adds code or import lines (:attr:`Change.pure_addition`, or 

952 #: :func:`imports_only`) **and** every error its run reports is a missing name 

953 #: (:func:`missing_names_only`) — the one error-only result :func:`judge` reads as a 

954 #: ``nit``. 

955 adds_only: bool = False 

956 

957 

958@dataclass(frozen=True) 

959class Report: 

960 """What :func:`execute` observed.""" 

961 

962 #: Set when the baseline run cannot anchor the check; nothing was reverted then. 

963 baseline: str | None 

964 results: tuple[ChangeResult, ...] = () 

965 #: Changes left unchecked by ``max_changes`` or ``budget_s``, with the reason. 

966 not_checked: tuple[tuple[Change, str], ...] = () 

967 

968 

969#: ``revert(change)``: reset the scratch tree to ``HEAD`` and undo ``change`` in it. 

970Reverter = Callable[[Change], Reverted] 

971#: ``test(timeout_s)``: run the test command in the scratch tree as it stands. 

972Tester = Callable[[int], RunResult] 

973 

974 

975def execute( 

976 changes: Sequence[Change], 

977 settings: Settings, 

978 *, 

979 revert: Reverter, 

980 test: Tester, 

981 clock: Callable[[], float], 

982) -> Report: 

983 """Run the baseline, then revert each change alone, within the cost bounds. 

984 

985 ``max_changes`` caps how many changes are reverted; ``budget_s`` caps the wall clock 

986 of the baseline and every revert together, and each run's limit is the smaller of 

987 ``knobs.gate_timeout_s`` and what is left of it. A change either bound leaves out is 

988 reported **not checked** — never as caught. Every change that is reverted is tested: 

989 nothing is skipped for looking harmless. 

990 """ 

991 start = clock() 

992 

993 def left() -> float: 

994 return settings.budget_s - (clock() - start) 

995 

996 def limit(remaining: float) -> int: 

997 return max(1, min(settings.run_timeout_s, int(remaining))) 

998 

999 base = test(limit(left())) 

1000 problem = baseline_problem(exit_ok=base.exit_ok, timed_out=base.timed_out, output=base.output) 

1001 if problem is not None: 

1002 return Report(problem) 

1003 baseline_failed = read_output(base.output).failure_ids 

1004 results: list[ChangeResult] = [] 

1005 skipped: list[tuple[Change, str]] = [] 

1006 for index, change in enumerate(changes): 

1007 if index >= settings.max_changes: 

1008 skipped.append( 

1009 (change, f"over knobs.revert_check.max_changes ({settings.max_changes})") 

1010 ) 

1011 continue 

1012 remaining = left() 

1013 if remaining < 1: 

1014 skipped.append( 

1015 (change, f"the knobs.revert_check.budget_s budget ({settings.budget_s}s) ran out") 

1016 ) 

1017 continue 

1018 undone = revert(change) 

1019 if not undone.applied: 

1020 results.append( 

1021 ChangeResult(change, NOT_APPLIED, "git apply -R could not undo it on HEAD") 

1022 ) 

1023 continue 

1024 # Re-read the clock: resetting, cleaning, applying and reading took time too, and a 

1025 # run must never start with a limit the budget no longer has. 

1026 remaining = left() 

1027 if remaining < 1: 

1028 skipped.append( 

1029 (change, f"the knobs.revert_check.budget_s budget ({settings.budget_s}s) ran out") 

1030 ) 

1031 continue 

1032 outcome = test(limit(remaining)) 

1033 result, why = classify( 

1034 exit_ok=outcome.exit_ok, 

1035 timed_out=outcome.timed_out, 

1036 output=outcome.output, 

1037 baseline_failed=baseline_failed, 

1038 ) 

1039 adds_only = (change.pure_addition or imports_only(change)) and missing_names_only( 

1040 outcome.output 

1041 ) 

1042 results.append(ChangeResult(change, result, why, adds_only)) 

1043 return Report(None, tuple(results), tuple(skipped)) 

1044 

1045 

1046# --- the verdict --------------------------------------------------------------------- 

1047 

1048 

1049@dataclass(frozen=True) 

1050class Verdict: 

1051 """The gate's outcome, in :class:`keel.gates.GateOutcome` terms.""" 

1052 

1053 ok: bool 

1054 findings: tuple[Finding, ...] 

1055 #: The gate cannot judge — only the project's config or environment can fix it. 

1056 unconfigured: bool = False 

1057 #: Nothing to check: the diff has no production change. Reported ``SKIPPED``. 

1058 skipped: bool = False 

1059 

1060 

1061def cannot_judge(why: str) -> Verdict: 

1062 """A verdict for a check that could not run: failed, never a pass (#1364).""" 

1063 return Verdict(False, (Finding(_BLOCK, f"cannot judge: {why}", GATE_ID),), unconfigured=True) 

1064 

1065 

1066#: Why the gate cannot judge a plan with no guard or test gate beside it (#1289 review). 

1067PLANNED_ALONE = ( 

1068 "revert-check is not a test gate: it reverts each change against a suite the guard and " 

1069 "test gates proved green, and none is planned — list build (with knobs.build_gate_cmd) " 

1070 "beside it in gates:" 

1071) 

1072 

1073 

1074def precheck(settings: Settings, *, tests: Sequence[str], gates_green: bool) -> Verdict | None: 

1075 """The verdict when the check must not start, else ``None``. 

1076 

1077 Order matters: a missing command or test layout is a config problem no run can fix, 

1078 so it is reported before the red-gates case an implementer can. 

1079 """ 

1080 if settings.cmd is None: 

1081 return cannot_judge( 

1082 "no test command: set knobs.revert_check.cmd or knobs.build_gate_cmd " 

1083 "in .keel/project.yaml" 

1084 ) 

1085 if not tests: 

1086 return cannot_judge( 

1087 "revert-check cannot tell tests from production: declare where the tests live " 

1088 "in policy_pack.test_groups.<group>.test_paths" 

1089 ) 

1090 if not gates_green: 

1091 return Verdict( 

1092 False, 

1093 ( 

1094 Finding( 

1095 _BLOCK, 

1096 "not run: the guard and test gates are red, and a revert check against a " 

1097 "red suite cannot tell which failures the revert caused", 

1098 GATE_ID, 

1099 ), 

1100 ), 

1101 ) 

1102 return None 

1103 

1104 

1105def judge(plan: Plan, report: Report | None) -> Verdict: 

1106 """The gate's verdict: every production change must make a test fail as an assertion. 

1107 

1108 * **unnoticed**, **timed out**, **unreadable**, **not applied** and **not checked** 

1109 each block, naming the change: a change the gate did not see fail is not a pass. 

1110 * **errored** — and **unreadable**, where the suite crashed before its summary — block, 

1111 except for a change that only *adds* lines or only changes import lines **and** 

1112 whose run reported nothing but missing names (``NameError``, ``ImportError``, a 

1113 module or class attribute). Reverting such an addition removes a name, and a test 

1114 that calls it can then only error: no assertion can fail against code that is not 

1115 there. The error does prove a test depends on it, so it is a ``nit``. Any other 

1116 error — a ``KeyError``, a ``TypeError``, an instance attribute — is behaviour a test 

1117 should assert on, and blocks, as does any error after reverting a modification. 

1118 * A production file with no textual hunk (binary, or an empty file added or deleted) 

1119 blocks as **not checked**: nothing could be reverted, and the gate never certifies 

1120 what it did not check. 

1121 * A diff with no production change is ``SKIPPED`` — judged, with nothing to check. 

1122 

1123 ``report`` is ``None`` only when there was nothing to execute. 

1124 """ 

1125 findings: list[Finding] = [] 

1126 for path in plan.unrevertable: 

1127 findings.append( 

1128 Finding( 

1129 _BLOCK, 

1130 f"{path}: not checked — no textual hunk to revert (a binary file, or an empty file " 

1131 "added or deleted)", 

1132 GATE_ID, 

1133 ) 

1134 ) 

1135 if report is None: 

1136 if findings: 

1137 return Verdict(False, tuple(findings)) 

1138 touched = len(plan.touched) 

1139 return Verdict( 

1140 True, 

1141 ( 

1142 Finding( 

1143 "nit", 

1144 f"nothing to revert: none of the {touched} changed file(s) is production " 

1145 "code (knobs.revert_check.paths, or a source suffix outside the test paths)", 

1146 GATE_ID, 

1147 ), 

1148 ), 

1149 skipped=True, 

1150 ) 

1151 if report.baseline is not None: 

1152 return cannot_judge(report.baseline) 

1153 caught = 0 

1154 for item in report.results: 

1155 label = item.change.label 

1156 if item.result == CAUGHT: 

1157 caught += 1 

1158 elif item.result in (ERRORED, UNREADABLE) and item.adds_only: 

1159 findings.append( 

1160 Finding( 

1161 "nit", 

1162 f"{label}: noticed only as a missing name — {item.why}; the change only " 

1163 "adds code or import lines, so no assertion can fail without it", 

1164 GATE_ID, 

1165 ) 

1166 ) 

1167 elif item.result == UNNOTICED: 

1168 hint = ( 

1169 "; it looks comment-only, and keel tests every change, because a comment can " 

1170 "still change behaviour (encoding cookies, compiler directives, line " 

1171 "numbers) — add a test that notices it, or scope such files out with " 

1172 "knobs.revert_check.paths if you do not want them checked" 

1173 if looks_comment_only(item.change) 

1174 else "" 

1175 ) 

1176 findings.append( 

1177 Finding(_BLOCK, f"{label}: no test notices this change — {item.why}{hint}", GATE_ID) 

1178 ) 

1179 else: 

1180 findings.append( 

1181 Finding(_BLOCK, f"{label}: no test failed as an assertion — {item.why}", GATE_ID) 

1182 ) 

1183 for change, why in report.not_checked: 

1184 findings.append(Finding(_BLOCK, f"{change.label}: not checked — {why}", GATE_ID)) 

1185 total = len(report.results) + len(report.not_checked) 

1186 summary = ( 

1187 f"{caught} of {total} production change(s) made a test fail as an assertion when " 

1188 "reverted alone" 

1189 ) 

1190 findings.append(Finding("nit", summary, GATE_ID)) 

1191 blocked = any(f.severity == _BLOCK for f in findings) 

1192 return Verdict(not blocked, tuple(findings))