Coverage for src/keel/revertcheck.py: 100%
531 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
1"""The opt-in ``revert-check`` gate (#1289): does any test notice when a change is undone?
3Coverage proves that a line *ran*, not that an assertion *depends* on it. With
4``fail_under = 100`` enforced, "maintained 100 % coverage" is true of every merged pull
5request before anyone writes it, and an audit of 14 closed fixes found three whose tests
6still passed with the fix removed at the time their issue was closed (#877, and one arm each
7of #871 and #879); later audits added tests that catch them, and when #1289 recorded the audit
8on 2026-09-22 all 14 failed without their fix. This gate asks the question coverage cannot: for each
9production change on the branch, revert **that change alone** in a scratch worktree, run
10the project's tests, and require a test to **fail as an assertion**.
12It is mutation testing scoped to one mutation per change — the change itself.
14This module is the pure half, and deterministic:
16* :func:`resolve` — ``knobs.revert_check`` (+ ``build_gate_cmd``, ``gate_timeout_s``) ->
17 :class:`Settings`;
18* :func:`test_paths` — where the tests live, from ``policy_pack.test_groups.*.test_paths``;
19* :func:`parse_diff` / :func:`plan_changes` — the branch's unified diff -> the production
20 changes to revert, each with the patch that undoes it;
21* :func:`read_output` / :func:`classify` — a test run's output -> *caught* (a test failed as
22 an assertion), *errored*, *unnoticed*, *timed out* or *unreadable*;
23* :func:`execute` — the bounded loop (``max_changes``, ``budget_s``), driven through an
24 injected ``run`` callable and ``clock``, so the loop is tested offline;
25* :func:`judge` — the gate's verdict and findings.
27The git and subprocess work — the scratch worktree, ``git apply -R``, the test command —
28lives in :mod:`keel.cli` through :mod:`keel.git` and :mod:`keel.runner`; nothing here
29touches the file system.
31**What it does not do.** The unit is a git hunk (or a file), not a *behaviour*: #871's
32guarded and unguarded arms shared one hunk, so a per-hunk check passes while half that fix
33is unpinned. And a passing check is necessary, not sufficient: #873's test fails without
34its fix and still could not see the regression the fix shipped (#1268), because its fixture
35made the fix and the bug agree. Both remain a reviewer's questions.
36"""
38from __future__ import annotations
40import ast
41import fnmatch
42import posixpath
43import re
44from collections.abc import Callable, Mapping, Sequence
45from dataclasses import dataclass
46from typing import Any
48from .findings import Finding
50#: The gate id, as ``gates:`` lists it.
51GATE_ID = "revert-check"
53#: ``knobs.revert_check.unit``: revert each hunk alone (the default), or each file.
54UNIT_HUNK = "hunk"
55UNIT_FILE = "file"
56UNITS = (UNIT_HUNK, UNIT_FILE)
58#: At most this many changes are reverted per run unless ``max_changes`` says otherwise.
59DEFAULT_MAX_CHANGES = 10
60#: Wall-clock seconds the whole check — the baseline run and every revert — may spend.
61DEFAULT_BUDGET_S = 1800
63#: What counts as production when ``knobs.revert_check.paths`` is not set: a changed file
64#: with a source-code suffix that is not under the project's test paths. Docs, config,
65#: lockfiles and data are left out — reverting a README cannot make a test fail, and
66#: reporting it as unnoticed would be noise, not evidence.
67SOURCE_SUFFIXES: tuple[str, ...] = (
68 ".py",
69 ".pyi",
70 ".js",
71 ".jsx",
72 ".mjs",
73 ".cjs",
74 ".ts",
75 ".tsx",
76 ".go",
77 ".rs",
78 ".java",
79 ".kt",
80 ".kts",
81 ".scala",
82 ".rb",
83 ".php",
84 ".cs",
85 ".fs",
86 ".c",
87 ".h",
88 ".cc",
89 ".cpp",
90 ".cxx",
91 ".hpp",
92 ".m",
93 ".mm",
94 ".swift",
95 ".dart",
96 ".ex",
97 ".exs",
98 ".erl",
99 ".clj",
100 ".lua",
101 ".pl",
102 ".pm",
103 ".r",
104 ".sh",
105 ".bash",
106)
108# Per-change results.
109CAUGHT = "caught" # a test failed as an assertion
110ERRORED = "errored" # tests failed, none as an assertion (an exception, an import error)
111UNNOTICED = "unnoticed" # the test command passed with the change reverted
112TIMED_OUT = "timed-out"
113UNREADABLE = "unreadable" # the command failed and its output says no test failed
114NOT_APPLIED = "not-applied" # the reverse patch did not apply on HEAD
116_BLOCK = "major"
119@dataclass(frozen=True)
120class Settings:
121 """The resolved ``knobs.revert_check`` for one run."""
123 #: The test command each revert runs; ``None`` when neither knob sets one.
124 cmd: str | None
125 #: The knob :attr:`cmd` came from (``knobs.build_gate_cmd`` when neither sets one).
126 cmd_source: str
127 #: Globs that select production files; empty means :data:`SOURCE_SUFFIXES`.
128 paths: tuple[str, ...]
129 unit: str
130 max_changes: int
131 budget_s: int
132 #: The wall-clock limit of one test run (``knobs.gate_timeout_s``).
133 run_timeout_s: int
136def resolve(
137 knob: Mapping[str, Any] | None, *, build_cmd: str | None, gate_timeout_s: int
138) -> Settings:
139 """``knobs.revert_check`` -> :class:`Settings`, defaults filled in.
141 The test command is ``knobs.revert_check.cmd`` when set, else ``knobs.build_gate_cmd``:
142 a project whose suite is slow can point the check at the tests that matter without
143 changing what its ``build`` gate runs. The schema owns the value shapes; anything it
144 would refuse reads as the default here, because this resolver runs on every gate run.
145 """
146 raw = knob if isinstance(knob, Mapping) else {}
147 own = raw.get("cmd")
148 if isinstance(own, str) and own.strip():
149 cmd: str | None = own
150 source = "knobs.revert_check.cmd"
151 else:
152 cmd = build_cmd if isinstance(build_cmd, str) and build_cmd.strip() else None
153 source = "knobs.build_gate_cmd"
154 paths = raw.get("paths")
155 unit = raw.get("unit")
156 return Settings(
157 cmd=cmd,
158 cmd_source=source,
159 paths=tuple(_strings(paths)),
160 unit=unit if unit in UNITS else UNIT_HUNK,
161 max_changes=_positive(raw.get("max_changes"), DEFAULT_MAX_CHANGES),
162 budget_s=_positive(raw.get("budget_s"), DEFAULT_BUDGET_S),
163 run_timeout_s=max(1, int(gate_timeout_s)),
164 )
167def _positive(value: Any, default: int) -> int:
168 """A positive integer knob, or ``default`` (``bool`` is not a number here)."""
169 if isinstance(value, int) and not isinstance(value, bool) and value >= 1:
170 return value
171 return default
174def _strings(raw: Any) -> list[str]:
175 """Non-blank string entries of a list (anything else contributes nothing)."""
176 if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes, bytearray)):
177 return []
178 return [entry.strip() for entry in raw if isinstance(entry, str) and entry.strip()]
181def test_paths(policy_pack: Mapping[str, Any] | None) -> tuple[str, ...]:
182 """Where the project's tests live: every ``policy_pack.test_groups.*.test_paths`` glob.
184 Only the **declared** ``test_paths`` count. A group's ``paths`` are selectors and
185 routinely include the implementation surface (keel's own ``unit`` group selects
186 ``src/**``); read as test paths they would classify every production file as a test,
187 and the gate would skip every change it exists to check.
188 """
189 pack = policy_pack if isinstance(policy_pack, Mapping) else {}
190 groups = pack.get("test_groups")
191 if not isinstance(groups, Mapping):
192 return ()
193 globs: list[str] = []
194 for _name, group in sorted(groups.items(), key=lambda item: str(item[0])):
195 if isinstance(group, Mapping):
196 globs.extend(_strings(group.get("test_paths")))
197 return tuple(dict.fromkeys(globs))
200def is_production(path: str, *, tests: Sequence[str], paths: Sequence[str]) -> bool:
201 """Is ``path`` production code this gate reverts?
203 Never a test path. Then ``paths`` (``fnmatch`` globs, the matcher ``tier3_globs`` and
204 ``test_paths`` use) when the project set it, else a source-code suffix.
205 """
206 if any(fnmatch.fnmatch(path, glob) for glob in tests):
207 return False
208 if paths:
209 return any(fnmatch.fnmatch(path, glob) for glob in paths)
210 return path.lower().endswith(SOURCE_SUFFIXES)
213# --- the diff ------------------------------------------------------------------------
215_HUNK_RE = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@")
218@dataclass(frozen=True)
219class Hunk:
220 """One ``@@`` hunk: its header line and body, verbatim."""
222 header: str
223 lines: tuple[str, ...]
224 added: int
225 removed: int
226 #: Context lines. :func:`keel.git.revert_diff` asks for none, so any here means the
227 #: diff is not the shape the plan needs (:func:`context_problem`).
228 context: int = 0
230 @property
231 def pure_addition(self) -> bool:
232 """Only adds lines: reverting it removes code rather than restoring old code."""
233 return self.removed == 0
236@dataclass(frozen=True)
237class FileDiff:
238 """One file's part of a unified diff."""
240 path: str
241 #: The lines from ``diff --git`` up to the first hunk — what ``git apply`` needs to
242 #: know which file, and whether it is new or deleted.
243 header: tuple[str, ...]
244 hunks: tuple[Hunk, ...]
245 status: str # added | deleted | modified
247 @property
248 def mode(self) -> tuple[str, str] | None:
249 """``(old, new)`` when the diff changes the file's mode (``old mode`` / ``new mode``)."""
250 old = next((line[9:] for line in self.header if line.startswith("old mode ")), None)
251 new = next((line[9:] for line in self.header if line.startswith("new mode ")), None)
252 return (old, new) if old and new else None
254 @property
255 def binary(self) -> bool:
256 """git printed no text for it (``Binary files … differ``)."""
257 return any(line.startswith(("Binary files ", "GIT binary patch")) for line in self.header)
259 @property
260 def content_header(self) -> tuple[str, ...]:
261 """The header without its mode lines: a content hunk must not carry the mode along."""
262 return tuple(
263 line for line in self.header if not line.startswith(("old mode ", "new mode "))
264 )
267#: git's C-style escapes in a quoted path (``quote_c_style``), beside ``\\ooo`` octal bytes.
268_C_ESCAPES = {"a": 7, "b": 8, "t": 9, "n": 10, "v": 11, "f": 12, "r": 13, '"': 34, "\\": 92}
271def _unquote(name: str) -> str:
272 """A path as git wrote it -> the path itself.
274 git wraps a name in double quotes when it holds a control character, a quote or a
275 backslash, and escapes those inside (``"a/q\\"x.py"``); a byte is ``\\ooo`` octal. An
276 unquoted name is returned as it is.
277 """
278 if len(name) < 2 or not (name.startswith('"') and name.endswith('"')):
279 return name
280 body, out, i = name[1:-1], bytearray(), 0
281 while i < len(body):
282 char = body[i]
283 octal = body[i + 1 : i + 4]
284 if char == "\\" and body[i + 1 : i + 2] in _C_ESCAPES:
285 out.append(_C_ESCAPES[body[i + 1]])
286 i += 2
287 elif char == "\\" and len(octal) == 3 and all(c in "01234567" for c in octal):
288 out.append(int(octal, 8) & 0xFF)
289 i += 4
290 else:
291 out += char.encode("utf-8", "surrogateescape")
292 i += 1
293 return out.decode("utf-8", "surrogateescape")
296def _closing_quote(text: str) -> int:
297 """The index of the quote that closes the quoted name opening ``text``; ``-1`` if none."""
298 i = 1
299 while i < len(text):
300 if text[i] == "\\":
301 i += 2
302 continue
303 if text[i] == '"':
304 return i
305 i += 1
306 return -1
309def _strip(name: str, prefix: str) -> str:
310 return name[len(prefix) :] if name.startswith(prefix) else name
313def _path_from(line: str, prefix: str) -> str | None:
314 """The path on a ``--- a/x`` / ``+++ b/x`` line; ``None`` for ``/dev/null``.
316 git ends the line with a tab when the name contains a space, and quotes a name that
317 holds a control character, a quote or a backslash (:func:`_unquote`); the path is read
318 back so it matches the project's globs.
319 """
320 name = line[4:].rstrip("\t")
321 if name == "/dev/null":
322 return None
323 return _strip(_unquote(name), prefix)
326def _header_path(line: str) -> str:
327 """The path on a ``diff --git a/x b/x`` line — the only name a file with no ``---`` /
328 ``+++`` lines (a mode change, a binary file) carries.
330 With renames off both names are the same file, so an unquoted pair splits in the middle
331 (a name may hold ``" b/"``), and a quoted one ends at its closing quote. A header that
332 reads neither way falls back to the text after the last ``" b/"``.
333 """
334 rest = line[len("diff --git ") :]
335 if rest.startswith('"'):
336 end = _closing_quote(rest)
337 if end > 0:
338 return _strip(_unquote(rest[: end + 1]), "a/")
339 half = (len(rest) - 5) // 2
340 name = rest[2 : 2 + half]
341 if rest.startswith("a/") and rest[2 + half :] == f" b/{name}":
342 return name
343 return rest.rsplit(" b/", 1)[-1]
346def parse_diff(text: str) -> tuple[FileDiff, ...]:
347 """Split ``git diff --no-renames --src-prefix=a/ --dst-prefix=b/`` output into files.
349 Hunk bodies are consumed by the counts in their ``@@`` header, not by their first
350 character, so a blank context line (``diff.suppressBlankEmpty``) cannot end a hunk
351 early. A file with no hunk — binary, or a mode change — is kept with ``hunks=()``, so
352 the plan can say it was not checked rather than drop it.
353 """
354 files: list[FileDiff] = []
355 lines = text.split("\n")
356 i = 0
357 while i < len(lines):
358 if not lines[i].startswith("diff --git "):
359 i += 1
360 continue
361 header = [lines[i]]
362 fallback = _header_path(lines[i])
363 old = new = None
364 i += 1
365 while i < len(lines) and not lines[i].startswith(("@@", "diff --git ")):
366 line = lines[i]
367 if line.startswith("--- "):
368 old = _path_from(line, "a/")
369 elif line.startswith("+++ "):
370 new = _path_from(line, "b/")
371 header.append(line)
372 i += 1
373 hunks: list[Hunk] = []
374 while i < len(lines) and lines[i].startswith("@@"):
375 hunk, i = _read_hunk(lines, i)
376 hunks.append(hunk)
377 text_header = "\n".join(header)
378 if "\nnew file mode " in text_header:
379 status = "added"
380 elif "\ndeleted file mode " in text_header:
381 status = "deleted"
382 else:
383 status = "modified"
384 files.append(FileDiff(new or old or fallback, tuple(header), tuple(hunks), status))
385 return tuple(files)
388def _read_hunk(lines: list[str], i: int) -> tuple[Hunk, int]:
389 """Read the hunk whose header is ``lines[i]``; return it and the index after it."""
390 header = lines[i]
391 match = _HUNK_RE.match(header)
392 old_left = int(match.group(2) or "1") if match else 0
393 new_left = int(match.group(4) or "1") if match else 0
394 body: list[str] = []
395 added = removed = context = 0
396 i += 1
397 while i < len(lines) and (old_left > 0 or new_left > 0):
398 line = lines[i]
399 tag = line[:1]
400 if tag == "+":
401 added += 1
402 new_left -= 1
403 elif tag == "-":
404 removed += 1
405 old_left -= 1
406 elif tag == "\\":
407 pass # "\ No newline at end of file" belongs to the line before it
408 else: # context, including an empty line git wrote for a blank one
409 old_left -= 1
410 new_left -= 1
411 context += 1
412 body.append(line)
413 i += 1
414 # A trailing "\ No newline at end of file" marker after the last counted line.
415 while i < len(lines) and lines[i].startswith("\\"):
416 body.append(lines[i])
417 i += 1
418 return Hunk(header, tuple(body), added, removed, context), i
421def context_problem(files: Sequence[FileDiff]) -> str | None:
422 """Why the diff cannot be split into independent changes, or ``None`` when it can.
424 Every hunk must carry **no context line**. :func:`keel.git.revert_diff` pins
425 ``--unified=0`` and ``--inter-hunk-context=0``, but ``GIT_DIFF_OPTS`` or a setting keel
426 does not know could still widen a hunk — and a hunk that carries context is either
427 two edits merged into one change, where one caught edit passes the other, or a patch
428 whose context ``--unidiff-zero`` would misplace. Checked here, on what git printed.
429 """
430 widened = [f.path for f in files if any(h.context for h in f.hunks)]
431 if not widened:
432 return None
433 return (
434 f"the diff carries context lines (in {', '.join(widened[:3])}), so its hunks may merge "
435 "independent edits; check GIT_DIFF_OPTS and the diff settings in your git config"
436 )
439@dataclass(frozen=True)
440class Change:
441 """One production change to revert: a label for the report and its patch."""
443 label: str
444 path: str
445 #: A unified diff that ``git apply -R`` undoes on HEAD.
446 patch: str
447 #: Every hunk in it only adds lines — see :func:`judge` for why that matters.
448 pure_addition: bool
451@dataclass(frozen=True)
452class Plan:
453 """What the gate will revert, and what it had to leave out."""
455 changes: tuple[Change, ...]
456 #: Production files with no textual hunk (a binary file, or an empty file added or
457 #: deleted): not checked. A mode-only change is a change of its own, and is checked.
458 unrevertable: tuple[str, ...]
459 #: Every path the diff touched, for the finding that says none was production.
460 touched: tuple[str, ...]
463def plan_changes(
464 files: Sequence[FileDiff], *, tests: Sequence[str], paths: Sequence[str], unit: str
465) -> Plan:
466 """The production changes to revert, ordered by path.
468 Ordered here, not by git: ``diff.orderFile`` reorders git's output, and the order
469 decides which changes ``max_changes`` reaches. Each change is reverted alone, so each
470 must carry only itself:
472 * **A mode change is its own change** (``old mode``/``new mode``). Copied into every
473 content hunk's patch, it was undone with each of them, and one test of the
474 executable bit "caught" every hunk of the file.
475 * **Added Python code is split at its top-level definitions** (:func:`_blocks`) — a
476 hunk that only adds lines, and a new file's content, whatever ``unit`` says for a
477 new file. Undone whole, three added functions where a test calls one read as
478 "noticed" for all three; undone one at a time, each must be noticed on its own. A
479 new file that is one block stays one change, which deletes it. Other languages, and
480 added lines that do not parse on their own, are not split.
481 * Otherwise one change per hunk (``unit: hunk``) or per file (``unit: file``); a
482 deleted file is one change, which restores it.
483 """
484 changes: list[Change] = []
485 unrevertable: list[str] = []
486 for f in sorted(files, key=lambda f: f.path):
487 if not is_production(f.path, tests=tests, paths=paths):
488 continue
489 if f.mode is not None and f.status == "modified":
490 old, new = f.mode
491 mode_patch = "\n".join((f.header[0], f"old mode {old}", f"new mode {new}")) + "\n"
492 changes.append(Change(f"{f.path} (mode {old} -> {new})", f.path, mode_patch, False))
493 if f.binary or not f.hunks:
494 if f.binary or f.mode is None:
495 unrevertable.append(f.path)
496 continue
497 header = f.content_header
498 if f.status == "added":
499 blocks = _blocks(f.hunks[0], f.path) if len(f.hunks) == 1 else []
500 if len(blocks) > 1:
501 plain = (header[0], _old_side(header), *(h for h in header if h[:4] == "+++ "))
502 changes.extend(_block_change(f.path, plain, block) for block in blocks)
503 else:
504 changes.append(
505 Change(f"{f.path} (new file)", f.path, _patch(header, f.hunks), True)
506 )
507 continue
508 if unit == UNIT_FILE or f.status == "deleted":
509 suffix = " (deleted file)" if f.status == "deleted" else ""
510 changes.append(
511 Change(
512 f"{f.path}{suffix}",
513 f.path,
514 _patch(header, f.hunks),
515 all(h.pure_addition for h in f.hunks),
516 )
517 )
518 continue
519 for hunk in f.hunks:
520 blocks = _blocks(hunk, f.path) if hunk.pure_addition else []
521 if len(blocks) > 1:
522 changes.extend(_block_change(f.path, header, block) for block in blocks)
523 continue
524 label = f"{f.path} {hunk.header.split(' @@', 1)[0]} @@"
525 changes.append(Change(label, f.path, _patch(header, (hunk,)), hunk.pure_addition))
526 return Plan(tuple(changes), tuple(unrevertable), tuple(f.path for f in files))
529#: Files whose added code :func:`_blocks` can split: it parses them.
530_PYTHON_SUFFIXES = (".py", ".pyi")
531_DEFINITIONS = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
534def _old_side(header: Sequence[str]) -> str:
535 """A new file's ``--- a/x`` line, from its ``+++ b/x`` line as git wrote it (quoting and
536 all), so a patch that removes part of the file names the file that stays."""
537 new = next(h for h in header if h[:4] == "+++ ")
538 name = new[4:]
539 return "--- " + ('"a/' + name[3:] if name.startswith('"b/') else "a/" + name[2:])
542def _blocks(hunk: Hunk, path: str) -> list[tuple[int, tuple[str, ...]]]:
543 """A pure-addition hunk's lines split at its top-level definitions, as ``(first line, lines)``.
545 Only Python is split, and only when the added lines parse on their own
546 (:func:`_block_starts`): a boundary read from the text alone was wrong both ways — two
547 functions with no blank line between them stayed one change, so a test of one vouched
548 for the other, and a blank line inside a string literal split it, so each half's revert
549 was a syntax error. ``[]`` (keep the hunk whole) otherwise, and for a header that cannot
550 be read. Lines between blocks ride with the block before them (leading ones with the
551 first); a ``\\ No newline`` marker stays with its line.
552 """
553 match = _HUNK_RE.match(hunk.header)
554 if not match or not path.endswith(_PYTHON_SUFFIXES):
555 return []
556 starts = _block_starts([line[1:] for line in hunk.lines if line.startswith("+")])
557 if not starts:
558 return []
559 first = int(match.group(3))
560 blocks: list[tuple[int, list[str]]] = []
561 offset = 0
562 for line in hunk.lines:
563 if line.startswith("+"):
564 if not blocks or offset in starts:
565 blocks.append((first + offset, []))
566 offset += 1
567 blocks[-1][1].append(line)
568 return [(start, tuple(lines)) for start, lines in blocks]
571def _block_starts(added: Sequence[str]) -> set[int]:
572 """The 0-based added lines where a new block starts; empty when there is one block.
574 The lines are dedented by their common leading whitespace and parsed. Each ``def`` and
575 ``class`` (from its first decorator) is a block, and so is each run of other statements
576 between them. Empty when the lines do not parse on their own — part of an expression,
577 a string whose continuation sits left of the rest — or do not share one indent.
578 """
579 texts = [text for text in added if text.strip()]
580 if not texts:
581 return set()
582 width = min(len(text) - len(text.lstrip()) for text in texts)
583 indent = texts[0][:width]
584 if any(not text.startswith(indent) for text in texts):
585 return set()
586 source = "\n".join(text[width:] if text.strip() else "" for text in added) + "\n"
587 try:
588 tree = ast.parse(source)
589 except (SyntaxError, ValueError, RecursionError):
590 return set()
591 starts: set[int] = set()
592 previous: bool | None = None
593 for node in tree.body:
594 definition = isinstance(node, _DEFINITIONS)
595 if previous is not None and (definition or previous):
596 decorators = getattr(node, "decorator_list", [])
597 starts.add(min([node.lineno, *(d.lineno for d in decorators)]) - 1)
598 previous = definition
599 return starts
602def _block_change(path: str, header: Sequence[str], block: tuple[int, tuple[str, ...]]) -> Change:
603 """One block of added lines as its own change: a patch that removes just those lines."""
604 first, lines = block
605 count = sum(1 for line in lines if line.startswith("+"))
606 hunk_header = f"@@ -{first - 1},0 +{first},{count} @@"
607 patch = "\n".join((*header, hunk_header, *lines)) + "\n"
608 return Change(f"{path} {hunk_header}", path, patch, True)
611def _patch(header: Sequence[str], hunks: Sequence[Hunk]) -> str:
612 lines = list(header)
613 for hunk in hunks:
614 lines.append(hunk.header)
615 lines.extend(hunk.lines)
616 return "\n".join(lines) + "\n"
619# --- what a change looks like ------------------------------------------------------
620#
621# Nothing here skips a run. Every production change in scope is reverted and tested: an
622# earlier version skipped changes it judged "inert" (comments, docstrings, formatting),
623# and each proof of inertness had a hole — a Go ``//go:embed`` directive, a Python
624# ``# coding:`` cookie, a comment that moves ``__LINE__`` or ``f_lineno`` — because "no
625# test can observe this change" is not provable in general (#1289 review rounds).
627#: Leading text that makes a line **look** like a comment in some language. A wording
628#: heuristic only (:func:`looks_comment_only`): it never decides whether a change is run.
629_COMMENT_LOOKS: tuple[str, ...] = ("#", "//", "/*", "*/", "* ", "--", ";", "%")
632def _changed_lines(patch: str) -> list[str]:
633 """The ``+``/``-`` lines of a change's hunks, without their marker."""
634 _header, _sep, body = patch.partition("\n@@")
635 return [line[1:] for line in body.split("\n") if line[:1] in ("+", "-")]
638def looks_comment_only(change: Change) -> bool:
639 """Does every changed line look blank or like a comment? **Wording only.**
641 Used to explain a blocking "no test notices this change" finding, never to skip a
642 run: a comment can still change behaviour.
643 """
644 lines = _changed_lines(change.patch)
645 return bool(lines) and all(
646 not line.strip() or line.strip() == "*" or line.strip().startswith(_COMMENT_LOOKS)
647 for line in lines
648 )
651_NAMES = r"[A-Za-z_]\w*(?:\s+as\s+[A-Za-z_]\w*)?(?:\s*,\s*[A-Za-z_]\w*(?:\s+as\s+[A-Za-z_]\w*)?)*"
652_DOTTED = r"[A-Za-z_][\w.]*(?:\s+as\s+[A-Za-z_]\w*)?"
653#: One complete import statement on one line, and nothing else on it — no trailing
654#: comment, no ``;``, no open parenthesis left for a continuation line.
655_IMPORT_LINE = re.compile(
656 rf"^\s*(?:import\s+{_DOTTED}(?:\s*,\s*{_DOTTED})*"
657 rf"|from\s+(?:\.+[\w.]*|[A-Za-z_][\w.]*)\s+import\s+"
658 rf"(?:\*|{_NAMES}|\(\s*{_NAMES}\s*,?\s*\)))\s*$"
659)
662def imports_only(change: Change) -> bool:
663 """Is ``change`` a Python change whose every changed line is a whole import statement?
665 Read from the change's **own** lines, never from a comparison of the two files: an
666 AST cannot see an encoding cookie, and ``# coding: latin-1`` beside an import once
667 passed as "imports only" while it changed what a string literal decodes to. Blank
668 lines are ignored; a comment, a cookie, a continuation line of a multi-line import or
669 anything else makes the answer ``False`` — that change's errors then block, which is
670 the safe side. It classifies a result *after* its run; it never skips one.
671 """
672 if posixpath.splitext(change.path)[1].lower() not in (".py", ".pyi"):
673 return False
674 lines = [line for line in _changed_lines(change.patch) if line.strip()]
675 return bool(lines) and all(_IMPORT_LINE.match(line) for line in lines)
678#: The exceptions that say "a name the tests reach is gone" — all a revert of pure
679#: added code (or of an imported name) can produce. ``AttributeError`` counts only for a
680#: *module* or *class* attribute: ``'NoneType' object has no attribute`` is behaviour.
681_MISSING_NAME = re.compile(
682 r"^(?:NameError|UnboundLocalError|ImportError|ModuleNotFoundError)\b"
683 r"|^AttributeError: (?:(?:partially initialized )?module '[^']*'|type object '[^']*') "
684 r"has no attribute\b"
685)
686#: The start of a Python traceback; its exception line is the first unindented line after it.
687_TRACEBACK = "Traceback (most recent call last):"
688#: A pytest short-summary line's reason (`` - <Exception>: message``), any exception name.
689_PYTEST_REASON = re.compile(r"^(?:FAILED|ERROR) .*? - ([A-Za-z_][\w.]*(?::.*)?)$", re.MULTILINE)
692def _raised(output: str) -> list[str]:
693 """Every exception the output reports, as ``Name: message`` with the module dropped.
695 Read from each traceback's final line — whatever the exception is called, so a
696 custom ``Boom`` or a ``StopIteration`` is seen too — and from pytest's short summary.
697 """
698 found: list[str] = []
699 lines = output.split("\n")
700 for i, line in enumerate(lines):
701 if line.strip() != _TRACEBACK:
702 continue
703 rest = (text for text in lines[i + 1 :] if text.strip() and not text[:1].isspace())
704 found.append(next(rest, ""))
705 found.extend(match.group(1) for match in _PYTEST_REASON.finditer(output))
706 return [
707 name.rsplit(".", 1)[-1] + sep + message
708 for name, sep, message in (text.partition(":") for text in found)
709 ]
712def missing_names_only(output: str) -> bool:
713 """Does every exception the run reports say only that a name is missing?
715 The evidence a lenient reading needs: the error-only result of undoing an addition is
716 a ``nit`` only when the errors are the ones removing code can cause. A ``KeyError``, a
717 ``TypeError``, an attribute missing from an *instance*, any exception keel cannot
718 name, and output with no traceback at all are behaviour, and block.
720 **Every** failure the run counts must be described: a summary of ``2 failed`` whose
721 short summary names one ``NameError`` says nothing about the other, which could be an
722 assertion or a crash (#1289 review round 9). So pytest's undescribed failures
723 (:attr:`Tally.unclassified`) refuse it, and so does a count of failures and errors
724 larger than the exceptions keel read.
725 """
726 found = _raised(output)
727 tally = read_output(output)
728 return (
729 bool(found)
730 and not tally.unclassified
731 and len(found) >= tally.errors + tally.assertions
732 and all(_MISSING_NAME.match(text) for text in found)
733 )
736# --- reading a test run --------------------------------------------------------------
738_UNITTEST_RAN = re.compile(r"^Ran (\d+) tests? in ", re.MULTILINE)
739_UNITTEST_FAILED = re.compile(r"^FAILED \(([^)]*)\)\s*$", re.MULTILINE)
740_PYTEST_SUMMARY = re.compile(
741 r"^=*\s*((?:\d+ [a-z]+(?:, )?)+) in [\d.]+s\b.*$|^=*\s*no tests ran in [\d.]+s\b.*$",
742 re.MULTILINE,
743)
744_PYTEST_COUNT = re.compile(r"(\d+) ([a-z]+)")
745#: A pytest short-summary line. The node id never starts with ``(``, which is what keeps
746#: unittest's own ``FAILED (failures=1)`` from being read as a failed pytest test.
747#:
748#: The node id may contain spaces — a parametrized id (``test_v[hello world]``) or a file
749#: name — so it is read as: a bracketed id up to its ``]``, else one word, else the
750#: shortest text before `` - ``; the reason is what follows `` - `` and may be absent.
751_PYTEST_LINE = re.compile(r"^(FAILED|ERROR) (?!\()(\S*?\[.*?\]|\S+|.+?)(?: - (.*))?$", re.MULTILINE)
752#: A unittest failure section's header: ``FAIL: test_x (pkg.T.test_x)`` is an assertion,
753#: ``ERROR: …`` anything else. What the differential reading compares.
754_UNITTEST_HEADER = re.compile(r"^(FAIL|ERROR): (.+?)\s*$", re.MULTILINE)
755#: A pytest short-summary reason that is an assertion: a rewritten ``assert``, an
756#: ``AssertionError`` (unittest-style assertions under pytest), or ``pytest.fail``.
757#: ``Failed: Timeout`` is excluded: pytest-timeout reports a timed-out test that way, and
758#: a hang is not an assertion.
759_ASSERTION_REASON = re.compile(r"^(?:assert\b|AssertionError\b|Failed:(?! Timeout\b))")
762@dataclass(frozen=True)
763class Tally:
764 """What a test run's output says happened."""
766 #: The output carries a unittest or pytest summary at all.
767 recognized: bool = False
768 #: Tests that ran.
769 ran: int = 0
770 #: Tests that failed as an assertion.
771 assertions: int = 0
772 #: Tests that failed any other way: an exception, an import or collection error, a
773 #: strict unexpected success.
774 errors: int = 0
775 #: pytest counted a failure its short summary does not describe (``-rN``, truncation).
776 unclassified: int = 0
777 #: The ids of the tests that failed as an assertion, where the runner names them.
778 assertion_ids: frozenset[str] = frozenset()
779 #: The ids of every failing test — assertion or not — where the runner names them.
780 failure_ids: frozenset[str] = frozenset()
782 @property
783 def failed(self) -> bool:
784 """Does the output report any failing or erroring test at all?"""
785 return bool(self.assertions or self.errors or self.unclassified or self.failure_ids)
788def read_output(output: str) -> Tally:
789 """Read unittest's and pytest's summaries out of a test command's combined output.
791 **unittest** separates the two cleanly: ``failures`` are the test's
792 ``failureException`` (an ``AssertionError``), ``errors`` are anything else, a test
793 module that failed to import included.
795 **pytest** does not: its ``failed`` count includes a ``NameError`` raised in the test
796 body. The short test summary (``-rfE``, pytest's default since 6.0) names each
797 failure's exception, and only an assertion reason counts; a failure the summary does
798 not describe is *unclassified*, never an assertion.
800 Every summary in the output is summed, so a command running two suites is read whole.
801 """
802 ran = assertions = errors = unclassified = 0
803 recognized = False
804 for match in _UNITTEST_RAN.finditer(output):
805 recognized = True
806 ran += int(match.group(1))
807 for match in _UNITTEST_FAILED.finditer(output):
808 for part in match.group(1).split(","):
809 key, _, value = part.strip().partition("=")
810 count = int(value) if value.isdigit() else 0
811 if key == "failures":
812 assertions += count
813 elif key in ("errors", "unexpected successes"):
814 errors += count
815 pytest_failed = 0
816 for match in _PYTEST_SUMMARY.finditer(output):
817 recognized = True
818 for count, word in _PYTEST_COUNT.findall(match.group(1) or ""):
819 if word in ("passed", "failed", "xfailed", "xpassed"):
820 ran += int(count)
821 if word == "failed":
822 pytest_failed += int(count)
823 elif word in ("error", "errors"):
824 errors += int(count)
825 described = 0
826 asserted: set[str] = set()
827 failing: set[str] = set()
828 for kind, test_id in _UNITTEST_HEADER.findall(output):
829 failing.add(test_id)
830 if kind == "FAIL":
831 asserted.add(test_id)
832 for match in _PYTEST_LINE.finditer(output):
833 failing.add(match.group(2))
834 if match.group(1) != "FAILED":
835 continue # ERROR lines are already in the summary's error count
836 described += 1
837 if _ASSERTION_REASON.match(match.group(3) or ""):
838 assertions += 1
839 asserted.add(match.group(2))
840 else:
841 errors += 1
842 unclassified = max(0, pytest_failed - described)
843 return Tally(
844 recognized,
845 ran,
846 assertions,
847 errors,
848 unclassified,
849 frozenset(asserted),
850 frozenset(failing),
851 )
854def classify(
855 *, exit_ok: bool, timed_out: bool, output: str, baseline_failed: frozenset[str] = frozenset()
856) -> tuple[str, str]:
857 """One reverted run -> ``(result, why)``. Only :data:`CAUGHT` is a pass.
859 The exit code decides pass/fail; the output decides *how* it failed. A test that
860 errored is not a test that asserted — reverting half a fix can raise ``NameError``
861 in every test that imports it, and that proves the import, not the behaviour.
863 **Differential.** ``baseline_failed`` names the tests that failed without the revert.
864 Where the runner names the assertion failures, the change is caught only by one
865 *not* among them, so a failure that was already there cannot certify it.
866 (:func:`baseline_problem` already refuses a baseline with any failure; this holds the
867 rule on its own too.)
868 """
869 if timed_out:
870 return TIMED_OUT, "the test command timed out with the change reverted"
871 if exit_ok:
872 return UNNOTICED, "the test command passed with the change reverted"
873 tally = read_output(output)
874 if tally.assertion_ids and tally.assertion_ids <= baseline_failed:
875 return UNNOTICED, (
876 "the only tests that failed as an assertion also fail without the revert"
877 )
878 if tally.assertions:
879 return CAUGHT, f"{tally.assertions} test(s) failed as an assertion"
880 if not tally.recognized:
881 return UNREADABLE, (
882 "the test command failed, and its output carries no unittest or pytest summary "
883 "to say whether a test failed"
884 )
885 if tally.errors:
886 return ERRORED, f"{tally.errors} test(s) errored and none failed as an assertion"
887 if tally.unclassified:
888 return UNREADABLE, (
889 f"pytest reported {tally.unclassified} failure(s) its short test summary does not "
890 "describe, so an assertion cannot be told from an error (keep pytest's -rf)"
891 )
892 return UNREADABLE, "the test command failed, but no test did"
895def baseline_problem(*, exit_ok: bool, timed_out: bool, output: str) -> str | None:
896 """Why the unreverted run cannot anchor the check, or ``None`` when it can.
898 A failure after a revert means something only if the same command passes — and is
899 readable — on HEAD itself, in the same kind of clean checkout.
900 """
901 if timed_out:
902 return "the test command timed out on a clean checkout of HEAD"
903 if not exit_ok:
904 return (
905 "the test command fails on a clean checkout of HEAD, so a failure with a change "
906 "reverted would prove nothing"
907 )
908 tally = read_output(output)
909 if not tally.recognized:
910 return (
911 "the test command's output carries no unittest or pytest summary; revert-check "
912 "reads those two to tell an assertion from an error"
913 )
914 if tally.ran == 0:
915 return "the test command ran no tests on a clean checkout of HEAD"
916 if tally.failed:
917 # `suite1; suite2` exits with suite2's status: a suite already failing an
918 # assertion would otherwise lend that assertion to every revert.
919 return (
920 "the test command exits 0 on a clean checkout of HEAD but its output reports "
921 "failing tests, so a failure with a change reverted would prove nothing"
922 )
923 return None
926# --- the bounded loop ----------------------------------------------------------------
929@dataclass(frozen=True)
930class RunResult:
931 """One execution of the test command, as the I/O layer observed it."""
933 exit_ok: bool = False
934 timed_out: bool = False
935 output: str = ""
938@dataclass(frozen=True)
939class Reverted:
940 """One change undone in the scratch tree, as the I/O layer observed it."""
942 #: The tree was reset and the reverse patch applied.
943 applied: bool
946@dataclass(frozen=True)
947class ChangeResult:
948 change: Change
949 result: str
950 why: str
951 #: The change only adds code or import lines (:attr:`Change.pure_addition`, or
952 #: :func:`imports_only`) **and** every error its run reports is a missing name
953 #: (:func:`missing_names_only`) — the one error-only result :func:`judge` reads as a
954 #: ``nit``.
955 adds_only: bool = False
958@dataclass(frozen=True)
959class Report:
960 """What :func:`execute` observed."""
962 #: Set when the baseline run cannot anchor the check; nothing was reverted then.
963 baseline: str | None
964 results: tuple[ChangeResult, ...] = ()
965 #: Changes left unchecked by ``max_changes`` or ``budget_s``, with the reason.
966 not_checked: tuple[tuple[Change, str], ...] = ()
969#: ``revert(change)``: reset the scratch tree to ``HEAD`` and undo ``change`` in it.
970Reverter = Callable[[Change], Reverted]
971#: ``test(timeout_s)``: run the test command in the scratch tree as it stands.
972Tester = Callable[[int], RunResult]
975def execute(
976 changes: Sequence[Change],
977 settings: Settings,
978 *,
979 revert: Reverter,
980 test: Tester,
981 clock: Callable[[], float],
982) -> Report:
983 """Run the baseline, then revert each change alone, within the cost bounds.
985 ``max_changes`` caps how many changes are reverted; ``budget_s`` caps the wall clock
986 of the baseline and every revert together, and each run's limit is the smaller of
987 ``knobs.gate_timeout_s`` and what is left of it. A change either bound leaves out is
988 reported **not checked** — never as caught. Every change that is reverted is tested:
989 nothing is skipped for looking harmless.
990 """
991 start = clock()
993 def left() -> float:
994 return settings.budget_s - (clock() - start)
996 def limit(remaining: float) -> int:
997 return max(1, min(settings.run_timeout_s, int(remaining)))
999 base = test(limit(left()))
1000 problem = baseline_problem(exit_ok=base.exit_ok, timed_out=base.timed_out, output=base.output)
1001 if problem is not None:
1002 return Report(problem)
1003 baseline_failed = read_output(base.output).failure_ids
1004 results: list[ChangeResult] = []
1005 skipped: list[tuple[Change, str]] = []
1006 for index, change in enumerate(changes):
1007 if index >= settings.max_changes:
1008 skipped.append(
1009 (change, f"over knobs.revert_check.max_changes ({settings.max_changes})")
1010 )
1011 continue
1012 remaining = left()
1013 if remaining < 1:
1014 skipped.append(
1015 (change, f"the knobs.revert_check.budget_s budget ({settings.budget_s}s) ran out")
1016 )
1017 continue
1018 undone = revert(change)
1019 if not undone.applied:
1020 results.append(
1021 ChangeResult(change, NOT_APPLIED, "git apply -R could not undo it on HEAD")
1022 )
1023 continue
1024 # Re-read the clock: resetting, cleaning, applying and reading took time too, and a
1025 # run must never start with a limit the budget no longer has.
1026 remaining = left()
1027 if remaining < 1:
1028 skipped.append(
1029 (change, f"the knobs.revert_check.budget_s budget ({settings.budget_s}s) ran out")
1030 )
1031 continue
1032 outcome = test(limit(remaining))
1033 result, why = classify(
1034 exit_ok=outcome.exit_ok,
1035 timed_out=outcome.timed_out,
1036 output=outcome.output,
1037 baseline_failed=baseline_failed,
1038 )
1039 adds_only = (change.pure_addition or imports_only(change)) and missing_names_only(
1040 outcome.output
1041 )
1042 results.append(ChangeResult(change, result, why, adds_only))
1043 return Report(None, tuple(results), tuple(skipped))
1046# --- the verdict ---------------------------------------------------------------------
1049@dataclass(frozen=True)
1050class Verdict:
1051 """The gate's outcome, in :class:`keel.gates.GateOutcome` terms."""
1053 ok: bool
1054 findings: tuple[Finding, ...]
1055 #: The gate cannot judge — only the project's config or environment can fix it.
1056 unconfigured: bool = False
1057 #: Nothing to check: the diff has no production change. Reported ``SKIPPED``.
1058 skipped: bool = False
1061def cannot_judge(why: str) -> Verdict:
1062 """A verdict for a check that could not run: failed, never a pass (#1364)."""
1063 return Verdict(False, (Finding(_BLOCK, f"cannot judge: {why}", GATE_ID),), unconfigured=True)
1066#: Why the gate cannot judge a plan with no guard or test gate beside it (#1289 review).
1067PLANNED_ALONE = (
1068 "revert-check is not a test gate: it reverts each change against a suite the guard and "
1069 "test gates proved green, and none is planned — list build (with knobs.build_gate_cmd) "
1070 "beside it in gates:"
1071)
1074def precheck(settings: Settings, *, tests: Sequence[str], gates_green: bool) -> Verdict | None:
1075 """The verdict when the check must not start, else ``None``.
1077 Order matters: a missing command or test layout is a config problem no run can fix,
1078 so it is reported before the red-gates case an implementer can.
1079 """
1080 if settings.cmd is None:
1081 return cannot_judge(
1082 "no test command: set knobs.revert_check.cmd or knobs.build_gate_cmd "
1083 "in .keel/project.yaml"
1084 )
1085 if not tests:
1086 return cannot_judge(
1087 "revert-check cannot tell tests from production: declare where the tests live "
1088 "in policy_pack.test_groups.<group>.test_paths"
1089 )
1090 if not gates_green:
1091 return Verdict(
1092 False,
1093 (
1094 Finding(
1095 _BLOCK,
1096 "not run: the guard and test gates are red, and a revert check against a "
1097 "red suite cannot tell which failures the revert caused",
1098 GATE_ID,
1099 ),
1100 ),
1101 )
1102 return None
1105def judge(plan: Plan, report: Report | None) -> Verdict:
1106 """The gate's verdict: every production change must make a test fail as an assertion.
1108 * **unnoticed**, **timed out**, **unreadable**, **not applied** and **not checked**
1109 each block, naming the change: a change the gate did not see fail is not a pass.
1110 * **errored** — and **unreadable**, where the suite crashed before its summary — block,
1111 except for a change that only *adds* lines or only changes import lines **and**
1112 whose run reported nothing but missing names (``NameError``, ``ImportError``, a
1113 module or class attribute). Reverting such an addition removes a name, and a test
1114 that calls it can then only error: no assertion can fail against code that is not
1115 there. The error does prove a test depends on it, so it is a ``nit``. Any other
1116 error — a ``KeyError``, a ``TypeError``, an instance attribute — is behaviour a test
1117 should assert on, and blocks, as does any error after reverting a modification.
1118 * A production file with no textual hunk (binary, or an empty file added or deleted)
1119 blocks as **not checked**: nothing could be reverted, and the gate never certifies
1120 what it did not check.
1121 * A diff with no production change is ``SKIPPED`` — judged, with nothing to check.
1123 ``report`` is ``None`` only when there was nothing to execute.
1124 """
1125 findings: list[Finding] = []
1126 for path in plan.unrevertable:
1127 findings.append(
1128 Finding(
1129 _BLOCK,
1130 f"{path}: not checked — no textual hunk to revert (a binary file, or an empty file "
1131 "added or deleted)",
1132 GATE_ID,
1133 )
1134 )
1135 if report is None:
1136 if findings:
1137 return Verdict(False, tuple(findings))
1138 touched = len(plan.touched)
1139 return Verdict(
1140 True,
1141 (
1142 Finding(
1143 "nit",
1144 f"nothing to revert: none of the {touched} changed file(s) is production "
1145 "code (knobs.revert_check.paths, or a source suffix outside the test paths)",
1146 GATE_ID,
1147 ),
1148 ),
1149 skipped=True,
1150 )
1151 if report.baseline is not None:
1152 return cannot_judge(report.baseline)
1153 caught = 0
1154 for item in report.results:
1155 label = item.change.label
1156 if item.result == CAUGHT:
1157 caught += 1
1158 elif item.result in (ERRORED, UNREADABLE) and item.adds_only:
1159 findings.append(
1160 Finding(
1161 "nit",
1162 f"{label}: noticed only as a missing name — {item.why}; the change only "
1163 "adds code or import lines, so no assertion can fail without it",
1164 GATE_ID,
1165 )
1166 )
1167 elif item.result == UNNOTICED:
1168 hint = (
1169 "; it looks comment-only, and keel tests every change, because a comment can "
1170 "still change behaviour (encoding cookies, compiler directives, line "
1171 "numbers) — add a test that notices it, or scope such files out with "
1172 "knobs.revert_check.paths if you do not want them checked"
1173 if looks_comment_only(item.change)
1174 else ""
1175 )
1176 findings.append(
1177 Finding(_BLOCK, f"{label}: no test notices this change — {item.why}{hint}", GATE_ID)
1178 )
1179 else:
1180 findings.append(
1181 Finding(_BLOCK, f"{label}: no test failed as an assertion — {item.why}", GATE_ID)
1182 )
1183 for change, why in report.not_checked:
1184 findings.append(Finding(_BLOCK, f"{change.label}: not checked — {why}", GATE_ID))
1185 total = len(report.results) + len(report.not_checked)
1186 summary = (
1187 f"{caught} of {total} production change(s) made a test fail as an assertion when "
1188 "reverted alone"
1189 )
1190 findings.append(Finding("nit", summary, GATE_ID))
1191 blocked = any(f.severity == _BLOCK for f in findings)
1192 return Verdict(not blocked, tuple(findings))