Coverage for src/keel/mergeverify.py: 100%

84 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 19:17 +0000

1"""Did the merge apply what was reviewed? — the pure comparison (issue #561). 

2 

3`keel merge` proves a merge *succeeded*. It does not prove the merge applied the 

4diff that was reviewed, and those came apart twice in one day while shipping 

51.8.1/1.8.2: a `gh api …/update-branch` merge commit followed by a GitHub 

6squash-merge silently reverted unrelated already-merged work. Neither revert was 

7caught by CI, because the reverted state was internally consistent — old code with 

8no test for the removed behaviour — so the suite stayed green throughout. 

9 

10**Why the obvious check does not work.** The first version of this module compared 

11the PR's file set against the merge's file set, on the theory that a revert shows up 

12as the merge touching files the PR never did. Run against the actual incident 

13(#543's squash reverting #550) it reported **clean**, and the reason is the whole 

14point: `update-branch` had already pulled the reverting state into the branch, so 

15GitHub computed the PR's own diff — the thing a reviewer reads — as *including* 

16those files. The revert was inside the reviewed diff. Scope comparison cannot see it. 

17 

18**What does work** is the timing fingerprint the incident actually leaves: 

19 

20* #550 merged at 13:02, touching ``src/keel/github.py`` and three others; 

21* #543 had branched on the 8th, **before** that; 

22* #543 merged at 15:19 and its commit **removed 41 lines** from ``github.py`` — 

23 a file a "label the search input" change has no reason to touch at all. 

24 

25So the question to ask is: *did this merge write to files that some other pull 

26request changed after this one branched?* That is the only way an update-branch 

27squash can undo merged work, and it is cheap to answer. It is a **look at this** 

28signal rather than a proof — two PRs editing one file in sequence is ordinary — so 

29the report names the overtaking PR for each file and lets a human judge. 

30 

31Pure: lists in, report out. The CLI does every GitHub read. 

32""" 

33 

34from __future__ import annotations 

35 

36from collections.abc import Mapping, Sequence 

37 

38#: Report shape version, so a consumer can tell an old report from a new one. 

39SCHEMA_VERSION = "keel.merge-verify.v1" 

40 

41 

42def verify_merge( 

43 landed: Sequence[str] | None, 

44 overtaken: Mapping[str, int | Sequence[int]] | None = None, 

45 intended: Sequence[str] | None = None, 

46 *, 

47 added_by_pr: Mapping[int, Mapping[str, Sequence[str] | None]] | None = None, 

48 parent_contents: Mapping[str, str | None] | None = None, 

49 merged_contents: Mapping[str, str | None] | None = None, 

50) -> dict: 

51 """Judge whether a merge may have silently reverted other merged work. 

52 

53 ``landed`` is what the merge commit changed. ``overtaken`` maps a path to the 

54 pull request (or pull requests) that changed it **after this PR branched and 

55 before this PR merged** — the window in which a stale branch can carry a revert. 

56 ``intended`` is the PR's own file list, used only for the weaker secondary signal. 

57 

58 ``None`` for ``landed`` means *not observed* and yields ``unknown`` rather than 

59 a clean bill: failing to look is not evidence that nothing drifted. 

60 

61 ``status``: 

62 

63 * ``drift`` — the merge wrote to files another PR changed after this one 

64 branched and lost added lines present in the merge commit's first parent (or 

65 was unverifiable). 

66 * ``out-of-scope`` — no drift, but the merge changed files the PR's own 

67 diff did not list. A different (rarer) way for a merge to do more than it said. 

68 * ``clean`` — neither (or all shared files retained their added lines). 

69 * ``unknown`` — nothing could be read. 

70 """ 

71 if landed is None: 

72 return _report("unknown", "could not read the merge commit's file list from GitHub") 

73 landed_set = {p.strip() for p in landed if p.strip()} 

74 raw_collisions = { 

75 path.strip(): ([pr] if isinstance(pr, int) else list(dict.fromkeys(pr))) 

76 for path, pr in sorted((overtaken or {}).items()) 

77 if path.strip() and path.strip() in landed_set and pr 

78 } 

79 collisions: dict[str, int] = {} 

80 if raw_collisions: 

81 collisions, lost_lines, unverifiable, unverifiable_by_path = _check_survival( 

82 raw_collisions, added_by_pr, parent_contents, merged_contents 

83 ) 

84 if lost_lines or unverifiable: 

85 parts: list[str] = [] 

86 for p, info in lost_lines.items(): 

87 by_pr = info.get("by_pr") or {info["pr"]: {"count": info["count"]}} 

88 sub = [f"#{pr_num}: {pr_info['count']} lost" for pr_num, pr_info in by_pr.items()] 

89 parts.append(f"{p} ({', '.join(sub)})") 

90 for p in unverifiable: 

91 sub = [f"#{pr_num}: unverifiable" for pr_num in unverifiable_by_path[p]] 

92 parts.append(f"{p} ({', '.join(sub)})") 

93 return _report( 

94 "drift", 

95 f"the merge wrote to {len(parts)} file(s) that another pull request " 

96 f"changed after this one branched — the shape of a silent revert: " 

97 f"{', '.join(parts)}", 

98 overtaken=collisions, 

99 lost_lines=lost_lines, 

100 unverifiable=unverifiable, 

101 unverifiable_prs=unverifiable_by_path, 

102 landed_count=len(landed_set), 

103 ) 

104 if intended is not None: 

105 unexpected = sorted(landed_set - {p.strip() for p in intended if p.strip()}) 

106 if unexpected: 

107 return _report( 

108 "out-of-scope", 

109 f"the merge changed {len(unexpected)} file(s) the pull request's own " 

110 "diff did not list", 

111 overtaken=collisions, 

112 unexpected=unexpected, 

113 landed_count=len(landed_set), 

114 ) 

115 if collisions: 

116 return _report( 

117 "clean", 

118 f"clean ({len(collisions)} shared files, 0 lost lines)", 

119 overtaken=collisions, 

120 landed_count=len(landed_set), 

121 ) 

122 return _report( 

123 "clean", 

124 "no file in this merge was changed by another pull request after this one branched", 

125 landed_count=len(landed_set), 

126 ) 

127 

128 

129def _check_survival( 

130 collisions: Mapping[str, Sequence[int]], 

131 added_by_pr: Mapping[int, Mapping[str, Sequence[str] | None]] | None, 

132 parent_contents: Mapping[str, str | None] | None, 

133 merged_contents: Mapping[str, str | None] | None, 

134) -> tuple[dict[str, int], dict[str, dict], list[str], dict[str, list[int]]]: 

135 overtaken: dict[str, int] = {} 

136 lost_lines: dict[str, dict] = {} 

137 unverifiable: list[str] = [] 

138 unverifiable_by_path: dict[str, list[int]] = {} 

139 for path, prs in collisions.items(): 

140 parent = (parent_contents or {}).get(path) 

141 merged = (merged_contents or {}).get(path) 

142 pr_added = {pr: ((added_by_pr or {}).get(pr) or {}).get(path) for pr in prs} 

143 if parent is None or merged is None or any(lines is None for lines in pr_added.values()): 

144 overtaken[path] = prs[0] 

145 unverifiable.append(path) 

146 unverifiable_by_path[path] = list(prs) 

147 continue 

148 in_parent = {line for line in parent.splitlines() if line.strip()} 

149 in_merged = {line for line in merged.splitlines() if line.strip()} 

150 missing_all: list[str] = [] 

151 by_pr: dict[int, dict] = {} 

152 for pr, lines in pr_added.items(): 

153 missing = [ 

154 line 

155 for line in (lines or ()) 

156 if line.strip() and line in in_parent and line not in in_merged 

157 ] 

158 if missing: 

159 by_pr[pr] = {"count": len(missing), "samples": missing[:3]} 

160 missing_all.extend(missing) 

161 if by_pr: 

162 first_pr = next(iter(by_pr)) 

163 overtaken[path] = first_pr 

164 entry: dict = { 

165 "pr": first_pr, 

166 "count": len(missing_all), 

167 "samples": missing_all[:3], 

168 } 

169 if len(by_pr) > 1: 

170 entry["by_pr"] = by_pr 

171 lost_lines[path] = entry 

172 else: 

173 overtaken[path] = prs[0] 

174 return overtaken, lost_lines, unverifiable, unverifiable_by_path 

175 

176 

177def _report(status: str, reason: str, **extra) -> dict: 

178 report = { 

179 "schema_version": SCHEMA_VERSION, 

180 "status": status, 

181 "reason": reason, 

182 "overtaken": {}, 

183 "lost_lines": {}, 

184 "unverifiable": [], 

185 "unverifiable_prs": {}, 

186 "unexpected": [], 

187 "landed_count": 0, 

188 # Present on every report, so a consumer never has to distinguish 

189 # "complete" from "this key was added later". Set by the caller when a 

190 # finding is kept despite an input it could not read. 

191 "incomplete": False, 

192 } 

193 report.update(extra) 

194 return report 

195 

196 

197def is_drift(report: dict) -> bool: 

198 """True when the report is the loud case a human must look at.""" 

199 return isinstance(report, dict) and report.get("status") == "drift" 

200 

201 

202def render(report: dict) -> str: 

203 """Human-readable one-block summary.""" 

204 lines = [ 

205 f"keel verify-merge — {report.get('status', 'unknown')}", 

206 f" {report.get('reason', '')}", 

207 ] 

208 lost = report.get("lost_lines") or {} 

209 unverifiable = set(report.get("unverifiable") or []) 

210 for path, pr in (report.get("overtaken") or {}).items(): 

211 if path in lost: 

212 info = lost[path] 

213 by_pr = info.get("by_pr") or { 

214 info.get("pr", pr): { 

215 "count": info["count"], 

216 "samples": info.get("samples") or [], 

217 } 

218 } 

219 for pr_num, pr_info in by_pr.items(): 

220 lines.append( 

221 f" {path} — also changed by #{pr_num} ({pr_info['count']} lost line(s))" 

222 ) 

223 for sample in pr_info.get("samples") or []: 

224 lines.append(f" - {sample}") 

225 elif path in unverifiable: 

226 for pr_num in report.get("unverifiable_prs", {}).get(path, [pr]): 

227 lines.append(f" {path} — also changed by #{pr_num} (unverifiable)") 

228 for path in report.get("unexpected") or []: 

229 lines.append(f" {path} — not in the PR's own diff") 

230 return "\n".join(lines)