Coverage for src/keel/lock.py: 100%
140 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-02 20:26 +0000
1"""Single-host resource claims backed by atomic ``mkdir``.
3Every merge goes through the merge lock (a keel invariant) so concurrent ``ship``
4runs on the same checkout cannot race the branch tip. The merge lock is now one
5consumer of the generalized resource-claim primitive below: ``mkdir`` is atomic,
6so a resource directory is either created for one owner or already held.
7"""
9from __future__ import annotations
11import hashlib
12import json
13import os
14import re
15from collections.abc import Iterator
16from contextlib import contextmanager
17from dataclasses import asdict, dataclass
18from pathlib import Path
19from typing import Any
21from . import workspace
23SCHEMA_VERSION = "keel.resource-claim.v1"
25#: Holder of a claim whose owner cannot be read — ``owner.json`` missing, corrupt,
26#: unreadable, or the wrong shape. Deliberately *not* ``None``: the claim directory
27#: exists, so the resource **is** held; we simply cannot name by whom. The window is
28#: narrow but real, because :func:`_claim_path` creates the directory before it writes
29#: the owner file. An *error* in that window is now unwound on a **best-effort** basis
30#: (#1077), which leaves three ways to end up here: the unmaskable one — ``SIGKILL``,
31#: container teardown, power loss — where no handler of ours ever runs; a cleanup the
32#: handler cannot finish, such as an unwritable directory or a stray file the failed step
33#: left behind; and one the handler deliberately declines, because it can no longer prove
34#: the directory is the one it created and refuses to delete a stranger's claim. ``keel
35#: release <resource>`` with no ``--owner`` recovers all three.
36UNKNOWN_HOLDER = "<unknown>"
39class LockError(RuntimeError):
40 """Raised when the merge lock is already held."""
43@dataclass(frozen=True)
44class ClaimResult:
45 """Structured result for a single-host resource claim operation."""
47 schema_version: str
48 resource: str
49 owner: str
50 path: str
51 granted: bool
52 status: str
53 reason: str
54 holder: str | None = None
56 def as_dict(self) -> dict[str, Any]:
57 """Return a JSON-compatible deterministic representation."""
58 return asdict(self)
61def contract_as_dict() -> dict[str, Any]:
62 """Return the stable resource-claim contract."""
63 return {
64 "schema_version": SCHEMA_VERSION,
65 "consumer_neutral": True,
66 "deterministic": True,
67 "stdlib_only": True,
68 "scope": "single-host",
69 "primitive": "mkdir",
70 "deny_mode": "structured-feedback",
71 "statuses": ["granted", "denied", "released", "missing", "not-owner"],
72 "merge_lock_consumer": True,
73 "stale_recovery": "caller-owned",
74 }
77def claim_resource(root: str | Path, resource: str, *, owner: str) -> ClaimResult:
78 """Claim a named resource under ``root`` for exactly one owner.
80 Contention is structured feedback: a resource already held comes back as a ``denied``
81 result, never an exception. An **I/O failure** is not contention and still raises, the
82 same way :func:`release_resource` propagates one — ``denied`` has to keep meaning
83 "somebody else holds this" for callers that back off and retry on it. A failure after
84 the directory is created unwinds the half-built claim, so a raise normally leaves the
85 resource free rather than held by nobody; that unwind is best-effort and removes only
86 the directory this very call created (see :func:`_discard_partial_claim`).
87 """
88 path = resource_path(root, resource)
89 return _claim_path(path, resource=_clean(resource, "unknown-resource"), owner=owner)
92def release_resource(
93 root: str | Path,
94 resource: str,
95 *,
96 owner: str | None = None,
97 best_effort: bool = False,
98) -> ClaimResult:
99 """Release a named resource claim, optionally requiring the owner to match."""
100 path = resource_path(root, resource)
101 return _release_path(
102 path,
103 resource=_clean(resource, "unknown-resource"),
104 owner=owner,
105 best_effort=best_effort,
106 )
109@contextmanager
110def resource_claim(root: str | Path, resource: str, *, owner: str) -> Iterator[ClaimResult]:
111 """Context manager that yields a structured claim result and releases on success."""
112 result = claim_resource(root, resource, owner=owner)
113 try:
114 yield result
115 finally:
116 if result.granted:
117 release_resource(root, resource, owner=owner)
120def resource_path(root: str | Path, resource: str) -> Path:
121 """Return the deterministic lock directory path for ``resource``."""
122 name = _clean(resource, "unknown-resource")
123 slug = re.sub(r"[^A-Za-z0-9_.-]+", "-", name).strip(".-").lower()
124 slug = slug or "resource"
125 digest = hashlib.sha256(name.encode("utf-8")).hexdigest()[:12]
126 return Path(root) / f"{slug}-{digest}.lock"
129@contextmanager
130def merge_lock(lock_dir: str | Path) -> Iterator[Path]:
131 """Acquire the merge lock for the duration of the ``with`` block."""
132 path = Path(lock_dir)
133 result = _claim_path(path, resource="merge", owner="merge-lock")
134 if not result.granted:
135 raise LockError(f"merge lock already held: {path}")
136 try:
137 yield path
138 finally:
139 _release_path(path, resource="merge", owner="merge-lock", best_effort=True)
142def _claim_path(path: Path, *, resource: str, owner: str) -> ClaimResult:
143 clean_owner = _clean(owner, "unknown-owner")
144 try:
145 path.mkdir(parents=True)
146 except FileExistsError:
147 return ClaimResult(
148 schema_version=SCHEMA_VERSION,
149 resource=resource,
150 owner=clean_owner,
151 path=str(path),
152 granted=False,
153 status="denied",
154 reason="resource-already-claimed",
155 holder=_holder(path),
156 )
157 # Which directory this call made, recorded — and, where the platform allows it, held
158 # open — while it is still unambiguously ours. The descriptor keeps the inode
159 # allocated for as long as this call runs, so the identity recorded here cannot be
160 # handed to a directory somebody else creates at this path; the unwind below touches
161 # nothing that does not answer with that same identity.
162 with _pinned(path) as pin:
163 # Read through the pin where there is one: the identity must be the inode the pin
164 # holds, not whatever the path denotes by now (round 7 of #1077 — the two can
165 # differ as soon as the path is retargeted, and a path-read identity then names
166 # the stranger). An unlistable directory raises out of here as the I/O failure it
167 # is, unwinding nothing (round 5): "could not look" must not read as "empty".
168 identity = _identity(path, pin)
169 if _has_entries(path, pin):
170 # Between the ``mkdir`` and the pin the path can already have been
171 # retargeted: the any-owner recovery removed our empty directory and somebody
172 # claimed the resource behind it, and what the pin latched is *their*
173 # directory — under the same owner name, as often as not. A directory of ours
174 # is empty at this point, so anything in it is proof the claim is not ours:
175 # report contention and touch nothing. An empty stranger's directory taken in
176 # that same gap cannot be told from ours — the by-name floor the docs state.
177 return ClaimResult(
178 schema_version=SCHEMA_VERSION,
179 resource=resource,
180 owner=clean_owner,
181 path=str(path),
182 granted=False,
183 status="denied",
184 reason="resource-already-claimed",
185 holder=_holder(path),
186 )
187 try:
188 workspace.ensure_runtime_gitignore_for(path)
189 try:
190 _write_owner(path, clean_owner, pin=pin)
191 except FileExistsError:
192 # The owner file is created *exclusively*, through the pin where there
193 # is one. A directory of ours has no owner file at this point, so one
194 # already there means the claim was taken underneath us while the
195 # scaffold ran — released by the any-owner recovery and re-claimed, under
196 # the same name as often as not (round 5 of #1077). Report contention and
197 # unwind nothing: the live claim is theirs. A pinned directory that was
198 # *removed* underneath surfaces as ``FileNotFoundError`` from the same
199 # create, an I/O failure that propagates below; the unwind then finds no
200 # directory and leaves it. The pin is released by the context, once.
201 return ClaimResult(
202 schema_version=SCHEMA_VERSION,
203 resource=resource,
204 owner=clean_owner,
205 path=str(path),
206 granted=False,
207 status="denied",
208 reason="resource-already-claimed",
209 holder=_holder(path),
210 )
211 except BaseException:
212 # Left in place, the half-built claim is worse than the error that caused it
213 # — it is held forever by ``UNKNOWN_HOLDER``, denies every later claim, and
214 # nothing releases it (#1077). The unwind identifies the directory rather
215 # than trusting either the path or the recorded name; see
216 # :func:`_discard_partial_claim`.
217 _discard_partial_claim(path, owner=clean_owner, identity=identity, pin=pin)
218 raise
219 return ClaimResult(
220 schema_version=SCHEMA_VERSION,
221 resource=resource,
222 owner=clean_owner,
223 path=str(path),
224 granted=True,
225 status="granted",
226 reason="claim-acquired",
227 holder=clean_owner,
228 )
231def _release_path(
232 path: Path,
233 *,
234 resource: str,
235 owner: str | None,
236 best_effort: bool = False,
237) -> ClaimResult:
238 clean_owner = _clean(owner, "unknown-owner") if owner is not None else "any-owner"
239 if not path.exists():
240 return ClaimResult(
241 schema_version=SCHEMA_VERSION,
242 resource=resource,
243 owner=clean_owner,
244 path=str(path),
245 granted=False,
246 status="missing",
247 reason="resource-not-claimed",
248 )
249 holder = _holder(path)
250 # An unidentifiable holder refuses a *named* release, exactly as a differently
251 # named one does. Releasing with `owner=None` stays the deliberate any-owner
252 # escape for clearing a stuck claim.
253 if owner is not None and holder != clean_owner:
254 return ClaimResult(
255 schema_version=SCHEMA_VERSION,
256 resource=resource,
257 owner=clean_owner,
258 path=str(path),
259 granted=False,
260 status="not-owner",
261 reason="resource-held-by-different-owner",
262 holder=holder,
263 )
264 try:
265 owner_file = path / "owner.json"
266 if owner_file.exists():
267 owner_file.unlink()
268 path.rmdir()
269 except OSError:
270 if not best_effort:
271 raise
272 return ClaimResult(
273 schema_version=SCHEMA_VERSION,
274 resource=resource,
275 owner=clean_owner,
276 path=str(path),
277 granted=False,
278 status="released",
279 reason="claim-released",
280 holder=holder,
281 )
284def _discard_partial_claim(
285 path: Path,
286 *,
287 owner: str,
288 identity: tuple[int, int, float | None] | None,
289 pin: int | None = None,
290) -> None:
291 """Unwind a claim directory whose initialisation failed — if it is still the one we made.
293 That this call created *a* directory at ``path`` does not prove the path still denotes
294 that directory. The documented any-owner recovery (``keel release <resource>`` with no
295 ``--owner``) can have removed the ownerless half-built claim while this call was
296 stalled, and another owner can have claimed the resource behind it.
298 The recorded **owner name cannot tell those apart**, because an owner is a name, not a
299 claim id: :func:`merge_lock` always claims as ``merge-lock`` and ``keel merge`` derives
300 one name per pull request, so the live claim this unwind must not touch routinely
301 carries the very string this call does. The ownerless reading is no safer — between
302 another caller's ``mkdir`` and its finished ``owner.json`` its live claim reads as
303 :data:`UNKNOWN_HOLDER` too, and so does a torn write, because :func:`_write_owner` is a
304 plain ``write_text``.
306 So the directory is **identified**, not named. ``identity`` is what :func:`_identity`
307 read straight after this call's own ``mkdir`` — ``st_dev``, ``st_ino``, and the birth
308 time where the platform reports one; the path is re-stat'ed here and nothing is removed
309 unless the answer matches. A directory removed and re-created in between is a different
310 inode, so neither a same-named re-claim nor another caller's ownerless window can be
311 deleted. The owner check is kept as a second condition: a directory that still is ours
312 but by now names a *different* owner is left alone as well.
314 What the check proves: the directory being removed is the one this call created, at the
315 same path, not unlinked and re-created since. An inode number on its own would not
316 prove that — a filesystem may reuse one as soon as the directory is unlinked, and on
317 ext4 the next ``mkdir`` in the same parent is a likely taker — which is why
318 :func:`_claim_path` holds the directory open (:func:`_pin`) for the whole of this call:
319 a pinned inode is not recycled, so within that window the pair names one directory and
320 only one.
322 What it does **not** prove: that nothing can change between the last identity read
323 and the ``rmdir`` itself. ``owner.json`` is unlinked through the pinned descriptor, so
324 that removal is bound to our inode; a directory has no by-descriptor removal in POSIX,
325 so the ``rmdir`` is by name and the identity is re-read right before it. A directory
326 swapped in between those two calls survives if it holds anything — ``rmdir`` refuses a
327 non-empty directory — so the residual exposure is another caller's *empty* claim
328 directory created in that same instant, which its own ``_write_owner`` would then
329 fail on and which it would unwind itself. Nor the gap between the ``mkdir`` and the
330 pin: a stranger's directory taken *there* is what the pin latches, so anything already
331 in it is treated as contention and nothing is unwound, while an empty one cannot be
332 told from ours. Nor anything on a platform that cannot pin.
333 Windows answers no
334 descriptor, and there this rests on ``stat`` alone — an NTFS file id whose sequence
335 number is bumped on reuse, plus ``st_birthtime`` on 3.12+. It also does not survive
336 something that removes and re-creates the directory *and* reproduces its recorded
337 identity, which no ordinary filesystem does. Combined with the owner check and the
338 window this runs in — the microseconds between a failed initialisation and its own
339 handler — that is the practical guarantee. This is a single-host primitive, not a
340 distributed lock, and it has never claimed to be one.
342 The removal is best-effort: the caller re-raises the failure that got us here, and that
343 error is the one worth reporting. If the cleanup itself cannot finish — the directory is
344 unwritable, or the failed step left a file behind that ``rmdir`` refuses — masking the
345 original cause with the cleanup's own ``OSError`` would hide *why* the claim failed and
346 leave the operator with the leaked directory either way. Declining to remove is the same
347 outcome: clearing what is left over is the caller-owned stale recovery, as for a claim
348 orphaned by an unmaskable kill.
349 """
350 if identity is None:
351 # The identity was never established (the ``stat`` right after ``mkdir`` failed).
352 # Unable to prove the directory is ours, we leave it: a leaked claim an operator
353 # can release is recoverable, another run's deleted lock is not.
354 return
355 if _holder(path) not in (UNKNOWN_HOLDER, owner):
356 return
357 # Everything that removes is kept as close to the identity check as the platform
358 # allows, and bound to the pinned inode where it can be (round 3 of #1077): the
359 # owner file is unlinked *through the pin* (``dir_fd``), so it is our directory's
360 # ``owner.json`` or nothing, whatever the path denotes by now. ``rmdir`` has no such
361 # form — POSIX removes directories by name, never by descriptor — so the identity is
362 # re-read immediately before it and the remaining window is the two syscalls in
363 # between. A directory swapped in *there* survives if it has anything in it
364 # (``rmdir`` refuses a non-empty directory), so what that window can still take is
365 # another caller's empty, just-``mkdir``-ed claim in the same instant. That is the
366 # honest floor of a by-name primitive, and the docs state it.
367 if _identity(path) != identity:
368 return
369 try:
370 if pin is not None: # pragma: no cover - needs a directory descriptor (POSIX)
371 # Bound to our inode: this is our directory's owner file or nothing.
372 try:
373 os.unlink("owner.json", dir_fd=pin)
374 except FileNotFoundError:
375 pass # the failing step never got as far as writing one
376 # Without a descriptor there is no unlink that is bound to an inode, and a
377 # by-name unlink after a by-name check is the very defect this exists to close —
378 # so the unpinned path removes no file at all. The ``rmdir`` below then refuses a
379 # directory holding our own torn ``owner.json``, which stays as the documented
380 # leak, and equally refuses a stranger's live claim.
381 if _identity(path) != identity:
382 return
383 path.rmdir()
384 except OSError:
385 pass # best effort: the caller re-raises the failure that brought us here
388def _has_entries(path: Path, pin: int | None) -> bool:
389 """True when the claim directory already holds something.
391 Read through the pin where there is one, so the answer is about the inode the
392 unwind would later act on. An unreadable directory raises: the caller treats that
393 as the I/O failure it is, because "could not look" must not be read as "empty" —
394 that reading adopted a stranger's claim as ours (round 5 of #1077).
395 """
396 entries = os.listdir(pin) if pin is not None else os.listdir(path)
397 return bool(entries)
400def _identity(path: Path, pin: int | None = None) -> tuple[int, int, float | None] | None:
401 """Return ``(st_dev, st_ino, birth time)`` for ``path``, or ``None`` if unknowable.
403 ``None`` is the fail-closed answer — the directory is gone, or its metadata cannot be
404 read — and callers treat it as "not the directory I am looking for". With ``pin`` the
405 answer is ``fstat`` of the held descriptor: the inode this call created, whatever the
406 path denotes by the time it is asked. The unwind then compares the *path's* identity
407 against it, so a path retargeted to somebody else's directory never matches.
409 The device and inode pair is the portable half, and :func:`_pin` is what makes it
410 conclusive. ``st_birthtime`` is added where the platform reports one (macOS/APFS, the
411 BSDs, Windows on 3.12+) and is simply absent elsewhere, which stays consistent within a
412 run: both stats of the same path answer the same way, so the comparison never turns on
413 the platform. What is deliberately **not** in the tuple is ``st_ctime``: writing
414 ``owner.json`` into the claim directory changes the directory's own ctime, so including
415 it would report our own initialisation as a foreign directory and refuse to unwind
416 exactly the failure this cleanup exists for.
417 """
418 try:
419 info = os.fstat(pin) if pin is not None else path.stat()
420 except OSError:
421 return None
422 return (info.st_dev, info.st_ino, getattr(info, "st_birthtime", None))
425def _pin(path: Path) -> int | None:
426 """Hold the directory at ``path`` open, or ``None`` where that is not possible.
428 An inode number is only unique while the inode is allocated: once the directory is
429 unlinked the number is free for the next one, and on ext4 the next ``mkdir`` in the
430 same parent is a likely taker. An open descriptor keeps the inode alive — ``rmdir``
431 still succeeds, the number is simply not recycled until the last reference goes — so
432 for as long as this pin is held, ``(st_dev, st_ino)`` names one directory and only one.
434 Windows cannot open a directory this way and answers ``None``; there the identity
435 falls back to what ``stat`` alone reports, which is stronger than POSIX's to begin
436 with, because an NTFS file id folds in a sequence number that is bumped when the record
437 is reused. Nothing here fails on a ``None``: the pin narrows a theoretical window, it
438 is not the guard itself.
439 """
440 try:
441 return os.open(path, os.O_RDONLY)
442 except OSError:
443 return None
446@contextmanager
447def _pinned(path: Path) -> Iterator[int | None]:
448 """Hold ``path`` open for the block (:func:`_pin`), releasing it exactly once.
450 Every exit from the block — granted, denied at the pin, denied at the create, an
451 exception on the way — closes the descriptor here and nowhere else; round 6 of
452 #1077 was a second close on the contention path turning a denied result into
453 ``EBADF``.
454 """
455 pin = _pin(path)
456 try:
457 yield pin
458 finally:
459 if pin is not None: # pragma: no cover - needs a directory descriptor (POSIX)
460 os.close(pin)
463def _write_owner(path: Path, owner: str, *, pin: int | None = None) -> None:
464 """Create ``owner.json`` for a claim — exclusively, and through ``pin`` if given.
466 ``O_EXCL`` is what makes a claim taken underneath us visible: a directory this call
467 made has no owner file, so an existing one is somebody else's live claim and the
468 create fails with ``FileExistsError`` instead of overwriting it. Through the pinned
469 descriptor the create is bound to the inode this call created, so it cannot land in
470 a directory that replaced ours at the same path; if ours was removed underneath, it
471 fails with ``FileNotFoundError`` rather than writing anywhere. Without a descriptor
472 the create is by name but still exclusive (#1077, round 5).
474 The write itself is still not atomic: a caller killed between the create and the
475 write leaves a torn file, which :func:`_holder` reads as :data:`UNKNOWN_HOLDER`.
476 """
477 payload = json.dumps({"owner": owner}, sort_keys=True) + "\n"
478 if pin is not None: # pragma: no cover - needs a directory descriptor (POSIX)
479 fd = os.open("owner.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600, dir_fd=pin)
480 with os.fdopen(fd, "w", encoding="utf-8") as handle:
481 handle.write(payload)
482 return
483 with open(path / "owner.json", "x", encoding="utf-8") as handle:
484 handle.write(payload)
487def _holder(path: Path) -> str:
488 """The named owner of an **existing** claim, or :data:`UNKNOWN_HOLDER`.
490 Every caller reaches here only with the claim directory present, so there is no
491 "unheld" answer to give. Each way of failing to read the name — file missing,
492 corrupt JSON, unreadable, wrong shape — means the resource is held by someone we
493 cannot identify, not that it is free. Collapsing those to ``None`` made the
494 ownership guard *vanish* rather than fail closed, letting a second run release a
495 live merge claim and take the lock (#631).
496 """
497 owner_file = path / "owner.json"
498 if not owner_file.exists():
499 return UNKNOWN_HOLDER
500 try:
501 data = json.loads(owner_file.read_text(encoding="utf-8"))
502 except (OSError, json.JSONDecodeError):
503 return UNKNOWN_HOLDER
504 owner = data.get("owner") if isinstance(data, dict) else None
505 return owner if isinstance(owner, str) and owner.strip() else UNKNOWN_HOLDER
508def _clean(value: str | None, fallback: str) -> str:
509 return value.strip() if isinstance(value, str) and value.strip() else fallback