Coverage for src/keel/workspace.py: 100%
113 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-02 20:26 +0000
1"""keel runtime workspace: the ``.keel`` directory, its gitignore, and scratch.
3Keel writes runtime artifacts (checkpoints, activity records, the run ledger,
4merge locks) and its agentic steps stage transient scratch (PR diffs, issue
5dumps, draft review/closure prose) while driving a workflow. None of this
6belongs in the consumer's primary checkout: it is keel-owned, disposable
7runtime state. This module owns the one place those artifacts live - the
8project's ``.keel`` directory - and keeps them out of ``git status`` by
9scaffolding a ``.keel/.gitignore`` that ignores the runtime subtrees while
10leaving committed config (``project.yaml``) and extensions tracked.
12It also exposes the **scratch directory** (``.keel/scratch``): the sanctioned
13place for any ad-hoc transient file an agent needs to stage. Routing scratch
14here - instead of the repo root - is why a consumer no longer sees
15``plan.json``, ``pr_<n>.diff``, ``issue.md`` and friends accumulate in their
16checkout.
18Finally it owns **reclamation** of the disposable runtime trees so they do not
19grow without bound: :func:`clean_scratch` empties ``.keel/scratch`` and
20:func:`prune_activity` applies count-based retention to ``.keel/activity``.
21Reclamation is deliberately scoped to those two trees - it never touches the
22run ledger, checkpoint, or locks, which have their own bounded lifecycles and
23(for the ledger) are durable by design.
25Pure-core + thin I/O, mirroring :mod:`keel.checkpoint` and :mod:`keel.activity`:
26:func:`runtime_gitignore_body` is a deterministic pure function; only the
27``ensure_*``, :func:`scratch_dir`, and the reclamation helpers touch the
28filesystem.
29"""
31from __future__ import annotations
33import contextlib
34import os
35import shutil
36import tempfile
37from pathlib import Path, PurePosixPath, PureWindowsPath
39KEEL_DIRNAME = ".keel"
40GITIGNORE_NAME = ".gitignore"
41SCRATCH_DIRNAME = "scratch"
43# Runtime subtrees of ``.keel`` that are disposable per-run state, expressed as
44# patterns relative to the ``.keel/.gitignore`` that lists them. ``project.yaml``
45# and ``extensions/`` are intentionally absent: they are committed config.
46#
47# ``/worktrees/`` is where ``keel swarm`` checks out one isolated tree per cluster
48# (``swarm_runtime.build_worktree_path``). A parallel run leaves a full working
49# copy per worker there, so without this entry a swarm turns ``git status`` into
50# hundreds of untracked files — and the operator's habit of trusting a clean
51# status is the thing that notices a real stray file (#877).
52#
53# Anchored with a leading slash, unlike its neighbours. An unanchored
54# ``worktrees/`` matches at any depth, including
55# ``.keel/extensions/<ext>/worktrees/`` — and ``extensions/`` is committed config,
56# deliberately absent from this list. A third-party extension is far likelier to
57# contain a directory called ``worktrees`` than one called ``scratch``, so the
58# looseness that is theoretical for the others is not for this one.
59RUNTIME_IGNORE_ENTRIES: tuple[str, ...] = (
60 "state/",
61 "activity/",
62 "scratch/",
63 "/worktrees/",
64 "*.tmp",
65)
67_GITIGNORE_HEADER = (
68 "# keel runtime artifacts - generated by `keel`; commit this file.",
69 "# These subtrees are disposable per-run state (checkpoints, activity,",
70 "# locks, agent scratch); keeping them ignored stops keel from polluting",
71 "# your checkout. Delete an entry only if you deliberately track that path.",
72)
75def is_root_anchored(raw: str) -> bool:
76 """Whether a configured path escapes ``root`` by being anchored, on any platform.
78 ``Path("/tmp/x").is_absolute()`` is **False** on Windows — there is no drive
79 letter — so a leading slash slipped past every "must be relative to the
80 project root" check there. The path was still caught, one step later, by the
81 escape check, but with a different message and for a different reason: the
82 config was wrong in the same way on every platform, and only one platform
83 said so (#953).
85 A keel config is the same text wherever it is read, so the question is asked
86 of both flavours: ``/tmp/x``, ``C:\\tmp\\x`` and ``\\\\server\\share`` are all
87 anchored no matter which runner reads them.
88 """
89 return PurePosixPath(raw).is_absolute() or PureWindowsPath(raw).is_absolute()
92def keel_dir(root: str | Path = ".") -> Path:
93 """Return the project's ``.keel`` directory under ``root`` (not created)."""
94 return Path(root) / KEEL_DIRNAME
97def runtime_gitignore_body() -> str:
98 """Return the canonical ``.keel/.gitignore`` body (deterministic)."""
99 lines = [*_GITIGNORE_HEADER, *RUNTIME_IGNORE_ENTRIES]
100 return "\n".join(lines) + "\n"
103def ensure_runtime_gitignore(keel_directory: str | Path) -> bool:
104 """Scaffold or top up ``<keel_directory>/.gitignore``; idempotent.
106 Creates the gitignore with the canonical body when absent. When it already
107 exists, appends only the runtime entries that are missing - preserving any
108 operator additions and never rewriting an already-complete file. Returns
109 ``True`` when the file was created or changed, ``False`` when it was already
110 a superset of the runtime entries (or the ``.keel`` directory is absent).
111 """
112 directory = Path(keel_directory)
113 if not directory.is_dir():
114 return False
115 gitignore = directory / GITIGNORE_NAME
116 if gitignore.is_symlink() or directory.is_symlink():
117 # A committed `.keel/.gitignore` symlink — or a symlinked `.keel` above it — would
118 # otherwise have keel's ignore lines written through it to the link's target, anywhere
119 # on disk (#1247).
120 return False
121 if not gitignore.exists():
122 gitignore.write_text(runtime_gitignore_body(), encoding="utf-8")
123 return True
124 existing = gitignore.read_text(encoding="utf-8")
125 present = {line.strip() for line in existing.splitlines()}
126 missing = [entry for entry in RUNTIME_IGNORE_ENTRIES if entry not in present]
127 if not missing:
128 return False
129 prefix = existing if existing.endswith("\n") or existing == "" else existing + "\n"
130 gitignore.write_text(prefix + "\n".join(missing) + "\n", encoding="utf-8")
131 return True
134def ensure_runtime_gitignore_for(artifact_path: str | Path) -> bool:
135 """Self-heal the gitignore for a runtime artifact about to be written.
137 Walks the artifact's ancestors for a ``.keel`` directory and scaffolds its
138 gitignore. A no-op (returns ``False``) when the artifact lives outside any
139 ``.keel`` tree - e.g. an operator pointed an output path elsewhere on
140 purpose, which keel must not silently ignore.
141 """
142 target = Path(artifact_path)
143 for ancestor in target.parents:
144 if ancestor.name == KEEL_DIRNAME and ancestor.is_dir():
145 return ensure_runtime_gitignore(ancestor)
146 return False
149def write_text_atomic(path: str | Path, content: str, *, encoding: str = "utf-8") -> None:
150 """Write ``content`` to ``path`` atomically **and durably**.
152 One writer for every runtime artifact that must survive a crash mid-write:
153 the checkpoint, activity records, and swarm run state. Each had its own copy
154 of the temp-file-and-rename dance, which is how the third one
155 (:func:`keel.swarm.save_swarm_state`) was still a bare ``write_text`` — a
156 torn file on interruption, the exact bug class #872 was filed about (#932).
158 ``os.replace`` makes the *swap* atomic: a reader sees the old file or the new
159 one, never a partial. It does not make the new bytes **durable** — after a
160 power loss the rename can survive with the file's contents still in the page
161 cache. #872's own Impact section named that scenario and only the atomicity
162 half shipped, so the data is fsynced before the rename and the directory
163 entry after it.
165 The directory fsync is best-effort: Windows cannot open a directory with
166 ``os.open``, and there the rename's durability is the filesystem's business
167 rather than something this call can request. Failing the write over it would
168 trade a real guarantee on POSIX for a broken one everywhere.
169 """
170 target = Path(path)
171 target.parent.mkdir(parents=True, exist_ok=True)
172 ensure_runtime_gitignore_for(target)
173 fd, temp_file = tempfile.mkstemp(prefix=f".{target.name}.", dir=target.parent, text=True)
174 try:
175 with os.fdopen(fd, "w", encoding=encoding) as handle:
176 handle.write(content)
177 handle.flush()
178 os.fsync(handle.fileno())
179 os.replace(temp_file, target)
180 except Exception:
181 with contextlib.suppress(OSError):
182 os.unlink(temp_file)
183 raise
184 _fsync_directory(target.parent)
187def _fsync_directory(directory: Path) -> bool:
188 """Flush a directory entry so a completed rename survives a power loss.
190 Best-effort, and its own function so the success path is reachable by a test
191 on every platform: Windows cannot ``os.open`` a directory, so leaving this
192 inline left three lines uncovered there and dropped the Windows coverage run
193 below the 100% bar — a real failure hidden behind #953's masked job.
195 Returns whether the sync happened, so a caller (today only a test) can tell
196 "flushed" from "this platform would not let us ask".
197 """
198 try:
199 dir_fd = os.open(directory, os.O_RDONLY)
200 except OSError:
201 return False
202 try:
203 os.fsync(dir_fd)
204 finally:
205 os.close(dir_fd)
206 return True
209def scratch_dir(root: str | Path = ".", *, create: bool = True) -> Path:
210 """Return ``.keel/scratch`` - the sanctioned home for transient artifacts.
212 With ``create`` (the default) the directory and the runtime gitignore are
213 materialised so callers can write into it immediately and it never surfaces
214 in ``git status``.
215 """
216 directory = keel_dir(root)
217 scratch = directory / SCRATCH_DIRNAME
218 if create:
219 scratch.mkdir(parents=True, exist_ok=True)
220 ensure_runtime_gitignore(directory)
221 return scratch
224def _escapes_root(path: Path, root: str | Path) -> bool:
225 """Does ``path`` resolve outside the project root? (#1247)
227 ``Path.is_symlink`` only inspects the final segment, so a symlinked *parent* — a checkout
228 that committed ``.keel`` itself as a symlink — slips past a leaf check while ``is_dir``
229 still follows it. Resolving the whole path and requiring it under the resolved root catches
230 a symlink anywhere in the chain.
231 """
232 try:
233 path.resolve().relative_to(Path(root).resolve())
234 return False
235 except (ValueError, OSError):
236 return True
239def scratch_entries(root: str | Path = ".") -> list[str]:
240 """Sorted top-level names currently under ``.keel/scratch`` (``[]`` if none)."""
241 scratch = keel_dir(root) / SCRATCH_DIRNAME
242 # A symlinked `.keel/scratch` (or a symlinked `.keel` above it) points its contents
243 # somewhere else on disk; do not list (or, in `clean_scratch`, delete) through it (#1247).
244 if scratch.is_symlink() or _escapes_root(scratch, root) or not scratch.is_dir():
245 return []
246 return sorted(p.name for p in scratch.iterdir())
249def clean_scratch(root: str | Path = ".") -> list[str]:
250 """Empty ``.keel/scratch`` contents; return the entries that were removed.
252 Scratch is transient by definition, so this empties its contents while
253 keeping the directory node intact. A no-op (``[]``) when scratch does not exist.
254 """
255 scratch = keel_dir(root) / SCRATCH_DIRNAME
256 if scratch.is_symlink() or _escapes_root(scratch, root):
257 # Refuse rather than delete through the link's target (#1247) — whether the link is
258 # `.keel/scratch` itself or a symlinked `.keel` above it. gc catches this and reports
259 # it as degraded instead of aborting.
260 raise OSError(f"{scratch} is a symlink or escapes the project root; refusing to clean it")
261 entries = scratch_entries(root)
262 if scratch.is_dir():
263 for child in scratch.iterdir():
264 # A symlinked child is unlinked (the link, not its target); only a real directory
265 # is recursed. `shutil.rmtree` refuses a symlink, and `is_dir()` follows one, so
266 # the symlink test has to come first.
267 if child.is_symlink() or not child.is_dir():
268 child.unlink(missing_ok=True)
269 else:
270 shutil.rmtree(child)
271 return entries
274def activity_prune_plan(activity_dir: str | Path, *, keep_last: int) -> list[str]:
275 """Names of activity records that exceed ``keep_last`` (oldest first removed).
277 Retention is count-based: the newest ``keep_last`` ``.json`` records (by
278 mtime, then name for a stable tiebreak) are kept; the rest are returned as
279 the prune plan. Only ``.json`` files are considered, so a stray file never
280 counts against - or gets caught by - retention. ``keep_last`` must be
281 non-negative. A no-op (``[]``) when the directory is absent or within budget.
282 """
283 if keep_last < 0:
284 raise ValueError("keep_last must be non-negative")
285 directory = Path(activity_dir)
286 if not directory.is_dir():
287 return []
288 records = [p for p in directory.iterdir() if p.is_file() and p.suffix == ".json"]
289 if len(records) <= keep_last:
290 return []
291 ordered = sorted(records, key=lambda p: (p.stat().st_mtime, p.name), reverse=True)
292 return sorted(p.name for p in ordered[keep_last:])
295def prune_activity(activity_dir: str | Path, *, keep_last: int) -> list[str]:
296 """Apply count-based retention to ``activity_dir``; return the names removed.
298 Removes exactly the records named by :func:`activity_prune_plan`. Never
299 touches non-``.json`` entries, and is only ever pointed at the activity
300 directory - the run ledger, checkpoint, and locks live elsewhere and are
301 out of reach by construction.
302 """
303 directory = Path(activity_dir)
304 doomed = activity_prune_plan(directory, keep_last=keep_last)
305 for name in doomed:
306 (directory / name).unlink()
307 return doomed