Coverage for src/keel/swarm_runtime.py: 100%
547 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
1"""Keel Swarm Runtime — Isolated multi-worktree execution & cluster orchestration.
3Thin I/O execution layer for running parallel swarm workers in isolated Git worktrees,
4handling worker state persistence, and managing fail-soft rebalancing across waves.
6A dry run's worker is a ``keel ship --dry-run`` assessment per cluster. A live run's worker
7(#1400) is the cluster's implementer seat, dispatched through :mod:`keel.delegaterun` in
8the cluster's own worktree, followed by a commit, the project's gates, a push and one pull
9request, stamped with the ship-provenance comment a live ``keel ship`` run posts on its own,
10labelled with the seat's attribution, and with the gates it ran recorded in the run ledger
11for the head it pushed (#1420) — every mutation behind the consent scopes the parent
12delegated (:mod:`keel.swarm_worker`, where each of those decisions is made).
13"""
15from __future__ import annotations
17import concurrent.futures
18import contextlib
19import datetime
20import functools
21import hashlib
22import os
23import shutil
24import subprocess # nosec B404
25import sys
26import tempfile
27import threading
28import traceback
29import uuid
30from collections.abc import Callable, Mapping
31from dataclasses import dataclass, field, replace
32from pathlib import Path
33from typing import Any
35from . import delegaterun, doctor, github, ledger, redaction, runner, swarm_worker
36from .delegate import RunPlan
37from .runner import CommandResult
38from .swarm import (
39 SwarmCluster,
40 SwarmPlan,
41 SwarmRunResult,
42 SwarmRunState,
43 SwarmWorkerStatus,
44 load_swarm_state,
45 pull_request_number,
46 rebalance_swarm_plan,
47 save_swarm_state,
48 ship_handoff_args,
49 tail_child_output,
50 update_worker_state,
51 worker_seed,
52)
54SubprocessRunner = Callable[[list[str], Path], CommandResult]
55#: Runs one implementer plan with the given child environment; returns the delegate result.
56Implementer = Callable[[RunPlan, dict[str, str]], dict[str, Any]]
57#: ``(argv, cwd)`` -> the push's result. The worker builds the argv — its hardened ``git
58#: push --no-verify <url> <commit>:<ref>`` (:func:`keel.swarm_worker.keel_git`), to the URL
59#: it read before the implementer ran — and the seat runs it in keel's own process, with the
60#: operator's environment, which is where the credentials are.
61Pusher = Callable[[list[str], Path], CommandResult]
62#: ``(title, body, base, head, cwd)`` -> ``gh pr create``'s result.
63PullRequestOpener = Callable[[str, str, str, str, Path], CommandResult]
64#: ``(owner_repo, number, body, cwd)`` -> the result of posting ``body`` on that pull request.
65CommentPoster = Callable[[str, int, str, Path], CommandResult]
66#: ``(owner_repo, number, labels, cwd)`` -> the result of applying ``labels`` to that pull
67#: request, creating any the repository lacks first (#1420).
68PullRequestLabeler = Callable[[str, int, list[str], Path], CommandResult]
70#: The runner's own wall-clock limit, for the short git commands swarm and canary run
71#: through it (worktree add/remove, checkout, merge). A worker's child ``keel ship`` is
72#: not one of them: it runs the gate suite, and gets the swarm's worker timeout (#1279).
73DEFAULT_RUNNER_TIMEOUT_S = 300
76def default_runner(
77 cmd: list[str],
78 cwd: Path,
79 timeout_s: int = DEFAULT_RUNNER_TIMEOUT_S,
80 env: dict[str, str] | None = None,
81) -> CommandResult:
82 """Run a subprocess command in cwd and return a CommandResult.
84 A command still running after ``timeout_s`` seconds is killed and reported
85 ``timed_out`` with ``code=124``, the gate runner's code for the same case.
86 ``env`` replaces the child's environment when given (``None`` inherits it).
87 """
88 try:
89 proc = subprocess.run( # nosec B603
90 cmd,
91 cwd=cwd,
92 env=env,
93 stdin=subprocess.DEVNULL,
94 stdout=subprocess.PIPE,
95 stderr=subprocess.STDOUT,
96 text=True,
97 timeout=timeout_s,
98 check=False,
99 )
100 return CommandResult(
101 ok=(proc.returncode == 0),
102 code=proc.returncode,
103 output=proc.stdout or "",
104 timed_out=False,
105 stdout=proc.stdout or "",
106 stderr="",
107 )
108 except subprocess.TimeoutExpired as exc:
109 out = (
110 exc.output
111 if isinstance(exc.output, str)
112 else (exc.stdout if isinstance(exc.stdout, str) else "")
113 )
114 return CommandResult(
115 ok=False,
116 code=124,
117 output=out or "",
118 timed_out=True,
119 stdout=out or "",
120 stderr="",
121 )
122 except Exception as exc: # noqa: BLE001
123 return CommandResult(
124 ok=False,
125 code=1,
126 output=str(exc),
127 timed_out=False,
128 stdout="",
129 stderr=str(exc),
130 )
133#: Called with each :data:`keel.swarm_worker.STAGES` entry as a live worker enters it.
134StageReporter = Callable[[str], None]
137def _no_stage(_stage: str) -> None:
138 """The stage reporter of a worker nobody watches."""
141def _now() -> str:
142 """The current UTC time, ISO 8601 — the timestamps a worker record carries."""
143 return datetime.datetime.now(datetime.UTC).isoformat()
146def build_worktree_path(swarm_id: str, cluster_id: str, root: str | Path = ".") -> Path:
147 """Return the isolated worktree filesystem path for a specific swarm cluster worker."""
148 return Path(root) / ".keel" / "worktrees" / swarm_id / cluster_id
151def build_brief_path(swarm_id: str, cluster_id: str, root: str | Path = ".") -> Path:
152 """Where a live worker's implementer brief is written: beside the run's state file,
153 outside the cluster's worktree, so the brief is never committed with the work."""
154 return Path(root) / ".keel" / "state" / "swarm" / swarm_id / f"{cluster_id}.brief.md"
157def build_gh_config_path(swarm_id: str, cluster_id: str, root: str | Path = ".") -> Path:
158 """The ``GH_CONFIG_DIR`` a live worker's implementer runs with: a directory beside the
159 brief, outside the worktree, that holds no ``gh`` login (#1400)."""
160 return Path(root) / ".keel" / "state" / "swarm" / swarm_id / f"{cluster_id}.no-gh-login"
163def cluster_branch(swarm_id: str, cluster_id: str) -> str:
164 """The branch a cluster's worktree is cut on, and the head of its pull request."""
165 return f"swarm/{swarm_id}/{cluster_id}"
168def _default_implement(plan: RunPlan, env: dict[str, str]) -> dict[str, Any]:
169 """``keel delegate run``'s executor, with the worker's scrubbed environment."""
170 return delegaterun.execute(plan, _run=functools.partial(runner.run_argv, env=env))
173def _default_push(argv: list[str], cwd: Path) -> CommandResult:
174 """The worker's push, with the operator's environment (``env`` not passed)."""
175 return runner.run_argv(argv, cwd=str(cwd))
178def _default_open_pr(title: str, body: str, base: str, head: str, cwd: Path) -> CommandResult:
179 return github.open_pr(title, body, base, head, cwd=str(cwd))
182def _default_post_comment(owner_repo: str, number: int, body: str, cwd: Path) -> CommandResult:
183 """The call ``keel post-comment`` makes for a live ship run's provenance, with the
184 operator's environment — so the comment's author is the account keel's ``gh`` is."""
185 return github.post_issue_comment(owner_repo, number, body, cwd=str(cwd))
188def _default_label_pr(owner_repo: str, number: int, labels: list[str], cwd: Path) -> CommandResult:
189 """Apply ``labels`` to the pull request with the operator's environment, creating each
190 one the repository lacks first, as ``keel doctor --fix`` creates them (#1420).
192 The answer is the label post's: a failed listing or create is not, since a sibling
193 worker may have created the same label a moment earlier — the post then still succeeds,
194 and when the label really is missing, the post is what fails and says so.
195 """
196 listed = github.list_labels(repo=owner_repo, cwd=str(cwd))
197 existing = swarm_worker.label_names(listed.stdout or listed.output) if listed.ok else ()
198 for name in doctor.missing_labels(labels, existing):
199 github.create_label(name, repo=owner_repo, cwd=str(cwd))
200 return github.add_issue_labels(owner_repo, number, labels, cwd=str(cwd))
203#: The key a live worker hands its built ``ship_run`` record to the run under (#1420). The
204#: run appends it on the one thread that collects results, then drops the key.
205GATES_RECORD_KEY = "gates_record"
208@dataclass(frozen=True)
209class LiveRun:
210 """What a live run's parent hands its workers: the operator's consent, delegated, and
211 each cluster's planned implementer seat. The five callables are the I/O seams."""
213 consent: swarm_worker.ConsentDelegation
214 #: ``cluster_id`` -> the cluster's implementer :class:`keel.delegate.RunPlan`.
215 dispatches: Mapping[str, RunPlan]
216 #: ``issue`` -> its scope, for the brief and the pull request's title.
217 issue_scopes: Mapping[int, Any] = field(default_factory=dict)
218 #: ``cluster_id`` -> where its implementer seat came from (``assignment`` source).
219 seat_sources: Mapping[str, str] = field(default_factory=dict)
220 remote: str = "origin"
221 #: The run ledger the parent recorded the delegation in (``swarm-run --live`` writes
222 #: its ``delegated`` event before any worker starts); each cluster pull request a
223 #: worker opens is recorded there too (#1400). ``None`` records nothing.
224 ledger_path: Path | None = None
225 #: The project config, for the capture redaction a ledger record passes before it is
226 #: written (:func:`keel.ledger.sanitize_record`); ``None`` applies the default policy.
227 config: Any = None
228 implement: Implementer = _default_implement
229 push: Pusher = _default_push
230 open_pr: PullRequestOpener = _default_open_pr
231 post_comment: CommentPoster = _default_post_comment
232 label_pr: PullRequestLabeler = _default_label_pr
235def create_swarm_worktree(
236 repo_root: Path,
237 worktree_path: Path,
238 branch_name: str,
239 base_branch: str = "main",
240 runner: SubprocessRunner | None = None,
241) -> bool:
242 """Create an isolated git worktree for a swarm cluster worker."""
243 worktree_path.parent.mkdir(parents=True, exist_ok=True)
244 run = runner or default_runner
245 cmd = [
246 "git",
247 "worktree",
248 "add",
249 "-B",
250 branch_name,
251 str(worktree_path),
252 base_branch,
253 ]
254 res = run(cmd, repo_root)
255 return res.ok
258def remove_swarm_worktree(
259 repo_root: Path,
260 worktree_path: Path,
261 runner: SubprocessRunner | None = None,
262) -> bool:
263 """Remove a previously created isolated git worktree; ``True`` only when it is gone.
265 ``git worktree remove --force`` first. Whatever it leaves on disk is removed with
266 ``rmtree``, and when git refused, ``git worktree prune`` then drops the registration of
267 a directory that is now gone — a registration left behind makes the next ``git
268 worktree add`` of the same branch fail. The answer is the directory's own state
269 afterwards: this used to return ``True`` unconditionally, even when nothing was
270 removed (#1278).
271 """
272 run = runner or default_runner
273 res = run(["git", "worktree", "remove", "--force", str(worktree_path)], repo_root)
274 if worktree_path.exists():
275 shutil.rmtree(worktree_path, ignore_errors=True)
276 if not res.ok:
277 run(["git", "worktree", "prune"], repo_root)
278 return not worktree_path.exists()
281class ReviewCheckoutError(RuntimeError):
282 """Raised when an isolated review checkout cannot be created."""
285def review_worktree_path(name: str, root: str | Path = ".", *, run_id: str | None = None) -> Path:
286 """Resolved checkout path under .keel/worktrees/ for an isolated review worktree."""
287 leaf = f"review-{name}-{run_id}" if run_id else f"review-{name}"
288 return Path(root).resolve() / ".keel" / "worktrees" / leaf
291@contextlib.contextmanager
292def review_checkout(
293 head_sha: str,
294 name: str = "review",
295 *,
296 root: str | Path = ".",
297 run_id: str | None = None,
298 runner: SubprocessRunner | None = None,
299):
300 """Context manager providing an isolated detached worktree at ``head_sha`` for a review run.
302 Allocates a unique invocation-owned path (refusing without deleting if an explicit
303 ``run_id`` path already exists), copies ``jury.toml``, ``jury-policy.toml``, and
304 ``.jury`` entries only when untracked at ``head_sha`` and in the working-tree index,
305 and removes only that path on exit.
306 """
307 root_path = Path(root).resolve()
308 token = run_id if run_id is not None else uuid.uuid4().hex[:12]
309 checkout = review_worktree_path(name, root=root_path, run_id=token)
310 try:
311 checkout.parent.mkdir(parents=True, exist_ok=True)
312 exists = checkout.exists()
313 except OSError as exc:
314 raise ReviewCheckoutError(
315 f"could not prepare review checkout at {checkout}: {exc}"
316 ) from exc
317 if exists:
318 raise ReviewCheckoutError(
319 f"review checkout path already exists at {checkout}; refusing to remove active checkout"
320 )
321 run = runner or default_runner
322 added = run(["git", "worktree", "add", "--detach", str(checkout), head_sha], root_path)
323 if not added.ok:
324 why = added.output.strip()[:200] or f"exit {added.code}"
325 raise ReviewCheckoutError(f"could not create review checkout at {checkout}: {why}")
326 try:
327 try:
328 for cfg_name in ("jury.toml", "jury-policy.toml", ".jury"):
329 src_cfg = root_path / cfg_name
330 if src_cfg.is_dir():
331 srcs = [p for p in sorted(src_cfg.rglob("*")) if not p.is_dir()]
332 elif os.path.lexists(src_cfg):
333 srcs = [src_cfg]
334 else:
335 srcs = []
336 for src_file in srcs:
337 rel_file = src_file.relative_to(root_path)
338 rel_posix = rel_file.as_posix()
339 dst_file = checkout / rel_file
340 results = [
341 run(cmd, root_path)
342 for cmd in (
343 ["git", "ls-tree", head_sha, "--", rel_posix],
344 ["git", "ls-files", "--", rel_posix],
345 )
346 ]
347 if not all(res.ok for res in results):
348 # Fail closed: an unanswered tracking query must never let the
349 # working tree overwrite what the reviewed head carries.
350 print(
351 f"warning: not copying {rel_posix} into the review checkout:"
352 " git could not say whether it is tracked",
353 file=sys.stderr,
354 )
355 continue
356 if not any(res.stdout.strip() for res in results):
357 dst_file.parent.mkdir(parents=True, exist_ok=True)
358 shutil.copy2(src_file, dst_file)
359 except OSError as exc:
360 raise ReviewCheckoutError(
361 f"could not copy review configuration into {checkout}: {exc}"
362 ) from exc
363 yield checkout
364 finally:
365 if not remove_swarm_worktree(root_path, checkout, runner=run):
366 print(f"warning: could not remove review checkout at {checkout}", file=sys.stderr)
369def prune_worktrees(repo_root: Path, runner: SubprocessRunner | None = None) -> str:
370 """Run ``git worktree prune``; ``""``, or why it failed (#1278).
372 It drops the registration of every worktree whose directory is gone — git's own
373 ``gc`` does the same after ``gc.worktreePruneExpire`` — and touches no directory, no
374 branch, and no worktree that is locked.
375 """
376 res = (runner or default_runner)(["git", "worktree", "prune"], repo_root)
377 return "" if res.ok else (res.output.strip() or f"exit {res.code}")
380def remove_empty_swarm_dirs(root: Path, swarm_id: str) -> None:
381 """Remove ``.keel/worktrees/<swarm_id>/``, then ``.keel/worktrees/``, when each is empty.
383 ``rmdir`` only — a directory with anything in it (a sibling worker's worktree, one kept
384 for inspection) stays, and so does a symlink (#1278).
385 """
386 base = Path(root) / ".keel" / "worktrees"
387 for directory in (base / swarm_id, base):
388 try:
389 # Refuses a directory with anything in it, a missing one, and a symlink.
390 directory.rmdir()
391 except OSError:
392 continue
395def settle_live_worktree(
396 root: Path,
397 worktree_path: Path,
398 branch: str,
399 disposal: swarm_worker.WorktreeDisposal,
400 runner: SubprocessRunner | None = None,
401) -> dict[str, Any]:
402 """Carry out a :class:`keel.swarm_worker.WorktreeDisposal`, and say what happened.
404 Returns the keys a worker's result gains: ``worktree`` (where it is still on disk, or
405 ``None``), ``worktree_state`` (``removed``, ``kept``, ``remove-failed`` or ``none``),
406 ``branch_deleted`` and ``warnings`` — so a removal that failed is recorded, not
407 swallowed (#1278).
408 """
409 warnings: list[str] = []
410 branch_deleted = False
411 state = "kept" if disposal.worktree == "keep" else "none"
412 if disposal.worktree == "remove":
413 removed = remove_swarm_worktree(root, worktree_path, runner=runner)
414 if not removed:
415 warnings.append(
416 f"the worktree {worktree_path} could not be removed; "
417 "`keel swarm-status --clean` retries it"
418 )
419 elif disposal.delete_branch:
420 deleted = (runner or default_runner)(["git", "branch", "-D", branch], root)
421 branch_deleted = deleted.ok
422 if not deleted.ok:
423 warnings.append(
424 f"the branch {branch} could not be deleted: {deleted.output.strip()}"
425 )
426 state = "removed" if removed else "remove-failed"
427 return {
428 # A kept worktree, or one whose removal failed (`remove_swarm_worktree` answers
429 # from the directory's own state, so it is still there).
430 "worktree": str(worktree_path) if state in ("kept", "remove-failed") else None,
431 "worktree_state": state,
432 "branch_deleted": branch_deleted,
433 "warnings": warnings,
434 }
437def execute_cluster_worker(
438 project_yaml: str,
439 issue: int,
440 root: Path,
441 worktree_dir: Path,
442 *,
443 dry_run: bool = True,
444 role: str = "core",
445 extra_args: list[str] | None = None,
446 runner: SubprocessRunner | None = None,
447 timeout_s: int = DEFAULT_RUNNER_TIMEOUT_S,
448) -> dict[str, Any]:
449 """Execute a single cluster worker pipeline (keel ship) within its worktree.
451 ``timeout_s`` bounds the child when keel runs it (no ``runner`` passed): the child
452 runs the project's gate suite even in a dry run, so the runner's own 300 s cut off
453 a suite the project allows longer (#1279). A passed ``runner`` owns its own bound.
454 """
455 run = runner or functools.partial(default_runner, timeout_s=timeout_s)
456 cmd = [
457 sys.executable,
458 "-m",
459 "keel",
460 "ship",
461 project_yaml,
462 "--root",
463 str(worktree_dir if worktree_dir.exists() else root),
464 "--issue",
465 str(issue),
466 "--json",
467 ]
468 # `keel ship` gates every live-only path on `--live`; leaving out `--dry-run` does
469 # not turn it on, so a live swarm's children ran the dry assessment (#1269).
470 cmd.append("--dry-run" if dry_run else "--live")
471 if extra_args:
472 cmd.extend(extra_args)
474 target_cwd = worktree_dir if worktree_dir.exists() else root
475 res = run(cmd, target_cwd)
476 output = res.stdout or res.output
477 if res.timed_out:
478 # Said last, where the tail the swarm keeps of the output still holds it: a
479 # killed child printed no verdict, and its partial output read like a failure
480 # of the change rather than of the budget (#1279).
481 prefix = f"{output.rstrip()}\n" if output.strip() else ""
482 output = (
483 f"{prefix}keel ship timed out after {timeout_s}s (exit 124) and returned no "
484 "verdict; raise the limit with swarm-run --worker-timeout if the gate suite "
485 "legitimately needs longer."
486 )
488 return {
489 "issue": issue,
490 "role": role,
491 "ok": res.ok,
492 "code": res.code,
493 "timed_out": res.timed_out,
494 "output": output,
495 }
498def _last_line(result: CommandResult) -> str:
499 """The last line a successful command printed; ``""`` for a failed one."""
500 lines = (result.stdout or result.output or "").strip().splitlines() if result.ok else []
501 return lines[-1].strip() if lines else ""
504def execute_live_cluster_worker(
505 cluster: SwarmCluster,
506 *,
507 swarm_id: str,
508 root: Path,
509 worktree_dir: Path,
510 project_yaml: str,
511 base_branch: str,
512 live: LiveRun,
513 runner: SubprocessRunner | None = None,
514 timeout_s: int = DEFAULT_RUNNER_TIMEOUT_S,
515 progress: dict[str, bool] | None = None,
516 on_stage: StageReporter = _no_stage,
517) -> dict[str, Any]:
518 """Implement one cluster live: its implementer seat, a commit, the gates, a push, a PR.
520 ``progress``, when given, is filled in as the worker goes — ``worktree_created`` once
521 its worktree exists, ``implementer_ran`` once the seat is started — so a caller can
522 settle the worktree correctly even when the worker raises part-way (#1278).
524 ``on_stage`` is called with each stage of :data:`keel.swarm_worker.STAGES` as the
525 worker enters it, so ``keel swarm-status`` can show how far a worker has got while it
526 runs (#1280); the stage it ends at is the result's ``stage``.
528 Before its first mutation the worker checks that the scopes the parent delegated to
529 *this* cluster cover every mutation it will make
530 (:func:`keel.swarm_worker.consent_refusal`), and does nothing when they do not. It then
531 stops at the first stage that fails — so a failed implementer or a red gate leaves the
532 branch unpushed and no pull request opened, and the result says which stage stopped it
533 and why. ``timeout_s`` bounds the gate run and the git commands; the implementer runs
534 under its own plan's timeout.
536 The worker's children — the implementer, git, the gates — run without the parent's
537 consent variables or forge tokens (:func:`keel.swarm_worker.worker_env`): consent
538 reaches a worker as the delegation it is handed, never through inheritance. The
539 implementer also runs without the forge (:func:`keel.swarm_worker.implementer_env`): no
540 ``gh`` login, no git credential helper and no network transport, so only keel's own
541 push, pull request, provenance comment and attribution labels — made here, in keel's
542 process, after the implementer has exited — reach the remote.
544 The worktree shares the operator's repository, so the implementer could also write its
545 config and hooks, which keel's own credentialed steps would then run. The worker reads
546 the push URL and the git setup before the implementer runs, stops at ``tamper`` when
547 that setup changed — after the implementer, and again after the gates — and runs its
548 own git steps with no hooks, fsmonitor or signing program
549 (:func:`keel.swarm_worker.keel_git`), pushing to the URL it read rather than a remote's
550 name.
551 """
552 env = swarm_worker.worker_env(os.environ)
553 run = runner or functools.partial(default_runner, timeout_s=timeout_s, env=env)
554 scopes = swarm_worker.worker_scopes(live.consent, cluster.cluster_id)
555 branch = cluster_branch(swarm_id, cluster.cluster_id)
556 plan = live.dispatches[cluster.cluster_id]
557 system = plan.attribution.get("system") or plan.provider
558 record: dict[str, Any] = {
559 "issue": cluster.issues[0] if cluster.issues else 0,
560 "issues": list(cluster.issues),
561 "role": cluster.role,
562 "branch": branch,
563 "scopes": list(scopes),
564 "implementer": system,
565 "commit": None,
566 "pr_url": None,
567 "pushed": False,
568 # Whether the pull request carries the ship-provenance comment that arms keel
569 # merge's evidence gate; an open pull request without it is held at landing.
570 "provenance_posted": False,
571 # Whether it carries the seat's attribution labels, and whether the gates this
572 # worker ran are in the run ledger as a ship run for its head: without either,
573 # keel merge holds it (#1420).
574 "labels_applied": False,
575 "gates_recorded": False,
576 }
577 progress = {} if progress is None else progress
579 def stop(stage: str, reason: str, code: int = 1) -> dict[str, Any]:
580 return {**record, "ok": False, "code": code, "stage": stage, "output": reason}
582 on_stage("consent")
583 if why := swarm_worker.consent_refusal(scopes):
584 return stop("consent", why)
585 # A registration whose directory is gone — a crashed run's, after its directory was
586 # deleted — makes `git worktree add -B` of the same branch fail, so it is dropped
587 # first (#1278). A sibling's worktree still being added is locked, and prune skips it.
588 on_stage("worktree")
589 pruned = prune_worktrees(root, runner=run)
590 created = create_swarm_worktree(root, worktree_dir, branch, base_branch=base_branch, runner=run)
591 if not created or not worktree_dir.exists():
592 # Most often a previous run of this --swarm-id left its worktree or branch behind.
593 prune_note = f" (git worktree prune failed first: {pruned})" if pruned else ""
594 return stop(
595 "worktree",
596 f"failed to create isolated worktree at {worktree_dir}{prune_note}; if a "
597 f"previous run of this swarm left it behind, `keel swarm-status <project.yaml> "
598 f"--swarm-id {swarm_id} --clean` removes it",
599 )
600 progress["worktree_created"] = True
601 # Read before the implementer runs, from configuration it has not touched yet: where
602 # keel will push, the commit the work must descend from, and the git setup keel's own
603 # steps will run under.
604 start = _last_line(run(["git", "rev-parse", "HEAD"], worktree_dir))
605 push_url = _last_line(run(["git", "remote", "get-url", "--push", live.remote], root))
606 if not push_url:
607 return stop("push", f"the push URL of remote {live.remote!r} could not be read")
608 before = _git_snapshot(run, worktree_dir)
609 if before is None or not start:
610 return stop("worktree", f"the git setup of {worktree_dir} could not be read")
612 on_stage("implement")
613 brief = Path(plan.prompt_path)
614 brief.parent.mkdir(parents=True, exist_ok=True)
615 brief.write_text(
616 swarm_worker.render_brief(
617 cluster, live.issue_scopes, swarm_id=swarm_id, branch=branch, base_branch=base_branch
618 ),
619 encoding="utf-8",
620 )
621 gh_config = build_gh_config_path(swarm_id, cluster.cluster_id, root)
622 # From here the worktree holds what the seat did, and a failure keeps it (#1278).
623 progress["implementer_ran"] = True
624 result = live.implement(
625 plan, swarm_worker.implementer_env(os.environ, gh_config_dir=str(gh_config))
626 )
627 if not result.get("ok"):
628 return stop(
629 "implement",
630 f"the implementer {system} failed ({result.get('error_code')}): {result.get('error')}",
631 )
633 on_stage("tamper")
634 if findings := _tampered(run, worktree_dir, before):
635 return stop("tamper", swarm_worker.tamper_reason(findings, during="the implementer ran"))
636 # Every git step from here is keel's own, and runs with no hooks (an empty directory
637 # made only now, so nothing could be planted in it), no fsmonitor and no signing program.
638 with tempfile.TemporaryDirectory(prefix="keel-no-hooks-") as no_hooks:
639 return _commit_gate_push_open(
640 cluster,
641 record,
642 stop,
643 git=functools.partial(swarm_worker.keel_git, hooks_dir=no_hooks),
644 run=run,
645 before=before,
646 start=start,
647 push_url=push_url,
648 swarm_id=swarm_id,
649 worktree_dir=worktree_dir,
650 project_yaml=project_yaml,
651 base_branch=base_branch,
652 branch=branch,
653 plan=plan,
654 live=live,
655 on_stage=on_stage,
656 )
659def _hook_digests(hooks_dir: Path) -> dict[str, str]:
660 """Each file under ``hooks_dir`` -> the digest of what it holds (through a symlink)."""
661 paths = sorted(hooks_dir.rglob("*")) if hooks_dir.is_dir() else []
662 return {
663 path.relative_to(hooks_dir).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
664 for path in paths
665 if path.is_file()
666 }
669def _git_snapshot(run: SubprocessRunner, worktree_dir: Path) -> swarm_worker.GitSnapshot | None:
670 """The worktree's git setup as keel's own steps would meet it, or ``None`` when git
671 could not say. Read with plain ``git`` — ``rev-parse`` and ``config`` run no hook."""
672 where = run(
673 ["git", "rev-parse", "--absolute-git-dir", "--git-common-dir", "--git-path", "hooks"],
674 worktree_dir,
675 )
676 config = run(["git", "config", "--list", "--show-origin", "--show-scope"], worktree_dir)
677 paths = (where.stdout or where.output).splitlines() if where.ok else []
678 if len(paths) != 3 or not config.ok:
679 return None
680 # `--git-common-dir` and `--git-path` may answer relative to the worktree.
681 git_dir, common_dir, hooks_dir = (str(worktree_dir / p) for p in paths)
682 return swarm_worker.GitSnapshot(
683 git_dir=git_dir,
684 common_dir=common_dir,
685 hooks_dir=hooks_dir,
686 config=tuple((config.stdout or config.output).splitlines()),
687 hooks=_hook_digests(Path(hooks_dir)),
688 )
691def _tampered(
692 run: SubprocessRunner, worktree_dir: Path, before: swarm_worker.GitSnapshot
693) -> tuple[str, ...]:
694 """What changed in the worktree's git setup since ``before``; unreadable is a change."""
695 after = _git_snapshot(run, worktree_dir)
696 if after is None:
697 return ("the git setup could no longer be read",)
698 return swarm_worker.tamper_findings(before, after)
701def _commit_gate_push_open(
702 cluster: SwarmCluster,
703 record: dict[str, Any],
704 stop: Callable[..., dict[str, Any]],
705 *,
706 git: Callable[[list[str]], list[str]],
707 run: SubprocessRunner,
708 before: swarm_worker.GitSnapshot,
709 start: str,
710 push_url: str,
711 swarm_id: str,
712 worktree_dir: Path,
713 project_yaml: str,
714 base_branch: str,
715 branch: str,
716 plan: RunPlan,
717 live: LiveRun,
718 on_stage: StageReporter,
719) -> dict[str, Any]:
720 """The worker's steps after the implementer: commit, gates, push, pull request, and the
721 pull request's ship-provenance comment."""
722 system = plan.attribution.get("system") or plan.provider
723 on_stage("commit")
724 status = run(git(["status", "--porcelain"]), worktree_dir)
725 if not status.ok:
726 return stop("commit", f"git status failed in {worktree_dir}: {status.output.strip()}")
727 if status.output.strip():
728 message = swarm_worker.commit_message(cluster, swarm_id=swarm_id, plan=plan)
729 for args in (["add", "-A"], ["commit", "--no-verify", "-m", message]):
730 step = run(git(args), worktree_dir)
731 if not step.ok:
732 return stop("commit", f"git {args[0]} failed: {step.output.strip()}")
733 head = _last_line(run(git(["rev-parse", "HEAD"]), worktree_dir))
734 if not head or head == start:
735 return stop(
736 "implement",
737 f"the implementer {system} changed nothing in the worktree, so there is no work "
738 "to commit or push",
739 )
740 record["commit"] = head
741 # A seat may commit its own work, but not replace the branch's history: what keel
742 # pushes must descend from the commit the worktree was cut at.
743 if not run(git(["merge-base", "--is-ancestor", start, head]), worktree_dir).ok:
744 return stop(
745 "tamper",
746 f"{head} does not descend from {start}, the commit the worktree was cut at; the "
747 "branch's history was replaced, so nothing is pushed",
748 )
750 on_stage("gates")
751 gates = run(
752 [
753 sys.executable,
754 "-m",
755 "keel",
756 "run-gates",
757 project_yaml,
758 "--root",
759 str(worktree_dir),
760 "--phases",
761 swarm_worker.GATE_PHASES,
762 "--defer-jury",
763 # The per-gate outcome, which the gates-pass record is written from (#1420):
764 # the exit code alone cannot say that a blocking gate did not run.
765 "--json",
766 ],
767 worktree_dir,
768 )
769 printed = gates.stdout or gates.output
770 outcomes = swarm_worker.parse_gate_report(printed)
771 if not gates.ok:
772 detail = printed.strip() if outcomes is None else swarm_worker.gate_report_summary(outcomes)
773 return stop(
774 "gates",
775 f"the project's gates failed at {head}; the branch is not pushed:\n{detail}",
776 code=gates.code,
777 )
779 # The gates ran the implementer's code, which could have written the git setup too.
780 if findings := _tampered(run, worktree_dir, before):
781 return stop("tamper", swarm_worker.tamper_reason(findings, during="the gates ran"))
782 # What the pushed head changes against the commit the worktree was cut at, for the
783 # gates-pass record; `None` (unreadable), never `[]`, when git cannot say.
784 diff = run(git(["diff", "--no-ext-diff", "--name-only", start, head, "--"]), worktree_dir)
785 listed = (diff.stdout or diff.output).splitlines() if diff.ok else None
786 changed = None if listed is None else [path for path in listed if path.strip()]
788 # To the URL read before the implementer ran, never to the remote's name.
789 on_stage("push")
790 pushed = live.push(
791 git(["push", "--no-verify", push_url, f"{head}:refs/heads/{branch}"]), worktree_dir
792 )
793 if not pushed.ok:
794 return stop("push", f"git push {live.remote} {branch} failed: {pushed.output.strip()}")
795 record["pushed"] = True
797 on_stage("pull_request")
798 opened = live.open_pr(
799 swarm_worker.pull_request_title(cluster, live.issue_scopes, swarm_id=swarm_id),
800 swarm_worker.pull_request_body(
801 cluster,
802 swarm_id=swarm_id,
803 branch=branch,
804 base_branch=base_branch,
805 commit=head,
806 plan=plan,
807 seat_source=live.seat_sources.get(cluster.cluster_id),
808 delegation=live.consent,
809 ),
810 base_branch,
811 branch,
812 worktree_dir,
813 )
814 if not opened.ok:
815 return stop(
816 "pull_request",
817 f"{branch} is pushed, but gh pr create failed: {opened.output.strip()}",
818 )
819 url = _last_line(opened)
820 # `/keel:ship` applies the seat's attribution labels to every pull request it opens,
821 # and keel merge holds one without its `agent:<vendor>` label (#1420).
822 unlabelled = _apply_labels(live, url, swarm_worker.attribution_labels(plan), worktree_dir)
823 # A keel-made pull request arms keel merge's evidence gate at creation, as a live
824 # `keel ship` run's does: its branch matches no ship-branch pattern, and without the
825 # stamp `swarm-land` is held on "evidence gate is not enforced" (found on the first
826 # end-to-end live run). Posted by keel, with the operator's credentials, after the
827 # implementer exited — like the push and the pull request.
828 missing = _post_provenance(
829 live,
830 url,
831 swarm_worker.ship_provenance_body(cluster, swarm_id=swarm_id, commit=head, plan=plan),
832 worktree_dir,
833 )
834 gates_record, gates_note = _gates_record(
835 cluster,
836 live=live,
837 plan=plan,
838 swarm_id=swarm_id,
839 branch=branch,
840 base_branch=base_branch,
841 head=head,
842 url=url,
843 outcomes=outcomes,
844 changed=changed,
845 )
846 done = {
847 **record,
848 "ok": True,
849 "code": 0,
850 "stage": "done",
851 "pr_url": url,
852 "provenance_posted": not missing,
853 "labels_applied": not unlabelled,
854 "output": f"pull request opened: {url}",
855 }
856 if gates_record is not None:
857 # Appended by the run, on the thread that collects results (`record_worker_gates`).
858 done[GATES_RECORD_KEY] = gates_record
859 # Not a failed worker: the pull request is open and stays open. It is held at landing
860 # until what is missing is posted, so the run says what and how.
861 notes = [note for note in (unlabelled, missing, gates_note) if note]
862 if notes:
863 done["output"] += "".join(f"\n{note}" for note in notes)
864 done["warnings"] = notes
865 return done
868def _apply_labels(live: LiveRun, url: str, labels: tuple[str, ...], cwd: Path) -> str:
869 """Apply the seat's attribution ``labels`` to the pull request at ``url`` (#1420);
870 ``""`` once applied, else the warning."""
871 owner_repo = swarm_worker.pull_request_repo(url)
872 number = pull_request_number(url)
873 if not labels:
874 why = "keel's attribution for the implementer seat names no agent label"
875 elif owner_repo is None or number is None:
876 why = "gh pr create printed no pull request URL keel can read"
877 else:
878 applied = live.label_pr(owner_repo, number, list(labels), cwd)
879 if applied.ok:
880 return ""
881 why = f"the label post failed: {applied.output.strip() or 'no output'}"
882 return swarm_worker.labels_warning(url or "the pull request", labels, why)
885def _gates_record(
886 cluster: SwarmCluster,
887 *,
888 live: LiveRun,
889 plan: RunPlan,
890 swarm_id: str,
891 branch: str,
892 base_branch: str,
893 head: str,
894 url: str,
895 outcomes: tuple[Any, ...] | None,
896 changed: list[str] | None,
897) -> tuple[dict[str, Any] | None, str]:
898 """The worker's gates-pass record, redacted as every ledger record is, and the
899 warning the run gives about it: ``(record, "")``, ``(record, why it is no pass)`` or
900 ``(None, why there is none)`` (#1420)."""
901 number = pull_request_number(url)
902 shown = url or "the pull request"
903 if number is None:
904 why = "gh pr create printed no pull request number keel can read"
905 return None, swarm_worker.gates_record_warning(shown, why)
906 if outcomes is None:
907 why = "keel run-gates printed no report keel can read, so what each gate did is unknown"
908 return None, swarm_worker.gates_record_warning(shown, why)
909 built = swarm_worker.gates_record(
910 cluster,
911 swarm_id=swarm_id,
912 plan=plan,
913 branch=branch,
914 base_branch=base_branch,
915 head=head,
916 pull_request=number,
917 outcomes=outcomes,
918 changed_files=changed,
919 config=live.config,
920 )
921 try:
922 gates_record = ledger.sanitize_record(built, live.config)
923 except redaction.RedactionError as exc:
924 why = f"the capture redaction policy is invalid: {exc}"
925 return None, swarm_worker.gates_record_warning(shown, why)
926 failed = swarm_worker.gates_not_a_pass(gates_record)
927 return gates_record, swarm_worker.gates_not_a_pass_warning(shown, failed) if failed else ""
930def record_worker_gates(live: LiveRun, outcome: Mapping[str, Any]) -> dict[str, Any]:
931 """Append the gates-pass record a live worker built to the run ledger (#1420).
933 Returns the worker's result without the record, saying ``gates_recorded``, and with a
934 warning when the append failed — never a failed worker: the pull request is open either
935 way. Appended here, on the run's one collecting thread, as the consent records are, so
936 no two workers write to the ledger at once. The ledger is the one ``keel merge --root
937 <root>`` and ``swarm-land`` read, so :func:`keel.ledger.gates_pass_for_head` finds the
938 record for the pull request's head. A run that records nothing (no ledger) appends
939 nothing.
940 """
941 result = {key: value for key, value in outcome.items() if key != GATES_RECORD_KEY}
942 gates_record = outcome.get(GATES_RECORD_KEY)
943 if live.ledger_path is None or gates_record is None:
944 return result
945 why = ""
946 try:
947 ledger.append_record(live.ledger_path, gates_record)
948 except (OSError, ledger.LedgerError) as exc:
949 why = f"{type(exc).__name__}: {exc}"
950 if not why:
951 return {**result, "gates_recorded": True}
952 shown = str(outcome.get("pr_url") or "its pull request")
953 warning = swarm_worker.gates_record_warning(shown, why)
954 return {
955 **result,
956 "output": f"{result.get('output', '')}\n{warning}",
957 "warnings": [*result.get("warnings", ()), warning],
958 }
961def _post_provenance(live: LiveRun, url: str, body: str, cwd: Path) -> str:
962 """Post ``body`` on the pull request at ``url``; ``""`` once posted, else the warning."""
963 owner_repo = swarm_worker.pull_request_repo(url)
964 number = pull_request_number(url)
965 if owner_repo is None or number is None:
966 why = "gh pr create printed no pull request URL keel can read"
967 else:
968 posted = live.post_comment(owner_repo, number, body, cwd)
969 if posted.ok:
970 return ""
971 why = f"the comment post failed: {posted.output.strip() or 'no output'}"
972 return swarm_worker.provenance_warning(url or "the pull request", why)
975def record_pull_request_consent(live: LiveRun, cluster_id: str, outcome: Mapping[str, Any]) -> str:
976 """Append the ``pull_request`` delegation event for a worker's open pull request (#1400).
978 Returns ``""`` once it is in the run ledger (or when the run records nothing), else a
979 warning. Never a failed worker: the pull request is open either way. Without this
980 event ``keel consent-verify`` reports no delegated consent for that pull request — it
981 matches a delegation only by the pull request's number and pushed head, never by its
982 branch name (a fork can name a branch anything).
983 """
984 if live.ledger_path is None:
985 return ""
986 url = str(outcome.get("pr_url") or "")
987 number = pull_request_number(url)
988 why = "gh pr create printed no pull request number keel can read"
989 if number is not None:
990 why = ""
991 try:
992 ledger.append_record(
993 live.ledger_path,
994 ledger.build_consent_delegation_record(
995 live.consent.to_dict(),
996 recorded_at=_now(),
997 pull_request={
998 "cluster": cluster_id,
999 "number": number,
1000 "branch": outcome.get("branch"),
1001 "head_sha": outcome.get("commit"),
1002 "url": url,
1003 },
1004 ),
1005 )
1006 except (OSError, ledger.LedgerError) as exc:
1007 why = f"{type(exc).__name__}: {exc}"
1008 if not why:
1009 return ""
1010 return (
1011 f"the consent delegation for {url or 'its pull request'} is not in the run ledger "
1012 f"({why}); keel consent-verify will report no delegated consent for it — record the "
1013 "consent by hand or re-run the cluster"
1014 )
1017def _swarm_ids_under(base: Path, path: str) -> tuple[str, str] | None:
1018 """``(swarm_id, cluster_id)`` of a worktree path exactly two levels under ``base``."""
1019 try:
1020 parts = Path(path).resolve().relative_to(base).parts
1021 except ValueError:
1022 return None
1023 return (parts[0], parts[1]) if len(parts) == 2 else None
1026def _run_record(swarm_id: str, root: Path) -> swarm_worker.SwarmRunRecord | None:
1027 """What ``swarm_id``'s state file says about its leftovers; ``None`` with no file."""
1028 state = load_swarm_state(swarm_id, root=root)
1029 if state is None:
1030 exists = (root / ".keel" / "state" / "swarm" / f"{swarm_id}.json").exists()
1031 # A state file keel cannot read may belong to a running run.
1032 return swarm_worker.SwarmRunRecord(unfinished=True) if exists else None
1033 return swarm_worker.SwarmRunRecord(
1034 unfinished=state.completed_at is None,
1035 pull_requests={
1036 w.cluster_id: w.pull_request for w in state.workers if w.pull_request is not None
1037 },
1038 pushed=frozenset(w.cluster_id for w in state.workers if w.pushed),
1039 )
1042def find_swarm_leftovers(
1043 root: str | Path,
1044 *,
1045 swarm_id: str | None = None,
1046 runner: SubprocessRunner | None = None,
1047) -> tuple[tuple[swarm_worker.SwarmLeftover, ...], str]:
1048 """Every worktree, directory and branch a swarm run left behind, and what to do (#1278).
1050 Looks only under keel's own paths — worktrees registered, or directories present,
1051 exactly at ``.keel/worktrees/<swarm_id>/<cluster_id>`` — and its own branch namespace,
1052 ``swarm/<swarm_id>/<cluster_id>``; ``swarm_id`` narrows it to one run.
1053 :func:`keel.swarm_worker.classify_leftovers` decides what may be removed. Returns
1054 ``(leftovers, "")``, or ``((), why)`` when git could not list them.
1055 """
1056 root_path = Path(root).resolve()
1057 run = runner or default_runner
1058 base = root_path / ".keel" / "worktrees"
1059 listed = run(["git", "worktree", "list", "--porcelain"], root_path)
1060 refs = run(["git", "for-each-ref", "--format=%(refname)", "refs/heads/swarm/"], root_path)
1061 if not listed.ok or not refs.ok:
1062 failed = listed if not listed.ok else refs
1063 return (), f"git could not list the worktrees or branches: {failed.output.strip()}"
1065 worktrees: list[tuple[str, str, str, bool]] = []
1066 entries = swarm_worker.parse_worktree_list(listed.stdout or listed.output)
1067 for entry in entries:
1068 ids = _swarm_ids_under(base, entry.path)
1069 if ids is not None:
1070 # In the platform's own spelling: git prints `C:/…` on Windows.
1071 worktrees.append((*ids, str(Path(entry.path).resolve()), entry.prunable))
1072 # Every worktree registered under `.keel/worktrees/`, of keel's shape or not: a
1073 # directory that is one, holds one or sits inside one is never taken for a leftover
1074 # directory. (The main worktree holds all of it, so only those below the base count.)
1075 resolved = (Path(entry.path).resolve() for entry in entries)
1076 registered = [p for p in resolved if base in p.parents]
1078 def unregistered(path: Path) -> bool:
1079 return not any(p == path or p in path.parents or path in p.parents for p in registered)
1081 directories: list[tuple[str, str, str]] = []
1082 runs_dirs = sorted(base.iterdir()) if base.is_dir() and not base.is_symlink() else []
1083 for run_dir in runs_dirs:
1084 if run_dir.is_symlink() or not run_dir.is_dir():
1085 continue
1086 children = sorted(run_dir.iterdir())
1087 if not children:
1088 directories.append((run_dir.name, "", str(run_dir)))
1089 for child in children:
1090 if child.is_dir() and not child.is_symlink() and unregistered(child):
1091 directories.append((run_dir.name, child.name, str(child)))
1093 branches: dict[str, str] = {} # branch -> its run
1094 for line in (refs.stdout or refs.output).splitlines():
1095 ids = swarm_worker.swarm_branch_ids(line.strip())
1096 if ids is not None:
1097 branches[line.strip()] = ids[0]
1098 if swarm_id is not None:
1099 worktrees = [w for w in worktrees if w[0] == swarm_id]
1100 directories = [d for d in directories if d[0] == swarm_id]
1101 branches = {b: sid for b, sid in branches.items() if sid == swarm_id}
1102 seen = {w[0] for w in worktrees} | {d[0] for d in directories} | set(branches.values())
1103 runs = {sid: record for sid in sorted(seen) if (record := _run_record(sid, root_path))}
1104 leftovers = swarm_worker.classify_leftovers(
1105 worktrees=worktrees,
1106 directories=directories,
1107 branches=branches,
1108 runs=runs,
1109 named=swarm_id,
1110 )
1111 return leftovers, ""
1114def clean_swarm_leftovers(
1115 root: str | Path,
1116 leftovers: tuple[swarm_worker.SwarmLeftover, ...],
1117 *,
1118 runner: SubprocessRunner | None = None,
1119) -> tuple[list[swarm_worker.SwarmLeftover], list[tuple[swarm_worker.SwarmLeftover, str]]]:
1120 """Remove every leftover marked ``remove``; ``(removed, [(failed, why)])`` (#1278).
1122 Registrations go with ``git worktree prune``, worktrees with
1123 :func:`remove_swarm_worktree`, unregistered directories with ``rmtree`` — before the
1124 branches, which git will not delete while a worktree has them checked out — and each
1125 run directory left empty with ``rmdir``. Nothing marked ``keep`` is touched.
1126 """
1127 root_path = Path(root).resolve()
1128 run = runner or default_runner
1129 todo = [x for x in leftovers if x.action == "remove"]
1130 removed: list[swarm_worker.SwarmLeftover] = []
1131 failed: list[tuple[swarm_worker.SwarmLeftover, str]] = []
1133 def settle(item: swarm_worker.SwarmLeftover, gone: bool, why: str) -> None:
1134 if gone:
1135 removed.append(item)
1136 else:
1137 failed.append((item, why))
1139 registrations = [x for x in todo if x.kind == "registration"]
1140 if registrations:
1141 why = prune_worktrees(root_path, runner=run)
1142 for item in registrations:
1143 settle(item, not why, f"git worktree prune failed: {why}")
1144 for item in todo:
1145 if item.kind == "worktree":
1146 gone = remove_swarm_worktree(root_path, Path(item.target), runner=run)
1147 settle(item, gone, "the directory is still there")
1148 elif item.kind == "directory" and item.cluster_id:
1149 shutil.rmtree(item.target, ignore_errors=True)
1150 settle(item, not Path(item.target).exists(), "the directory is still there")
1151 for item in todo:
1152 if item.kind == "branch":
1153 deleted = run(["git", "branch", "-D", item.target], root_path)
1154 settle(item, deleted.ok, deleted.output.strip() or f"exit {deleted.code}")
1155 for swarm_id in sorted({x.swarm_id for x in todo}):
1156 remove_empty_swarm_dirs(root_path, swarm_id)
1157 for item in todo:
1158 if item.kind == "directory" and not item.cluster_id:
1159 settle(item, not Path(item.target).exists(), "the directory is not empty")
1160 return removed, failed
1163def _disposal(ok: bool, progress: Mapping[str, bool]) -> swarm_worker.WorktreeDisposal:
1164 return swarm_worker.worktree_disposal(
1165 ok=ok,
1166 worktree_created=progress.get("worktree_created", False),
1167 implementer_ran=progress.get("implementer_ran", False),
1168 )
1171def _with_settlement(
1172 outcome: dict[str, Any], settled: dict[str, Any], branch: str
1173) -> dict[str, Any]:
1174 """A live worker's result with what became of its worktree; a kept worktree is named
1175 at the end of ``output``, where the tail the swarm keeps still holds it (#1278)."""
1176 output = outcome.get("output", "")
1177 if settled["worktree_state"] == "kept":
1178 output = (
1179 f"{output}\nthe worktree is kept for inspection at {settled['worktree']} (branch "
1180 f"{branch}); `keel swarm-status <project.yaml> --clean` removes it"
1181 )
1182 # The worker's own warnings (an unstamped pull request) come first, then the settle's:
1183 # a plain merge would let the settle's list replace the worker's.
1184 warnings = [*outcome.get("warnings", ()), *settled["warnings"]]
1185 return {**outcome, **settled, "output": output, "warnings": warnings}
1188def run_swarm_orchestration(
1189 plan: SwarmPlan,
1190 project_yaml: str,
1191 *,
1192 root: str | Path = ".",
1193 dry_run: bool = True,
1194 max_workers: int = 4,
1195 runner: SubprocessRunner | None = None,
1196 base_branch: str,
1197 timeout_s: int = DEFAULT_RUNNER_TIMEOUT_S,
1198 live: LiveRun | None = None,
1199) -> SwarmRunResult:
1200 """Execute the waves and clusters of a SwarmPlan with fail-soft isolation.
1202 ``base_branch`` is required, as it is for ``land_wave_clusters``: the worktrees
1203 are branched from it and the wave is later landed onto it, and a default of
1204 ``main`` branched a ``develop`` project's clusters off the wrong history (#1262).
1206 ``timeout_s`` is each worker's gate budget — the child ``keel ship`` of a dry run, the
1207 ``keel run-gates`` of a live one; ``swarm-run`` passes
1208 :func:`keel.swarm.worker_timeout_s` or its ``--worker-timeout`` (#1279).
1210 A live run (``dry_run=False``) needs ``live``: the operator's consent as the parent
1211 delegated it, and each cluster's planned implementer seat (#1400). Without it nothing
1212 starts — there is no live worker that runs without delegated consent. A live worker
1213 always gets its own worktree, and a dry run's never does (#1288); the
1214 ``create_worktrees`` switch that used to sit beside ``dry_run`` could only ever agree
1215 with it, and is gone (#1280).
1217 ``runner`` is an injection seam for tests, like the ``_run`` seams elsewhere in keel:
1218 ``swarm-run`` never passes it. It replaces the subprocess runner of every git command,
1219 gate run and dry-run child ``keel ship`` a worker makes; left out, each runs through
1220 :func:`default_runner` under ``timeout_s``.
1222 Each worker's record carries its wave, when it started and ended, and — a live worker
1223 — the stage it is in, written as it enters it, so ``keel swarm-status`` shows how far a
1224 run has got while it is in flight (#1280). A worker is ``running`` from the moment it
1225 starts, not from the moment its wave does: a dry run's workers take turns.
1226 """
1227 if not dry_run and live is None:
1228 raise ValueError("a live swarm run needs the operator's delegated consent (#1400)")
1229 root_path = Path(root).resolve()
1230 workers_list: list[SwarmWorkerStatus] = []
1232 # Initialize workers from the plan's own staffing, so the board reports the team the
1233 # planner resolved rather than the record's placeholder defaults (#1017). A live
1234 # worker's record also says which consent scopes it was handed (#1400).
1235 for w in plan.waves:
1236 for c in w.clusters:
1237 seed = worker_seed(c, wave=w.wave_index, updated_at=_now())
1238 if live is not None:
1239 seed = replace(seed, scopes=swarm_worker.worker_scopes(live.consent, c.cluster_id))
1240 workers_list.append(seed)
1242 state = SwarmRunState(
1243 swarm_id=plan.swarm_id,
1244 total_workers=len(workers_list),
1245 active_wave=1,
1246 workers=tuple(workers_list),
1247 started_at=_now(),
1248 consent=None if live is None else live.consent.to_dict(),
1249 )
1250 save_swarm_state(state, root=root_path)
1252 # Live workers report from their own threads while the loop below records the ones that
1253 # finished, so every update of the run's state — and its save — is made under one lock.
1254 state_lock = threading.Lock()
1256 def report(cluster_id: str, **fields: Any) -> None:
1257 nonlocal state
1258 with state_lock:
1259 state = update_worker_state(state, cluster_id, **fields)
1260 save_swarm_state(state, root=root_path)
1262 def _enter_stage(cluster_id: str, stage: str) -> None:
1263 report(cluster_id, stage=stage)
1265 passed_count = 0
1266 failed_count = 0
1267 wave_results: list[dict[str, Any]] = []
1268 warnings: list[str] = []
1269 current_plan = plan
1271 # Waves are followed by their index, not by position: a failure makes
1272 # rebalance_swarm_plan drop a wave, and a position counter over the shrunken
1273 # plan then stepped past the next, unrelated wave without running it (#1268).
1274 last_wave: int | None = None
1275 running = "implementing the cluster" if live is not None else "executing ship pipeline"
1276 while True:
1277 remaining = [w for w in current_plan.waves if last_wave is None or w.wave_index > last_wave]
1278 if not remaining:
1279 break
1280 wave = remaining[0]
1281 last_wave = wave.wave_index
1282 # No worker thread runs between waves, so this update needs no lock.
1283 state = replace(state, active_wave=wave.wave_index)
1285 cluster_tasks = list(wave.clusters)
1286 if not cluster_tasks:
1287 continue
1288 save_swarm_state(state, root=root_path)
1290 wave_record: dict[str, Any] = {
1291 "wave_index": wave.wave_index,
1292 "mode": wave.mode,
1293 "eligible_direct_landing": wave.eligible_direct_landing,
1294 "cluster_results": {},
1295 }
1297 def _worker_fn(cluster: Any) -> tuple[str, dict[str, Any]]:
1298 c_id = cluster.cluster_id
1299 wt_path = build_worktree_path(plan.swarm_id, c_id, root=root_path)
1300 # Running from when this worker starts, not when its wave did: a dry run's
1301 # workers run one at a time, and the rest are still queued (#1280).
1302 report(c_id, step="s4", status="running", details=running, started_at=_now())
1304 # A live worker implements its cluster in a worktree of its own, cut here and
1305 # settled here, whichever way the worker ends — a returned failure or a raise
1306 # (#1400, #1278): see `swarm_worker.worktree_disposal` for what stays.
1307 if not dry_run:
1308 branch = cluster_branch(plan.swarm_id, c_id)
1309 progress: dict[str, bool] = {}
1310 try:
1311 outcome = execute_live_cluster_worker(
1312 cluster,
1313 swarm_id=plan.swarm_id,
1314 root=root_path,
1315 worktree_dir=wt_path,
1316 project_yaml=project_yaml,
1317 base_branch=base_branch,
1318 live=live,
1319 runner=runner,
1320 timeout_s=timeout_s,
1321 progress=progress,
1322 on_stage=functools.partial(_enter_stage, c_id),
1323 )
1324 except BaseException:
1325 # What the seat touched is kept for inspection; nothing is lost to
1326 # a crash in keel's own code.
1327 settle_live_worktree(
1328 root_path, wt_path, branch, _disposal(False, progress), runner
1329 )
1330 raise
1331 else:
1332 settled = settle_live_worktree(
1333 root_path,
1334 wt_path,
1335 branch,
1336 _disposal(bool(outcome.get("ok")), progress),
1337 runner,
1338 )
1339 return c_id, _with_settlement(outcome, settled, branch)
1341 # A dry run's worker is an assessment in the operator's checkout.
1342 return c_id, execute_cluster_worker(
1343 project_yaml=project_yaml,
1344 issue=cluster.issues[0] if cluster.issues else 0,
1345 root=root_path,
1346 worktree_dir=wt_path,
1347 dry_run=True,
1348 role=cluster.role,
1349 # The lead hands its cluster's team to the child ship. Without this the
1350 # child re-resolved from config alone and dropped both the difficulty
1351 # bench and the operator's per-run overrides.
1352 extra_args=list(ship_handoff_args(cluster.assignment)),
1353 runner=runner,
1354 timeout_s=timeout_s,
1355 )
1357 # Run wave clusters in parallel thread pool
1358 # Without a worktree each child runs in the operator's own checkout (a dry run
1359 # creates none), and its gate suite is not written to share a tree with a
1360 # sibling's: `.coverage`, `.pytest_cache`, build output. Such children run one
1361 # at a time; only isolated workers run in parallel (#1288).
1362 pool_workers = min(max_workers, len(cluster_tasks)) if not dry_run else 1
1363 with concurrent.futures.ThreadPoolExecutor(max_workers=pool_workers) as executor:
1364 future_to_cluster = {
1365 executor.submit(_worker_fn, cluster): cluster for cluster in cluster_tasks
1366 }
1367 for future in concurrent.futures.as_completed(future_to_cluster):
1368 try:
1369 c_id, worker_res = future.result()
1370 except Exception as exc: # noqa: BLE001 - one worker must not end the run
1371 # A worker that raised (a malformed assignment, an OSError making
1372 # its path) is a failed cluster, not a failed run. Left unguarded,
1373 # the raise discarded the other workers' results and froze them
1374 # as `running` in the state file (#1271).
1375 cluster = future_to_cluster[future]
1376 c_id = cluster.cluster_id
1377 # The traceback goes to stderr: the failure may be keel's own bug,
1378 # and the one-line reason alone would hide where it came from.
1379 sys.stderr.write(
1380 f"swarm worker {c_id} raised:\n" + "".join(traceback.format_exception(exc))
1381 )
1382 worker_res = {
1383 "issue": cluster.issues[0] if cluster.issues else 0,
1384 "role": cluster.role,
1385 "ok": False,
1386 "code": 1,
1387 "output": f"worker raised {type(exc).__name__}: {exc}",
1388 }
1389 # A raising live worker keeps what its seat touched (#1278).
1390 kept = build_worktree_path(plan.swarm_id, c_id, root=root_path)
1391 if not dry_run and kept.exists():
1392 worker_res["worktree"] = str(kept)
1393 worker_res["output"] += (
1394 f"\nthe worktree is kept for inspection at {kept}; "
1395 "`keel swarm-status <project.yaml> --clean` removes it"
1396 )
1397 # The gates-pass a live worker built goes into the run ledger here, on this
1398 # one thread, before its result is kept anywhere (#1420).
1399 if live is not None and worker_res.get("ok", False):
1400 worker_res = record_worker_gates(live, worker_res)
1401 # Bounded once, here, where every origin of `output` — the child's
1402 # stdout, a worktree failure, a raised worker — meets both places it is
1403 # kept: this wave record and, on failure, the state file's `details`
1404 # (#1280).
1405 worker_res = {**worker_res, "output": tail_child_output(worker_res["output"])}
1406 wave_record["cluster_results"][c_id] = worker_res
1407 issue_val = worker_res.get("issue", 0)
1408 warnings.extend(f"{c_id}: {w}" for w in worker_res.get("warnings", ()))
1409 # Where the worker's worktree still is, and whether its branch was pushed:
1410 # what `keel swarm-status --clean` reads (#1278).
1411 left = {
1412 "worktree": worker_res.get("worktree") or "",
1413 "pushed": bool(worker_res.get("pushed")),
1414 }
1416 # The stage a live worker ended at: `done`, or the one that stopped it. A
1417 # dry run's worker, or one that raised, reports none, and keeps the last
1418 # stage it entered (#1280).
1419 ended = {"stage": worker_res.get("stage") or None, "finished_at": _now()}
1420 if worker_res.get("ok", False):
1421 passed_count += 1
1422 # Recorded here, on the one thread that collects results, so no two
1423 # workers append to the ledger at once (#1400).
1424 if live is not None and (
1425 missed := record_pull_request_consent(live, c_id, worker_res)
1426 ):
1427 warnings.append(f"{c_id}: {missed}")
1428 # A live worker stops at an open pull request, which CI (s6) and review
1429 # take from there; a dry assessment ran the whole backbone.
1430 report(
1431 c_id,
1432 step="s10" if live is None else "s6",
1433 status="passed",
1434 details="pipeline completed" if live is None else worker_res["output"],
1435 # The pull request `swarm-land` merges (#1287); a dry run opens none.
1436 pull_request=pull_request_number(str(worker_res.get("pr_url") or "")),
1437 **left,
1438 **ended,
1439 )
1440 else:
1441 failed_count += 1
1442 report(
1443 c_id,
1444 step="s4",
1445 status="failed",
1446 details=worker_res.get("output", ""),
1447 **left,
1448 **ended,
1449 )
1450 # Dynamically rebalance subsequent waves if needed
1451 current_plan = rebalance_swarm_plan(current_plan, issue_val)
1453 # Once the wave's workers are done, the run's directory goes when nothing is left
1454 # in it — it used to accumulate, one per run (#1278).
1455 if not dry_run:
1456 remove_empty_swarm_dirs(root_path, plan.swarm_id)
1457 wave_results.append(wave_record)
1459 # Finalize state
1460 overall_status = (
1461 "success"
1462 if failed_count == 0 and passed_count > 0
1463 else ("partial_failure" if passed_count > 0 else "failed")
1464 )
1465 if passed_count == 0 and failed_count == 0:
1466 overall_status = "success"
1468 state = replace(state, completed_at=_now())
1469 save_swarm_state(state, root=root_path)
1471 return SwarmRunResult(
1472 swarm_id=plan.swarm_id,
1473 status=overall_status,
1474 total_workers=len(workers_list),
1475 passed_count=passed_count,
1476 failed_count=failed_count,
1477 dry_run=dry_run,
1478 wave_results=tuple(wave_results),
1479 consent=state.consent,
1480 warnings=tuple(warnings),
1481 )