Coverage for src/keel/delegaterun.py: 100%
340 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
1"""Thin I/O: execute one planned delegate run, foreground or detached (#1012).
3The pure half — which argv, which framing, which effort spelling, which attribution — is
4:mod:`keel.delegate`. This module is the only place that *runs* a plan: one subprocess for
5the ``cli``/``profile`` transports, one loopback HTTP call for ``ollama``, one
6:func:`keel.api_delegate.generate` call for the hosted and OpenAI-compatible ones. Every
7edge is injectable (``_run``, ``_opener``, ``_env``, ``_now``, ``_read``) so the whole
8surface is unit-testable offline and deterministic.
10**Fail-soft, always.** A missing binary, a nonzero exit, a quota refusal, a timeout, an
11unparseable answer — each becomes ``ok: false`` with a machine-readable ``error_code`` in
12the same JSON document a success produces. Nothing here raises at an operator. The
13policy that reads those codes — do not retry a ``rate-limit``, refuse a non-tool provider
14on tier 3, fall back to the host agent — stays with the caller (ship s4/s7), because it is
15policy and this is a transport.
17**The detach primitive.** A delegated implementation runs for tens of minutes; a host
18LLM's turn does not. ``--detach`` spawns the same run as a background child in its own
19session, and the state file under ``.keel/state/delegate/<run-id>.json`` is authoritative:
20the parent may exit, the session may end, the operator may start a new one, and
21``keel delegate wait <run-id>`` still returns the result. That is the primitive an
22orchestrating agent uses instead of a sleep loop — the loop it would otherwise write burns
23its own context window on polling and cannot survive the turn ending, which is how a live
24run ended up with three reviewers finished and no verdict on the PR.
26A run id is validated before it becomes a path. It reaches this module from a CLI flag and
27names a file under the state directory; ``..`` or a separator there is refused rather than
28normalized, so ``wait`` on an unknown or hostile id fails closed instead of reading one.
29"""
31from __future__ import annotations
33import contextlib
34import datetime
35import json
36import os
37import re
38import subprocess # nosec B404
39import time
40import urllib.error
41import urllib.request
42import uuid
43from collections.abc import Callable
44from pathlib import Path
45from typing import Any
47from . import api_delegate, delegate, runner, workspace
48from .delegate import RunPlan
50SCHEMA_VERSION = "keel.delegate-run.v1"
52#: Where detached run state lives, relative to the project root. Inside the existing
53#: gitignored ``.keel/state/`` tree, so a run record is never committed.
54STATE_RELDIR = (".keel", "state", "delegate")
56#: Characters a run id may contain. Deliberately tight: the id becomes a file name.
57_RUN_ID_OK = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-")
59#: Cap on a single HTTP response body, matching :mod:`keel.api_delegate`.
60_MAX_RESPONSE_BYTES = 50 * 1024 * 1024
62#: Patterns that mark a quota refusal in a CLI's own output. A hosted API answers 429 and
63#: :mod:`keel.api_delegate` classifies it; a CLI exits nonzero with prose, and the caller's
64#: no-retry-on-quota rule needs the same ``rate-limit`` code either way.
65#:
66#: ``429`` needs a status-shaped context, because three digits are not a status code
67#: (#1133). Measured over the 368 vendor transcripts this repository has on disk: every
68#: single occurrence of a bare ``429`` was something else — ``2026-09-05 16:33:40.429`` in
69#: an xcodebuild line, ``git diff 4293a56``, ``a4810728429ea3…``, ``publish.yml:429``, and
70#: a ``429`` in the gutter of a code listing. Six false matches, no true one; and the
71#: transcript that triggered the bug was several hundred KB, where a sha or a line number
72#: carrying those digits is a near certainty. The other markers are phrases, which do not
73#: have that problem.
74_RATE_LIMIT_MARKERS = (
75 re.compile(r"rate[ _-]?limit"),
76 re.compile(r"resource_exhausted"),
77 re.compile(r"quota exceeded"),
78 re.compile(r"usage limit"),
79 re.compile(r"too many requests"),
80 # `HTTP 429`, `status: 429`, `429 Too Many Requests`, `error 429` — never a bare 429.
81 re.compile(r"(?:http[/ ]|status[: ]+|code[: ]+|error[: ]+)429\b"),
82 re.compile(r"\b429\s+(?:too many|client error|error\b)"),
83)
85#: Markers indicating a quota/rate-limit error is transient and retryable (#1549).
86_RETRYABLE_MARKERS = (re.compile(r"['\"]?retryable['\"]?\s*[:=]\s*(?:true|1)\b", re.IGNORECASE),)
89#: Phrases with which a vendor reports **its own** timeout, as distinct from the wall-clock
90#: bound keel imposes (exit 124, :attr:`keel.runner.CommandResult.timed_out`).
91#:
92#: One vendor's exact wording is measured — agy's ``timeout waiting for response``, seen at
93#: 298s under a keel ``--timeout 900`` — and the rest of this list is the same word in the
94#: shapes a CLI conventionally uses. That is deliberately short: a classifier tuned on one
95#: vendor's prose is how the defect this fixes arrived, so what makes matching prose safe
96#: here is not the list but :func:`failure_signal`, which keeps these patterns away from
97#: the delegate's *answer* and shows them only the vendor's own error.
98_VENDOR_TIMEOUT_MARKERS = (
99 re.compile(r"timeout waiting for"),
100 re.compile(r"\btimed[ -]out\b"),
101 re.compile(r"deadline exceeded"),
102 re.compile(r"\btimeout\b.*\bexceeded\b"),
103)
106class RunIdError(ValueError):
107 """A run id that may not become a path."""
110def check_run_id(run_id: str) -> str:
111 """Validate a run id and return it. Raises :class:`RunIdError` on anything unsafe."""
112 if not run_id or not _RUN_ID_OK.issuperset(run_id) or run_id.strip(".") == "":
113 raise RunIdError(
114 f"invalid run id {run_id!r}: use letters, digits, '.', '_' or '-' only "
115 "(a run id becomes a file name under .keel/state/delegate/)"
116 )
117 return run_id
120def new_run_id(
121 *, _clock: Callable[[], float] = time.time, _token: Callable[[], str] | None = None
122) -> str:
123 """A fresh, sortable run id: ``<epoch-seconds>-<8 hex>``.
125 Time-prefixed so ``keel delegate status`` lists runs in the order they started, and
126 random-suffixed so two runs started in the same second cannot collide.
127 """
128 token = _token() if _token is not None else uuid.uuid4().hex[:8]
129 return f"{int(_clock())}-{token}"
132def state_dir(root: str | Path = ".") -> Path:
133 """The detached-run state directory for ``root`` (not created)."""
134 return Path(root).joinpath(*STATE_RELDIR)
137def state_path(root: str | Path, run_id: str) -> Path:
138 """Path of one run's state document."""
139 return state_dir(root) / f"{check_run_id(run_id)}.json"
142def out_path(root: str | Path, run_id: str) -> Path:
143 """Path of one detached run's captured stdout+stderr."""
144 return state_dir(root) / f"{check_run_id(run_id)}.out"
147def pid_path(root: str | Path, run_id: str) -> Path:
148 """Path of one detached run's pid, kept **beside** the record rather than in it.
150 Two writers touch a detached run: the parent, which knows the pid, and the child,
151 which knows the result. If both wrote the same document the parent's write would be a
152 read-check-write over a file the child can replace at any instant, and the child's
153 terminal record is precisely what must never be lost. Separate files means neither
154 writer needs a lock, a guard, or a retry: the record is the child's alone, the pid
155 file is the parent's alone, and a reader that wants both reads both.
156 """
157 return state_dir(root) / f"{check_run_id(run_id)}.pid"
160def crashed_path(root: str | Path, run_id: str) -> Path:
161 """Path of the marker saying a run can no longer finish. A **sidecar**, like the pid.
163 This is the last read-check-write removed. Marking a crash by editing the record
164 meant a reader could load a ``running`` document, decide the run was gone, and write
165 ``crashed`` back over a ``done`` the child had produced in between — the same lost
166 update the pid file exists to avoid, on the same file, from a third direction. With
167 the marker beside the record the invariant becomes simple enough to state in one
168 line: **the record is written by the child alone.** The parent writes the pid, a
169 reaper writes this, and :func:`run_record` composes the three — with the child's own
170 terminal status always winning, because it is the only one that saw the result.
171 """
172 return state_dir(root) / f"{check_run_id(run_id)}.crashed"
175def write_pid(root: str | Path, run_id: str, pid: int) -> None:
176 """Record a detached child's pid. Best-effort: a run without one is bounded by its
177 ``deadline_at`` instead, so failing to write it must not fail the spawn."""
178 with contextlib.suppress(OSError):
179 workspace.write_text_atomic(pid_path(root, run_id), f"{pid}\n")
182def read_pid(root: str | Path, run_id: str) -> int | None:
183 """The recorded pid of a detached run, or ``None`` when there is none to read."""
184 try:
185 return int(pid_path(root, run_id).read_text(encoding="utf-8").strip())
186 except (OSError, ValueError, RunIdError):
187 return None
190def clear_sidecars(root: str | Path, run_id: str) -> None:
191 """Delete a run id's pid and crash markers. Called before a spawn reuses the id.
193 A reused ``--run-id`` is not exotic — an orchestrator naming a run after the issue it
194 is implementing will reuse it on the retry. Left behind, the previous run's pid file
195 pairs the *new* record with a **dead** pid, and `keel delegate status` (which reaps)
196 lands on a run that started milliseconds ago and marks it crashed; a stale crash
197 marker does the same thing without even needing the race. Both are cleared while the
198 new record is still being set up, before anything can observe the pair.
199 """
200 for path in (pid_path(root, run_id), crashed_path(root, run_id)):
201 with contextlib.suppress(OSError):
202 path.unlink(missing_ok=True)
205def read_crash(root: str | Path, run_id: str) -> dict[str, Any] | None:
206 """The crash marker for a run, or ``None`` when there is none."""
207 try:
208 data = json.loads(crashed_path(root, run_id).read_text(encoding="utf-8"))
209 except (OSError, ValueError, RunIdError):
210 return None
211 return data if isinstance(data, dict) else None
214def run_record(root: str | Path, run_id: str) -> dict[str, Any] | None:
215 """A run's record as a **reader** wants it: the child's document plus the sidecars.
217 Composed rather than stored, so no writer has to carry another writer's half. The
218 child's own terminal status wins over a crash marker: a marker only ever says "this
219 looked abandoned", while a ``done`` record says "the delegate answered", and the
220 second is the stronger claim even when the marker was written later.
221 """
222 record = load_state(root, run_id)
223 if record is None:
224 return None
225 record["pid"] = read_pid(root, run_id)
226 crash = read_crash(root, run_id)
227 if crash is not None and record.get("status") == "running":
228 record["status"] = "crashed"
229 record["finished_at"] = crash.get("finished_at")
230 record["result"] = _detached_failure(
231 record, code="lost", message=crash.get("reason", "the run recorded no result")
232 )
233 return record
236def _read_text(path: str) -> str:
237 return Path(path).read_text(encoding="utf-8")
240def _now_iso(clock: Callable[[], datetime.datetime]) -> str:
241 return clock().isoformat()
244def _utc_now() -> datetime.datetime:
245 return datetime.datetime.now(datetime.UTC)
248def result_document(
249 plan: RunPlan,
250 *,
251 ok: bool,
252 text: str = "",
253 exit_code: int | None = None,
254 duration_s: float = 0.0,
255 timed_out: bool = False,
256 error_code: str | None = None,
257 error: str | None = None,
258 usage: dict[str, int] | None = None,
259 quota_wait: float = 0.0,
260) -> dict[str, Any]:
261 """The one JSON document ``keel delegate run`` prints, success or failure.
263 ``exit_code`` is ``None`` for the HTTP transports: there is no process, and reporting
264 a synthetic ``1`` would let a caller mistake a refused API key for a crashed CLI.
266 ``usage`` is the vendor's own token count for the call (#1373) —
267 ``{"prompt_tokens": …, "completion_tokens": …}`` — and ``None`` whenever the transport
268 reported none. Only the ``api`` transport fills it today; a CLI's or Ollama's run
269 reports ``None``, which means "not counted", never "free".
270 """
271 return {
272 "schema_version": SCHEMA_VERSION,
273 "ok": ok,
274 "provider": plan.provider,
275 "vendor": plan.vendor,
276 "model": plan.model,
277 "role": plan.role,
278 "transport": plan.transport,
279 "text": text,
280 "exit_code": exit_code,
281 "duration_s": duration_s,
282 "timed_out": timed_out,
283 "error_code": error_code,
284 "error": error,
285 "attribution": dict(plan.attribution),
286 "read_only": plan.read_only,
287 # Reported beside `read_only` so a caller can refuse rather than discover
288 # afterwards that its "reviewer" held the implementer's write flags.
289 "read_only_backed": plan.read_only_backed,
290 "effort_applied": plan.effort_applied,
291 "warnings": list(plan.warnings),
292 "usage": dict(usage) if usage else None,
293 "quota-wait": round(quota_wait, 3),
294 }
297def rate_limited(text: str) -> bool:
298 """Does this text read as a quota refusal? (pure, best-effort)
300 Give it a :func:`failure_signal`, not a transcript. The patterns are prose, and a
301 delegate's *answer* is prose about code — a review that says the word "rate limit",
302 a diff that contains a sha with 429 in it. Reading the whole stream is how a run that
303 timed out was reported as a quota refusal (#1133).
304 """
305 lowered = (text or "").lower()
306 return any(marker.search(lowered) for marker in _RATE_LIMIT_MARKERS)
309def retryable_quota(text: str) -> bool:
310 """Does this text read as a retryable quota error? (pure, best-effort)"""
311 return rate_limited(text) and any(m.search(text) for m in _RETRYABLE_MARKERS)
314def vendor_timed_out(text: str) -> bool:
315 """Does this text read as the vendor reporting its own timeout? (pure, best-effort)
317 Distinct from :attr:`keel.runner.CommandResult.timed_out`, which is keel's wrapper
318 killing the process. Both are timeouts and both set ``timed_out`` on the contract —
319 the recovery is the same and the operator needs to be told the same thing — but only
320 one of them leaves an exit code of 124.
321 """
322 lowered = (text or "").lower()
323 return any(marker.search(lowered) for marker in _VENDOR_TIMEOUT_MARKERS)
326def final_stream_event(stdout: str) -> dict[str, Any] | None:
327 """agy's last ``{"event": "result", ...}`` frame, or ``None``.
329 The authoritative status of a stream-json run: ``status`` and ``error`` say what
330 happened, where ``response`` is only what the model said.
331 """
332 for line in reversed((stdout or "").splitlines()):
333 line = line.strip()
334 if not line.startswith("{"):
335 continue
336 try:
337 frame = json.loads(line)
338 except (ValueError, TypeError):
339 continue
340 if isinstance(frame, dict) and frame.get("event") == "result":
341 result = frame.get("result")
342 return result if isinstance(result, dict) else frame
343 return None
346def failure_signal(
347 *,
348 stderr: str,
349 stdout: str,
350 output: str = "",
351 stream_json: bool = False,
352 limit: int = 400,
353) -> str:
354 """The text a failure *classification* may read — never the delegate's answer.
356 ``CommandResult.output`` is stdout and stderr glued together, and this module already
357 says why that is a diagnostic rather than a parse: every agent CLI writes progress and
358 notices to stderr. Classifying on it is worse than parsing it. The whole transcript of
359 a long run is hundreds of KB of the model's own prose about code — including, in the
360 transcripts on disk here, git shas and file:line references carrying the digits 429,
361 and reviews that discuss keel's own rate-limit handling by name. #1133 is one such run:
362 the vendor said ``timeout waiting for response`` after 298s and keel reported
363 ``rate-limit``, which tells an operator to wait out a quota window that does not exist.
365 So the signal is bounded and it is the vendor's, not the model's:
367 * **stream-json with a final ``result`` frame** — that frame's ``status`` and
368 ``error``, and deliberately not its ``response``. stderr is *not* added: the frame
369 already said why the run failed, and every agent CLI writes progress there. Prepending
370 it unconditionally meant the same sentence this function refuses to read out of the
371 transcript flipped the classification when it appeared on stderr instead.
372 * **stream-json with no frame** — stderr alone. A crash or a stream truncated mid-write
373 leaves nothing authoritative, and the stdout tail there is the model's own prose, so
374 falling back to it would parse the transcript in exactly the case the frame was meant
375 to protect. Both of these were found by the gate review; the first cut kept the tail.
376 * **anything else** — stderr plus the tail of stdout, which is where a CLI that has no
377 structured output prints why it stopped.
379 The frame's fields are **labelled** (``status: …``), not concatenated bare. A numeric
380 ``status`` is the one ``429`` that really is a status code, and stripping it to a bare
381 token put it out of reach of the very pattern that requires status-shaped context —
382 self-defeating for the field this function chose to read. Also from the review.
384 ``output`` is the last resort, used only when a result carries neither separated
385 stream. :func:`keel.runner._result` always fills both, so this is for a caller that
386 built a :class:`~keel.runner.CommandResult` by hand — where the concatenation is the
387 only signal there is, and refusing to read it would classify every such failure as a
388 plain nonzero exit.
389 """
390 if not (stderr or stdout):
391 return _tail(output, limit)
392 if stream_json:
393 event = final_stream_event(stdout)
394 if event is None:
395 return stderr or ""
396 parts = [
397 f"{field}: {event.get(field)}"
398 for field in ("status", "error", "retryable")
399 if event.get(field) is not None
400 ]
401 # A frame that carries no error text has said nothing about *why*; stderr is then
402 # the only account there is, and discarding it would lose a real quota refusal.
403 if not event.get("error"):
404 parts.append(stderr or "")
405 return "\n".join(part for part in parts if part)
406 return "\n".join(part for part in (stderr or "", _tail(stdout, limit)) if part)
409def execute(
410 plan: RunPlan,
411 *,
412 _run: Callable[..., runner.CommandResult] | None = None,
413 _opener=None,
414 _env=os.environ,
415 _now: Callable[[], float] = time.monotonic,
416 _read: Callable[[str], str] = _read_text,
417 _sleep: Callable[[float], None] = time.sleep,
418) -> dict[str, Any]:
419 """Run one plan and return the JSON contract. Never raises.
421 ``_run`` defaults to :func:`keel.runner.run_argv`, resolved **at call time** rather
422 than bound as the default value: a default argument is evaluated once at import, so a
423 test patching ``keel.runner.run_argv`` would still reach the real subprocess — which
424 for this module means actually launching an agent CLI.
425 """
426 # Rebound onto the parameter rather than a fresh local: #879's AST sweep reads the
427 # keywords of every call spelled `_run(...)` / `_popen(...)`, and renaming the callee
428 # would take this spawn site out of the rule's sight.
429 _run = runner.run_argv if _run is None else _run
430 started = _now()
432 def finish(**kwargs) -> dict[str, Any]:
433 return result_document(plan, duration_s=round(_now() - started, 3), **kwargs)
435 if plan.prompt is not None:
436 prompt = plan.prompt
437 else:
438 try:
439 prompt = _read(plan.prompt_path)
440 except OSError as exc:
441 return finish(ok=False, error_code="no-prompt", error=str(exc))
442 if not prompt.strip():
443 source = plan.prompt_path or "--prompt"
444 return finish(
445 ok=False,
446 error_code="no-prompt",
447 error=f"{source} is empty; a delegate with no brief produces nothing",
448 )
449 if plan.transport == "ollama":
450 return _run_ollama(plan, prompt, finish, opener=_opener)
451 if plan.transport == "api":
452 return _run_api(plan, prompt, finish, env=_env, opener=_opener)
453 return _run_argv(plan, prompt, finish, run=_run, _sleep=_sleep)
456def _run_argv(
457 plan: RunPlan,
458 prompt: str,
459 finish,
460 *,
461 run,
462 _sleep: Callable[[float], None] = time.sleep,
463) -> dict[str, Any]:
464 """The ``cli`` and ``profile`` transports: one subprocess, prompt on stdin.
466 The delegate's answer is **stdout alone**. ``CommandResult.output`` glues stderr onto
467 it, and every agent CLI writes progress, warnings and login notices there — so reading
468 the concatenation lets a run that produced no answer come back ``ok: true`` with the
469 noise as its "output", which downstream is a diff to apply or a review to post. The
470 concatenation stays for the diagnostic tail of a *failure*, where both halves help.
471 """
472 argv = list(plan.argv)
473 if plan.stdin_mode == delegate.STDIN_STREAM_JSON:
474 stdin_text: str | None = delegate.stream_json_frame(prompt)
475 elif plan.stdin_mode == delegate.STDIN_TEXT:
476 stdin_text = prompt
477 else:
478 # A profile's `prompt_mode: arg`. The planner cannot build this argv — it is
479 # handed a path, never the text — so the prompt is appended here, at the one
480 # seam that has both.
481 argv.append(prompt)
482 stdin_text = None
483 max_quota_wait = getattr(plan, "quota_wait", 0.0)
484 quota_waited = 0.0
485 backoff = 2.0
486 attempt = 1
487 while True:
488 result = run(argv, cwd=plan.cwd, timeout=plan.timeout, stdin_text=stdin_text)
489 raw = result.stdout
490 if plan.stdin_mode == delegate.STDIN_STREAM_JSON:
491 text = delegate.parse_stream_json(raw)
492 else:
493 text = raw
494 if result.timed_out:
495 return finish(
496 ok=False,
497 exit_code=result.code,
498 timed_out=True,
499 error_code="timeout",
500 error=f"{argv[0]} timed out after {plan.timeout}s",
501 text=text,
502 quota_wait=quota_waited,
503 )
504 if getattr(result, "spawn_failed", False):
505 # The runner's own OSError signal, not exit 127: a command that really ran can
506 # exit 127 too, and reading the code alone made this branch unreachable.
507 return finish(
508 ok=False,
509 exit_code=result.code,
510 error_code="missing-binary",
511 error=f"{argv[0]} could not be executed: {result.stderr or result.output}",
512 quota_wait=quota_waited,
513 )
514 if not result.ok:
515 # Classified on the vendor's own error, never on the delegate's answer (#1133),
516 # and timeout-before-quota: the two codes send an operator in opposite directions
517 # — `rate-limit` says wait out a window, `timeout` says retry now, smaller or
518 # longer — and a vendor that times out often says so in a sentence that also
519 # mentions its limits.
520 signal = failure_signal(
521 stderr=result.stderr,
522 stdout=result.stdout,
523 output=result.output,
524 stream_json=plan.stdin_mode == delegate.STDIN_STREAM_JSON,
525 )
526 vendor_timeout = vendor_timed_out(signal)
527 if vendor_timeout:
528 code = "timeout"
529 elif rate_limited(signal):
530 code = "rate-limit"
531 else:
532 code = "nonzero-exit"
533 if code == "rate-limit" and retryable_quota(signal):
534 remaining = max_quota_wait - quota_waited
535 if remaining > 0:
536 delay = min(backoff, remaining)
537 _sleep(delay)
538 quota_waited += delay
539 backoff *= 2
540 attempt += 1
541 continue
542 return finish(
543 ok=False,
544 exit_code=result.code,
545 # True for either bound: keel's wall-clock wrapper (exit 124) or the vendor's
546 # own. The run did time out; which timer fired is in `exit_code` and `error`.
547 timed_out=vendor_timeout,
548 error_code=code,
549 # The tail still shows both streams — a diagnostic wants everything a reader
550 # might recognise, which is exactly what a decision must not read.
551 error=f"{argv[0]} exited {result.code}: {_tail(result.output)}",
552 text=text,
553 quota_wait=quota_waited,
554 )
555 if not text.strip():
556 return finish(
557 ok=False,
558 exit_code=result.code,
559 error_code="empty-output",
560 error=f"{argv[0]} exited 0 and produced no output",
561 quota_wait=quota_waited,
562 )
563 return finish(ok=True, exit_code=result.code, text=text, quota_wait=quota_waited)
566def _run_api(plan: RunPlan, prompt: str, finish, *, env, opener) -> dict[str, Any]:
567 """Hosted and OpenAI-compatible vendors: exactly one call through ``api_delegate``."""
568 request = plan.request or {}
569 result = api_delegate.generate(
570 request["vendor"],
571 request["model"],
572 prompt,
573 endpoint=request.get("endpoint"),
574 api_key_env=request.get("api_key_env"),
575 max_tokens=request.get("max_tokens", api_delegate.DEFAULT_MAX_TOKENS),
576 timeout=request.get("timeout", api_delegate.DEFAULT_TIMEOUT),
577 extra_payload=request.get("extra_payload") or None,
578 _env=env,
579 _opener=opener,
580 )
581 if not result.ok:
582 # A `bad-response` can still carry the vendor's counts: the call was made.
583 return finish(
584 ok=False, error_code=result.error_code, error=result.error, usage=result.usage
585 )
586 return finish(ok=True, text=result.text, usage=result.usage)
589def _run_ollama(plan: RunPlan, prompt: str, finish, *, opener) -> dict[str, Any]:
590 """The built-in local vendor: one POST to the hardcoded loopback ``/api/generate``."""
591 request = plan.request or {}
592 body = json.dumps(delegate.ollama_payload(request["model"], prompt)).encode("utf-8")
593 # The URL is delegate.OLLAMA_GENERATE_URL, a hardcoded constant — never config, never
594 # the registry, never model or prompt content.
595 http_request = urllib.request.Request( # nosec B310
596 delegate.OLLAMA_GENERATE_URL,
597 data=body,
598 headers={"content-type": "application/json"},
599 method="POST",
600 )
601 client = opener if opener is not None else api_delegate.build_http_only_opener()
602 try:
603 with client.open(http_request, timeout=plan.timeout) as response:
604 raw = response.read(_MAX_RESPONSE_BYTES).decode("utf-8", errors="replace")
605 status = getattr(response, "status", 200)
606 except urllib.error.HTTPError as exc:
607 code = "rate-limit" if exc.code == 429 else "http"
608 return finish(ok=False, error_code=code, error=f"HTTP {exc.code}")
609 except Exception as exc:
610 # Deliberately broad, as in providerprobe: urllib raises URLError, http.client
611 # exceptions (not OSError subclasses), socket timeouts and the address guard's
612 # own error. An operator sees "the local server did not answer", not a traceback.
613 return finish(ok=False, error_code="network", error=str(exc))
614 if not 200 <= status < 300:
615 return finish(ok=False, error_code="http", error=f"HTTP {status}")
616 try:
617 data = json.loads(raw)
618 except ValueError:
619 return finish(ok=False, error_code="bad-response", error="response is not valid JSON")
620 text = delegate.parse_ollama_response(data)
621 if not text:
622 return finish(
623 ok=False, error_code="bad-response", error="response carried no completion text"
624 )
625 return finish(ok=True, text=text)
628def _tail(text: str, limit: int = 400) -> str:
629 stripped = (text or "").strip()
630 return stripped[-limit:]
633# --------------------------------------------------------------------------- detach
636def load_state(root: str | Path, run_id: str) -> dict[str, Any] | None:
637 """One run's state document, or ``None`` when it does not exist / cannot be read."""
638 try:
639 path = state_path(root, run_id)
640 except RunIdError:
641 return None
642 try:
643 return json.loads(path.read_text(encoding="utf-8"))
644 except (OSError, ValueError):
645 return None
648def write_state(root: str | Path, record: dict[str, Any]) -> Path:
649 """Persist a run's state atomically, so a reader never sees a torn document."""
650 path = state_path(root, record["run_id"])
651 workspace.write_text_atomic(path, json.dumps(record, indent=2, sort_keys=True))
652 return path
655def list_runs(root: str | Path = ".") -> list[dict[str, Any]]:
656 """Every readable run record under the state directory, oldest id first."""
657 directory = state_dir(root)
658 records = []
659 try:
660 names = sorted(p.stem for p in directory.glob("*.json"))
661 except OSError: # pragma: no cover - glob on a readable parent does not fail
662 return []
663 for name in names:
664 record = run_record(root, name)
665 if record is not None:
666 records.append(record)
667 return records
670#: Extra wall-clock seconds a detached run gets beyond its own ``--timeout`` before
671#: ``wait`` calls it abandoned. The child enforces the timeout itself and then has to
672#: write its result; the grace is the room for that last write.
673DEADLINE_GRACE_S = 60
676def start_detached(
677 argv: list[str],
678 *,
679 root: str | Path,
680 run_id: str,
681 cwd: str | None = None,
682 timeout: int | None = None,
683 provider: str | None = None,
684 role: str | None = None,
685 _popen=None,
686 _clock: Callable[[], datetime.datetime] = _utc_now,
687) -> dict[str, Any]:
688 """Spawn ``argv`` as a background child and record it. Never raises.
690 The record is written **before** the spawn, so a ``wait`` issued immediately
691 afterwards always finds a file, and the parent never writes it again: the pid goes to
692 its own ``<run-id>.pid`` file (:func:`pid_path`). That is the fix for a lost update.
693 A guarded read-check-write from the parent was still a race — the child's terminal
694 record can land between the read and the write, and the parent would put ``running``
695 back over the result the caller is waiting for. Two files, one writer each, and no
696 window at all.
698 ``deadline_at`` is stamped from the run's own ``--timeout`` plus
699 :data:`DEADLINE_GRACE_S`. It is what stops a ``wait`` with no ``--timeout`` of its own
700 from blocking forever when the child dies without recording anything — a
701 ``SIGKILL``, an OOM kill, a reboot.
703 The child gets its own session where the platform has one, so killing the caller's
704 process group does not take the delegate with it — surviving the caller is the whole
705 point of ``--detach``.
707 ``_popen`` defaults to :class:`subprocess.Popen` resolved at call time, for the same
708 reason :func:`execute`'s ``_run`` does: a default bound at import cannot be patched,
709 and here the cost of missing the seam is a real agent CLI launched by a unit test.
710 """
711 _popen = subprocess.Popen if _popen is None else _popen
712 check_run_id(run_id)
713 started = _clock()
714 record: dict[str, Any] = {
715 "schema_version": SCHEMA_VERSION,
716 "run_id": run_id,
717 "provider": provider,
718 "role": role,
719 "started_at": started.isoformat(),
720 "timeout": timeout,
721 "deadline_at": (
722 None
723 if timeout is None
724 else (started + datetime.timedelta(seconds=timeout + DEADLINE_GRACE_S)).isoformat()
725 ),
726 "status": "running",
727 "argv": list(argv),
728 "out_path": str(out_path(root, run_id)),
729 "result": None,
730 }
731 # A conditional expression rather than an `if`: the assignment then executes on
732 # every platform, so the 100 % line gate holds on the Windows runner too, where
733 # `os.setsid` does not exist and sessions are not a thing.
734 kwargs: dict[str, Any] = {"start_new_session": True} if hasattr(os, "setsid") else {}
735 try:
736 # Creating the directory and opening the log are I/O like any other: an
737 # unwritable root (a read-only checkout, a root owned by another user) must come
738 # back as the same fail-soft contract every other failure does, not as a
739 # PermissionError traceback out of `keel delegate run --detach`.
740 state_dir(root).mkdir(parents=True, exist_ok=True)
741 write_state(root, record)
742 # Same reason the log is opened "w": a reused run id must inherit nothing from
743 # the run before it. See `clear_sidecars`.
744 clear_sidecars(root, run_id)
745 handle = out_path(root, run_id).open("w", encoding="utf-8")
746 except OSError as exc:
747 return _failed_spawn_record(root, record, exc)
748 try:
749 # argv is keel's own re-invocation (sys.executable -m keel), never a shell.
750 child = _popen(
751 list(argv),
752 cwd=cwd,
753 stdin=subprocess.DEVNULL,
754 stdout=handle,
755 stderr=subprocess.STDOUT,
756 **kwargs,
757 )
758 except OSError as exc:
759 return _failed_spawn_record(root, record, exc)
760 finally:
761 handle.close()
762 write_pid(root, run_id, child.pid)
763 record["pid"] = child.pid
764 return record
767def _failed_spawn_record(root: str | Path, record: dict[str, Any], exc: OSError) -> dict[str, Any]:
768 """Mark a run that never started, persisting the record when the disk allows it.
770 Best-effort on purpose: the failure being reported may itself be "this directory
771 cannot be written", and a second write would raise the very traceback the caller is
772 being spared. The returned document is the contract either way.
773 """
774 record["status"] = "done"
775 record["pid"] = None
776 record["result"] = _spawn_failure(record, exc)
777 with contextlib.suppress(OSError):
778 write_state(root, record)
779 return record
782def _spawn_failure(record: dict[str, Any], exc: OSError) -> dict[str, Any]:
783 return _detached_failure(record, code="spawn-failed", message=str(exc))
786def _detached_failure(record: dict[str, Any], *, code: str, message: str) -> dict[str, Any]:
787 """A full return contract for a detached run that never produced one itself.
789 Same keys as :func:`result_document`, so ``keel delegate wait`` hands back one shape
790 whether the delegate answered, failed, or vanished.
791 """
792 return {
793 "schema_version": SCHEMA_VERSION,
794 "ok": False,
795 "provider": record.get("provider"),
796 "vendor": None,
797 "model": None,
798 "role": record.get("role"),
799 "transport": None,
800 "text": "",
801 "exit_code": None,
802 "duration_s": 0.0,
803 "timed_out": code == "lost",
804 "error_code": code,
805 "error": message,
806 "attribution": {},
807 "read_only": None,
808 "read_only_backed": False,
809 "effort_applied": False,
810 "warnings": [],
811 "usage": None,
812 "quota-wait": 0.0,
813 "run_id": record.get("run_id"),
814 "out_path": record.get("out_path"),
815 }
818def record_planning_failure(
819 root: str | Path,
820 run_id: str,
821 failure: dict[str, Any],
822 *,
823 provider: str | None,
824 role: str | None,
825 _clock: Callable[[], datetime.datetime] = _utc_now,
826) -> dict[str, Any]:
827 """Persist a ``--detach`` run whose plan could not be made, so ``status`` lists it (#1510).
829 There is no child and no pid: the record is born ``done`` with the failure document
830 as its result, which is the shape :func:`_failed_spawn_record` already gives a run
831 that never started. Best-effort like it, because an unwritable root must not turn a
832 reported failure into a traceback. Returns the failure document carrying ``run_id``.
833 """
834 check_run_id(run_id)
835 now = _now_iso(_clock)
836 result = {**failure, "run_id": run_id, "out_path": None}
837 record = {
838 "schema_version": SCHEMA_VERSION,
839 "run_id": run_id,
840 "provider": provider,
841 "role": role,
842 "started_at": now,
843 "finished_at": now,
844 "timeout": None,
845 "deadline_at": None,
846 "status": "done",
847 "pid": None,
848 "argv": [],
849 "out_path": None,
850 "result": result,
851 }
852 with contextlib.suppress(OSError):
853 state_dir(root).mkdir(parents=True, exist_ok=True)
854 # A reused run id must not pair this record with the previous run's pid or crash
855 # marker (see `clear_sidecars`).
856 write_state(root, record)
857 # Only once the new record exists, so a failed write keeps the old markers.
858 clear_sidecars(root, run_id)
859 return result
862def finish_detached(
863 root: str | Path,
864 run_id: str,
865 result: dict[str, Any],
866 *,
867 _clock: Callable[[], datetime.datetime] = _utc_now,
868) -> Path:
869 """Record a detached run's result, keeping whatever the parent already wrote."""
870 record = load_state(root, run_id) or {
871 "schema_version": SCHEMA_VERSION,
872 "run_id": run_id,
873 "pid": None,
874 "started_at": _now_iso(_clock),
875 "argv": [],
876 "out_path": str(out_path(root, run_id)),
877 }
878 record["status"] = "done"
879 record["finished_at"] = _now_iso(_clock)
880 record["result"] = result
881 return write_state(root, record)
884#: How often :func:`wait` re-reads the state file.
885DEFAULT_POLL_S = 0.5
888def process_is_alive(pid: int) -> bool:
889 """Is ``pid`` still a process we can see? Best-effort, never raises.
891 ``os.kill(pid, 0)`` sends no signal; it asks the kernel whether the pid exists and
892 whether we may signal it. ``PermissionError`` therefore means **alive** (it exists and
893 belongs to someone else), which is the opposite of what a bare ``except`` would say.
895 Two honest limits, both bounded by the run's own ``deadline_at`` rather than papered
896 over: a child not yet reaped by its parent is a zombie and still answers "alive", and
897 a recycled pid can answer "alive" for an unrelated process. Neither can make ``wait``
898 return a wrong *result* — only make it wait longer before giving up.
899 """
900 if not hasattr(os, "kill"): # pragma: no cover - POSIX everywhere keel runs
901 return True
902 try:
903 os.kill(pid, 0)
904 except ProcessLookupError:
905 return False
906 except (PermissionError, OSError):
907 return True
908 return True
911def _abandoned(
912 record: dict[str, Any],
913 *,
914 alive: Callable[[int], bool],
915 clock: Callable[[], datetime.datetime],
916) -> str | None:
917 """Why this still-``running`` record can never finish, or ``None`` while it might."""
918 pid = record.get("pid")
919 if isinstance(pid, int) and not alive(pid):
920 return (
921 f"the delegate process (pid {pid}) is gone and recorded no result; "
922 f"its output is in {record.get('out_path')}"
923 )
924 deadline = record.get("deadline_at")
925 if isinstance(deadline, str) and deadline:
926 try:
927 when = datetime.datetime.fromisoformat(deadline)
928 except ValueError:
929 return None
930 if clock() >= when:
931 return (
932 f"the run passed its own deadline ({deadline}) without recording a "
933 f"result; its output is in {record.get('out_path')}"
934 )
935 return None
938def _mark_crashed(
939 root: str | Path,
940 run_id: str,
941 reason: str,
942 *,
943 clock: Callable[[], datetime.datetime],
944) -> dict[str, Any] | None:
945 """Note that a run vanished, in its own file, and return the composed record.
947 No read, no check, no write-back: the marker goes to :func:`crashed_path` and
948 :func:`run_record` decides what it means. So a child that wrote ``done`` while the
949 liveness probe was running still reports ``done`` — the marker is simply ignored —
950 and nothing can be lost, because nothing is overwritten.
951 """
952 with contextlib.suppress(OSError):
953 workspace.write_text_atomic(
954 crashed_path(root, run_id),
955 json.dumps({"reason": reason, "finished_at": _now_iso(clock)}, indent=2),
956 )
957 return run_record(root, run_id)
960def reap_abandoned(
961 root: str | Path = ".",
962 *,
963 _alive: Callable[[int], bool] | None = None,
964 _clock: Callable[[], datetime.datetime] = _utc_now,
965) -> list[str]:
966 """Mark every run that can no longer finish, and return their ids.
968 The same liveness and deadline test :func:`wait` applies, run across the whole state
969 directory. Without it a run only stops claiming to be ``running`` when somebody
970 happens to ``wait`` on it — so ``keel delegate status``, the command an operator uses
971 precisely because they are *not* waiting, would be the one view that never told the
972 truth about a killed child.
973 """
974 alive = process_is_alive if _alive is None else _alive
975 reaped = []
976 for record in list_runs(root):
977 if record.get("status") != "running":
978 continue
979 reason = _abandoned(record, alive=alive, clock=_clock)
980 if reason is not None:
981 _mark_crashed(root, record["run_id"], reason, clock=_clock)
982 reaped.append(record["run_id"])
983 return reaped
986def wait(
987 root: str | Path,
988 run_id: str,
989 *,
990 timeout: float | None = None,
991 poll: float = DEFAULT_POLL_S,
992 _sleep: Callable[[float], None] = time.sleep,
993 _now: Callable[[], float] = time.monotonic,
994 _clock: Callable[[], datetime.datetime] = _utc_now,
995 _alive: Callable[[int], bool] | None = None,
996) -> tuple[dict[str, Any] | None, str | None]:
997 """Block until ``run_id`` finishes. Returns ``(result, error_code)``.
999 The state file is the authority — a *result* is only ever read from it, so it survives
1000 the process that produced it, a new session, and a reboot.
1002 Three ways this returns without a result, and all three are bounded:
1004 * ``unknown-run`` — fails closed and immediately. An orchestrator that mistypes a run
1005 id, or asks about a run from another checkout, must not get a wait that can only end
1006 in a timeout.
1007 * ``lost`` — the child is gone, or the run passed the deadline stamped at spawn, and
1008 nothing was recorded. Without this a ``SIGKILL``ed child left ``running`` forever
1009 and a ``wait`` with no ``--timeout`` blocked indefinitely. The record is marked
1010 ``crashed`` so ``keel delegate status`` stops claiming it is running.
1011 * ``timeout`` — the caller's own ``--timeout`` elapsed first. The run may still be
1012 alive; nothing is marked.
1014 A dead pid is re-checked against the file before being called lost, because the child
1015 exits *after* writing its result and the two are observed in the other order often
1016 enough to matter.
1017 """
1018 deadline = None if timeout is None else _now() + timeout
1019 alive = process_is_alive if _alive is None else _alive
1020 while True:
1021 record = run_record(root, run_id)
1022 if record is None:
1023 return None, "unknown-run"
1024 status = record.get("status")
1025 if status == "done":
1026 return record.get("result"), None
1027 if status == "crashed":
1028 return record.get("result"), "lost"
1029 reason = _abandoned(record, alive=alive, clock=_clock)
1030 if reason is not None:
1031 record = _mark_crashed(root, run_id, reason, clock=_clock)
1032 if record is None:
1033 return None, "unknown-run"
1034 if record.get("status") == "done":
1035 return record.get("result"), None
1036 return record.get("result"), "lost"
1037 if deadline is not None and _now() >= deadline:
1038 return None, "timeout"
1039 _sleep(poll)
1042def planning_failure(
1043 provider: str,
1044 role: str,
1045 *,
1046 code: str,
1047 message: str,
1048) -> dict[str, Any]:
1049 """The JSON contract for a run that never started — a plan that could not be made.
1051 Same keys as :func:`result_document` with the unresolved halves null, so a caller
1052 parses one shape whether the provider was unknown, the model token unsafe, or the
1053 delegate simply exited nonzero.
1054 """
1055 return {
1056 "schema_version": SCHEMA_VERSION,
1057 "ok": False,
1058 "provider": provider,
1059 "vendor": None,
1060 "model": None,
1061 "role": role,
1062 "transport": None,
1063 "text": "",
1064 "exit_code": None,
1065 "duration_s": 0.0,
1066 "timed_out": False,
1067 "error_code": code,
1068 "error": message,
1069 "attribution": {},
1070 "read_only": None,
1071 "read_only_backed": False,
1072 "effort_applied": False,
1073 "warnings": [],
1074 "usage": None,
1075 "quota-wait": 0.0,
1076 }