Coverage for src/keel/delegaterun.py: 100%

340 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 19:17 +0000

1"""Thin I/O: execute one planned delegate run, foreground or detached (#1012). 

2 

3The pure half — which argv, which framing, which effort spelling, which attribution — is 

4:mod:`keel.delegate`. This module is the only place that *runs* a plan: one subprocess for 

5the ``cli``/``profile`` transports, one loopback HTTP call for ``ollama``, one 

6:func:`keel.api_delegate.generate` call for the hosted and OpenAI-compatible ones. Every 

7edge is injectable (``_run``, ``_opener``, ``_env``, ``_now``, ``_read``) so the whole 

8surface is unit-testable offline and deterministic. 

9 

10**Fail-soft, always.** A missing binary, a nonzero exit, a quota refusal, a timeout, an 

11unparseable answer — each becomes ``ok: false`` with a machine-readable ``error_code`` in 

12the same JSON document a success produces. Nothing here raises at an operator. The 

13policy that reads those codes — do not retry a ``rate-limit``, refuse a non-tool provider 

14on tier 3, fall back to the host agent — stays with the caller (ship s4/s7), because it is 

15policy and this is a transport. 

16 

17**The detach primitive.** A delegated implementation runs for tens of minutes; a host 

18LLM's turn does not. ``--detach`` spawns the same run as a background child in its own 

19session, and the state file under ``.keel/state/delegate/<run-id>.json`` is authoritative: 

20the parent may exit, the session may end, the operator may start a new one, and 

21``keel delegate wait <run-id>`` still returns the result. That is the primitive an 

22orchestrating agent uses instead of a sleep loop — the loop it would otherwise write burns 

23its own context window on polling and cannot survive the turn ending, which is how a live 

24run ended up with three reviewers finished and no verdict on the PR. 

25 

26A run id is validated before it becomes a path. It reaches this module from a CLI flag and 

27names a file under the state directory; ``..`` or a separator there is refused rather than 

28normalized, so ``wait`` on an unknown or hostile id fails closed instead of reading one. 

29""" 

30 

31from __future__ import annotations 

32 

33import contextlib 

34import datetime 

35import json 

36import os 

37import re 

38import subprocess # nosec B404 

39import time 

40import urllib.error 

41import urllib.request 

42import uuid 

43from collections.abc import Callable 

44from pathlib import Path 

45from typing import Any 

46 

47from . import api_delegate, delegate, runner, workspace 

48from .delegate import RunPlan 

49 

50SCHEMA_VERSION = "keel.delegate-run.v1" 

51 

52#: Where detached run state lives, relative to the project root. Inside the existing 

53#: gitignored ``.keel/state/`` tree, so a run record is never committed. 

54STATE_RELDIR = (".keel", "state", "delegate") 

55 

56#: Characters a run id may contain. Deliberately tight: the id becomes a file name. 

57_RUN_ID_OK = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") 

58 

59#: Cap on a single HTTP response body, matching :mod:`keel.api_delegate`. 

60_MAX_RESPONSE_BYTES = 50 * 1024 * 1024 

61 

62#: Patterns that mark a quota refusal in a CLI's own output. A hosted API answers 429 and 

63#: :mod:`keel.api_delegate` classifies it; a CLI exits nonzero with prose, and the caller's 

64#: no-retry-on-quota rule needs the same ``rate-limit`` code either way. 

65#: 

66#: ``429`` needs a status-shaped context, because three digits are not a status code 

67#: (#1133). Measured over the 368 vendor transcripts this repository has on disk: every 

68#: single occurrence of a bare ``429`` was something else — ``2026-09-05 16:33:40.429`` in 

69#: an xcodebuild line, ``git diff 4293a56``, ``a4810728429ea3…``, ``publish.yml:429``, and 

70#: a ``429`` in the gutter of a code listing. Six false matches, no true one; and the 

71#: transcript that triggered the bug was several hundred KB, where a sha or a line number 

72#: carrying those digits is a near certainty. The other markers are phrases, which do not 

73#: have that problem. 

74_RATE_LIMIT_MARKERS = ( 

75 re.compile(r"rate[ _-]?limit"), 

76 re.compile(r"resource_exhausted"), 

77 re.compile(r"quota exceeded"), 

78 re.compile(r"usage limit"), 

79 re.compile(r"too many requests"), 

80 # `HTTP 429`, `status: 429`, `429 Too Many Requests`, `error 429` — never a bare 429. 

81 re.compile(r"(?:http[/ ]|status[: ]+|code[: ]+|error[: ]+)429\b"), 

82 re.compile(r"\b429\s+(?:too many|client error|error\b)"), 

83) 

84 

85#: Markers indicating a quota/rate-limit error is transient and retryable (#1549). 

86_RETRYABLE_MARKERS = (re.compile(r"['\"]?retryable['\"]?\s*[:=]\s*(?:true|1)\b", re.IGNORECASE),) 

87 

88 

89#: Phrases with which a vendor reports **its own** timeout, as distinct from the wall-clock 

90#: bound keel imposes (exit 124, :attr:`keel.runner.CommandResult.timed_out`). 

91#: 

92#: One vendor's exact wording is measured — agy's ``timeout waiting for response``, seen at 

93#: 298s under a keel ``--timeout 900`` — and the rest of this list is the same word in the 

94#: shapes a CLI conventionally uses. That is deliberately short: a classifier tuned on one 

95#: vendor's prose is how the defect this fixes arrived, so what makes matching prose safe 

96#: here is not the list but :func:`failure_signal`, which keeps these patterns away from 

97#: the delegate's *answer* and shows them only the vendor's own error. 

98_VENDOR_TIMEOUT_MARKERS = ( 

99 re.compile(r"timeout waiting for"), 

100 re.compile(r"\btimed[ -]out\b"), 

101 re.compile(r"deadline exceeded"), 

102 re.compile(r"\btimeout\b.*\bexceeded\b"), 

103) 

104 

105 

106class RunIdError(ValueError): 

107 """A run id that may not become a path.""" 

108 

109 

110def check_run_id(run_id: str) -> str: 

111 """Validate a run id and return it. Raises :class:`RunIdError` on anything unsafe.""" 

112 if not run_id or not _RUN_ID_OK.issuperset(run_id) or run_id.strip(".") == "": 

113 raise RunIdError( 

114 f"invalid run id {run_id!r}: use letters, digits, '.', '_' or '-' only " 

115 "(a run id becomes a file name under .keel/state/delegate/)" 

116 ) 

117 return run_id 

118 

119 

120def new_run_id( 

121 *, _clock: Callable[[], float] = time.time, _token: Callable[[], str] | None = None 

122) -> str: 

123 """A fresh, sortable run id: ``<epoch-seconds>-<8 hex>``. 

124 

125 Time-prefixed so ``keel delegate status`` lists runs in the order they started, and 

126 random-suffixed so two runs started in the same second cannot collide. 

127 """ 

128 token = _token() if _token is not None else uuid.uuid4().hex[:8] 

129 return f"{int(_clock())}-{token}" 

130 

131 

132def state_dir(root: str | Path = ".") -> Path: 

133 """The detached-run state directory for ``root`` (not created).""" 

134 return Path(root).joinpath(*STATE_RELDIR) 

135 

136 

137def state_path(root: str | Path, run_id: str) -> Path: 

138 """Path of one run's state document.""" 

139 return state_dir(root) / f"{check_run_id(run_id)}.json" 

140 

141 

142def out_path(root: str | Path, run_id: str) -> Path: 

143 """Path of one detached run's captured stdout+stderr.""" 

144 return state_dir(root) / f"{check_run_id(run_id)}.out" 

145 

146 

147def pid_path(root: str | Path, run_id: str) -> Path: 

148 """Path of one detached run's pid, kept **beside** the record rather than in it. 

149 

150 Two writers touch a detached run: the parent, which knows the pid, and the child, 

151 which knows the result. If both wrote the same document the parent's write would be a 

152 read-check-write over a file the child can replace at any instant, and the child's 

153 terminal record is precisely what must never be lost. Separate files means neither 

154 writer needs a lock, a guard, or a retry: the record is the child's alone, the pid 

155 file is the parent's alone, and a reader that wants both reads both. 

156 """ 

157 return state_dir(root) / f"{check_run_id(run_id)}.pid" 

158 

159 

160def crashed_path(root: str | Path, run_id: str) -> Path: 

161 """Path of the marker saying a run can no longer finish. A **sidecar**, like the pid. 

162 

163 This is the last read-check-write removed. Marking a crash by editing the record 

164 meant a reader could load a ``running`` document, decide the run was gone, and write 

165 ``crashed`` back over a ``done`` the child had produced in between — the same lost 

166 update the pid file exists to avoid, on the same file, from a third direction. With 

167 the marker beside the record the invariant becomes simple enough to state in one 

168 line: **the record is written by the child alone.** The parent writes the pid, a 

169 reaper writes this, and :func:`run_record` composes the three — with the child's own 

170 terminal status always winning, because it is the only one that saw the result. 

171 """ 

172 return state_dir(root) / f"{check_run_id(run_id)}.crashed" 

173 

174 

175def write_pid(root: str | Path, run_id: str, pid: int) -> None: 

176 """Record a detached child's pid. Best-effort: a run without one is bounded by its 

177 ``deadline_at`` instead, so failing to write it must not fail the spawn.""" 

178 with contextlib.suppress(OSError): 

179 workspace.write_text_atomic(pid_path(root, run_id), f"{pid}\n") 

180 

181 

182def read_pid(root: str | Path, run_id: str) -> int | None: 

183 """The recorded pid of a detached run, or ``None`` when there is none to read.""" 

184 try: 

185 return int(pid_path(root, run_id).read_text(encoding="utf-8").strip()) 

186 except (OSError, ValueError, RunIdError): 

187 return None 

188 

189 

190def clear_sidecars(root: str | Path, run_id: str) -> None: 

191 """Delete a run id's pid and crash markers. Called before a spawn reuses the id. 

192 

193 A reused ``--run-id`` is not exotic — an orchestrator naming a run after the issue it 

194 is implementing will reuse it on the retry. Left behind, the previous run's pid file 

195 pairs the *new* record with a **dead** pid, and `keel delegate status` (which reaps) 

196 lands on a run that started milliseconds ago and marks it crashed; a stale crash 

197 marker does the same thing without even needing the race. Both are cleared while the 

198 new record is still being set up, before anything can observe the pair. 

199 """ 

200 for path in (pid_path(root, run_id), crashed_path(root, run_id)): 

201 with contextlib.suppress(OSError): 

202 path.unlink(missing_ok=True) 

203 

204 

205def read_crash(root: str | Path, run_id: str) -> dict[str, Any] | None: 

206 """The crash marker for a run, or ``None`` when there is none.""" 

207 try: 

208 data = json.loads(crashed_path(root, run_id).read_text(encoding="utf-8")) 

209 except (OSError, ValueError, RunIdError): 

210 return None 

211 return data if isinstance(data, dict) else None 

212 

213 

214def run_record(root: str | Path, run_id: str) -> dict[str, Any] | None: 

215 """A run's record as a **reader** wants it: the child's document plus the sidecars. 

216 

217 Composed rather than stored, so no writer has to carry another writer's half. The 

218 child's own terminal status wins over a crash marker: a marker only ever says "this 

219 looked abandoned", while a ``done`` record says "the delegate answered", and the 

220 second is the stronger claim even when the marker was written later. 

221 """ 

222 record = load_state(root, run_id) 

223 if record is None: 

224 return None 

225 record["pid"] = read_pid(root, run_id) 

226 crash = read_crash(root, run_id) 

227 if crash is not None and record.get("status") == "running": 

228 record["status"] = "crashed" 

229 record["finished_at"] = crash.get("finished_at") 

230 record["result"] = _detached_failure( 

231 record, code="lost", message=crash.get("reason", "the run recorded no result") 

232 ) 

233 return record 

234 

235 

236def _read_text(path: str) -> str: 

237 return Path(path).read_text(encoding="utf-8") 

238 

239 

240def _now_iso(clock: Callable[[], datetime.datetime]) -> str: 

241 return clock().isoformat() 

242 

243 

244def _utc_now() -> datetime.datetime: 

245 return datetime.datetime.now(datetime.UTC) 

246 

247 

248def result_document( 

249 plan: RunPlan, 

250 *, 

251 ok: bool, 

252 text: str = "", 

253 exit_code: int | None = None, 

254 duration_s: float = 0.0, 

255 timed_out: bool = False, 

256 error_code: str | None = None, 

257 error: str | None = None, 

258 usage: dict[str, int] | None = None, 

259 quota_wait: float = 0.0, 

260) -> dict[str, Any]: 

261 """The one JSON document ``keel delegate run`` prints, success or failure. 

262 

263 ``exit_code`` is ``None`` for the HTTP transports: there is no process, and reporting 

264 a synthetic ``1`` would let a caller mistake a refused API key for a crashed CLI. 

265 

266 ``usage`` is the vendor's own token count for the call (#1373) — 

267 ``{"prompt_tokens": …, "completion_tokens": …}`` — and ``None`` whenever the transport 

268 reported none. Only the ``api`` transport fills it today; a CLI's or Ollama's run 

269 reports ``None``, which means "not counted", never "free". 

270 """ 

271 return { 

272 "schema_version": SCHEMA_VERSION, 

273 "ok": ok, 

274 "provider": plan.provider, 

275 "vendor": plan.vendor, 

276 "model": plan.model, 

277 "role": plan.role, 

278 "transport": plan.transport, 

279 "text": text, 

280 "exit_code": exit_code, 

281 "duration_s": duration_s, 

282 "timed_out": timed_out, 

283 "error_code": error_code, 

284 "error": error, 

285 "attribution": dict(plan.attribution), 

286 "read_only": plan.read_only, 

287 # Reported beside `read_only` so a caller can refuse rather than discover 

288 # afterwards that its "reviewer" held the implementer's write flags. 

289 "read_only_backed": plan.read_only_backed, 

290 "effort_applied": plan.effort_applied, 

291 "warnings": list(plan.warnings), 

292 "usage": dict(usage) if usage else None, 

293 "quota-wait": round(quota_wait, 3), 

294 } 

295 

296 

297def rate_limited(text: str) -> bool: 

298 """Does this text read as a quota refusal? (pure, best-effort) 

299 

300 Give it a :func:`failure_signal`, not a transcript. The patterns are prose, and a 

301 delegate's *answer* is prose about code — a review that says the word "rate limit", 

302 a diff that contains a sha with 429 in it. Reading the whole stream is how a run that 

303 timed out was reported as a quota refusal (#1133). 

304 """ 

305 lowered = (text or "").lower() 

306 return any(marker.search(lowered) for marker in _RATE_LIMIT_MARKERS) 

307 

308 

309def retryable_quota(text: str) -> bool: 

310 """Does this text read as a retryable quota error? (pure, best-effort)""" 

311 return rate_limited(text) and any(m.search(text) for m in _RETRYABLE_MARKERS) 

312 

313 

314def vendor_timed_out(text: str) -> bool: 

315 """Does this text read as the vendor reporting its own timeout? (pure, best-effort) 

316 

317 Distinct from :attr:`keel.runner.CommandResult.timed_out`, which is keel's wrapper 

318 killing the process. Both are timeouts and both set ``timed_out`` on the contract — 

319 the recovery is the same and the operator needs to be told the same thing — but only 

320 one of them leaves an exit code of 124. 

321 """ 

322 lowered = (text or "").lower() 

323 return any(marker.search(lowered) for marker in _VENDOR_TIMEOUT_MARKERS) 

324 

325 

326def final_stream_event(stdout: str) -> dict[str, Any] | None: 

327 """agy's last ``{"event": "result", ...}`` frame, or ``None``. 

328 

329 The authoritative status of a stream-json run: ``status`` and ``error`` say what 

330 happened, where ``response`` is only what the model said. 

331 """ 

332 for line in reversed((stdout or "").splitlines()): 

333 line = line.strip() 

334 if not line.startswith("{"): 

335 continue 

336 try: 

337 frame = json.loads(line) 

338 except (ValueError, TypeError): 

339 continue 

340 if isinstance(frame, dict) and frame.get("event") == "result": 

341 result = frame.get("result") 

342 return result if isinstance(result, dict) else frame 

343 return None 

344 

345 

346def failure_signal( 

347 *, 

348 stderr: str, 

349 stdout: str, 

350 output: str = "", 

351 stream_json: bool = False, 

352 limit: int = 400, 

353) -> str: 

354 """The text a failure *classification* may read — never the delegate's answer. 

355 

356 ``CommandResult.output`` is stdout and stderr glued together, and this module already 

357 says why that is a diagnostic rather than a parse: every agent CLI writes progress and 

358 notices to stderr. Classifying on it is worse than parsing it. The whole transcript of 

359 a long run is hundreds of KB of the model's own prose about code — including, in the 

360 transcripts on disk here, git shas and file:line references carrying the digits 429, 

361 and reviews that discuss keel's own rate-limit handling by name. #1133 is one such run: 

362 the vendor said ``timeout waiting for response`` after 298s and keel reported 

363 ``rate-limit``, which tells an operator to wait out a quota window that does not exist. 

364 

365 So the signal is bounded and it is the vendor's, not the model's: 

366 

367 * **stream-json with a final ``result`` frame** — that frame's ``status`` and 

368 ``error``, and deliberately not its ``response``. stderr is *not* added: the frame 

369 already said why the run failed, and every agent CLI writes progress there. Prepending 

370 it unconditionally meant the same sentence this function refuses to read out of the 

371 transcript flipped the classification when it appeared on stderr instead. 

372 * **stream-json with no frame** — stderr alone. A crash or a stream truncated mid-write 

373 leaves nothing authoritative, and the stdout tail there is the model's own prose, so 

374 falling back to it would parse the transcript in exactly the case the frame was meant 

375 to protect. Both of these were found by the gate review; the first cut kept the tail. 

376 * **anything else** — stderr plus the tail of stdout, which is where a CLI that has no 

377 structured output prints why it stopped. 

378 

379 The frame's fields are **labelled** (``status: …``), not concatenated bare. A numeric 

380 ``status`` is the one ``429`` that really is a status code, and stripping it to a bare 

381 token put it out of reach of the very pattern that requires status-shaped context — 

382 self-defeating for the field this function chose to read. Also from the review. 

383 

384 ``output`` is the last resort, used only when a result carries neither separated 

385 stream. :func:`keel.runner._result` always fills both, so this is for a caller that 

386 built a :class:`~keel.runner.CommandResult` by hand — where the concatenation is the 

387 only signal there is, and refusing to read it would classify every such failure as a 

388 plain nonzero exit. 

389 """ 

390 if not (stderr or stdout): 

391 return _tail(output, limit) 

392 if stream_json: 

393 event = final_stream_event(stdout) 

394 if event is None: 

395 return stderr or "" 

396 parts = [ 

397 f"{field}: {event.get(field)}" 

398 for field in ("status", "error", "retryable") 

399 if event.get(field) is not None 

400 ] 

401 # A frame that carries no error text has said nothing about *why*; stderr is then 

402 # the only account there is, and discarding it would lose a real quota refusal. 

403 if not event.get("error"): 

404 parts.append(stderr or "") 

405 return "\n".join(part for part in parts if part) 

406 return "\n".join(part for part in (stderr or "", _tail(stdout, limit)) if part) 

407 

408 

409def execute( 

410 plan: RunPlan, 

411 *, 

412 _run: Callable[..., runner.CommandResult] | None = None, 

413 _opener=None, 

414 _env=os.environ, 

415 _now: Callable[[], float] = time.monotonic, 

416 _read: Callable[[str], str] = _read_text, 

417 _sleep: Callable[[float], None] = time.sleep, 

418) -> dict[str, Any]: 

419 """Run one plan and return the JSON contract. Never raises. 

420 

421 ``_run`` defaults to :func:`keel.runner.run_argv`, resolved **at call time** rather 

422 than bound as the default value: a default argument is evaluated once at import, so a 

423 test patching ``keel.runner.run_argv`` would still reach the real subprocess — which 

424 for this module means actually launching an agent CLI. 

425 """ 

426 # Rebound onto the parameter rather than a fresh local: #879's AST sweep reads the 

427 # keywords of every call spelled `_run(...)` / `_popen(...)`, and renaming the callee 

428 # would take this spawn site out of the rule's sight. 

429 _run = runner.run_argv if _run is None else _run 

430 started = _now() 

431 

432 def finish(**kwargs) -> dict[str, Any]: 

433 return result_document(plan, duration_s=round(_now() - started, 3), **kwargs) 

434 

435 if plan.prompt is not None: 

436 prompt = plan.prompt 

437 else: 

438 try: 

439 prompt = _read(plan.prompt_path) 

440 except OSError as exc: 

441 return finish(ok=False, error_code="no-prompt", error=str(exc)) 

442 if not prompt.strip(): 

443 source = plan.prompt_path or "--prompt" 

444 return finish( 

445 ok=False, 

446 error_code="no-prompt", 

447 error=f"{source} is empty; a delegate with no brief produces nothing", 

448 ) 

449 if plan.transport == "ollama": 

450 return _run_ollama(plan, prompt, finish, opener=_opener) 

451 if plan.transport == "api": 

452 return _run_api(plan, prompt, finish, env=_env, opener=_opener) 

453 return _run_argv(plan, prompt, finish, run=_run, _sleep=_sleep) 

454 

455 

456def _run_argv( 

457 plan: RunPlan, 

458 prompt: str, 

459 finish, 

460 *, 

461 run, 

462 _sleep: Callable[[float], None] = time.sleep, 

463) -> dict[str, Any]: 

464 """The ``cli`` and ``profile`` transports: one subprocess, prompt on stdin. 

465 

466 The delegate's answer is **stdout alone**. ``CommandResult.output`` glues stderr onto 

467 it, and every agent CLI writes progress, warnings and login notices there — so reading 

468 the concatenation lets a run that produced no answer come back ``ok: true`` with the 

469 noise as its "output", which downstream is a diff to apply or a review to post. The 

470 concatenation stays for the diagnostic tail of a *failure*, where both halves help. 

471 """ 

472 argv = list(plan.argv) 

473 if plan.stdin_mode == delegate.STDIN_STREAM_JSON: 

474 stdin_text: str | None = delegate.stream_json_frame(prompt) 

475 elif plan.stdin_mode == delegate.STDIN_TEXT: 

476 stdin_text = prompt 

477 else: 

478 # A profile's `prompt_mode: arg`. The planner cannot build this argv — it is 

479 # handed a path, never the text — so the prompt is appended here, at the one 

480 # seam that has both. 

481 argv.append(prompt) 

482 stdin_text = None 

483 max_quota_wait = getattr(plan, "quota_wait", 0.0) 

484 quota_waited = 0.0 

485 backoff = 2.0 

486 attempt = 1 

487 while True: 

488 result = run(argv, cwd=plan.cwd, timeout=plan.timeout, stdin_text=stdin_text) 

489 raw = result.stdout 

490 if plan.stdin_mode == delegate.STDIN_STREAM_JSON: 

491 text = delegate.parse_stream_json(raw) 

492 else: 

493 text = raw 

494 if result.timed_out: 

495 return finish( 

496 ok=False, 

497 exit_code=result.code, 

498 timed_out=True, 

499 error_code="timeout", 

500 error=f"{argv[0]} timed out after {plan.timeout}s", 

501 text=text, 

502 quota_wait=quota_waited, 

503 ) 

504 if getattr(result, "spawn_failed", False): 

505 # The runner's own OSError signal, not exit 127: a command that really ran can 

506 # exit 127 too, and reading the code alone made this branch unreachable. 

507 return finish( 

508 ok=False, 

509 exit_code=result.code, 

510 error_code="missing-binary", 

511 error=f"{argv[0]} could not be executed: {result.stderr or result.output}", 

512 quota_wait=quota_waited, 

513 ) 

514 if not result.ok: 

515 # Classified on the vendor's own error, never on the delegate's answer (#1133), 

516 # and timeout-before-quota: the two codes send an operator in opposite directions 

517 # — `rate-limit` says wait out a window, `timeout` says retry now, smaller or 

518 # longer — and a vendor that times out often says so in a sentence that also 

519 # mentions its limits. 

520 signal = failure_signal( 

521 stderr=result.stderr, 

522 stdout=result.stdout, 

523 output=result.output, 

524 stream_json=plan.stdin_mode == delegate.STDIN_STREAM_JSON, 

525 ) 

526 vendor_timeout = vendor_timed_out(signal) 

527 if vendor_timeout: 

528 code = "timeout" 

529 elif rate_limited(signal): 

530 code = "rate-limit" 

531 else: 

532 code = "nonzero-exit" 

533 if code == "rate-limit" and retryable_quota(signal): 

534 remaining = max_quota_wait - quota_waited 

535 if remaining > 0: 

536 delay = min(backoff, remaining) 

537 _sleep(delay) 

538 quota_waited += delay 

539 backoff *= 2 

540 attempt += 1 

541 continue 

542 return finish( 

543 ok=False, 

544 exit_code=result.code, 

545 # True for either bound: keel's wall-clock wrapper (exit 124) or the vendor's 

546 # own. The run did time out; which timer fired is in `exit_code` and `error`. 

547 timed_out=vendor_timeout, 

548 error_code=code, 

549 # The tail still shows both streams — a diagnostic wants everything a reader 

550 # might recognise, which is exactly what a decision must not read. 

551 error=f"{argv[0]} exited {result.code}: {_tail(result.output)}", 

552 text=text, 

553 quota_wait=quota_waited, 

554 ) 

555 if not text.strip(): 

556 return finish( 

557 ok=False, 

558 exit_code=result.code, 

559 error_code="empty-output", 

560 error=f"{argv[0]} exited 0 and produced no output", 

561 quota_wait=quota_waited, 

562 ) 

563 return finish(ok=True, exit_code=result.code, text=text, quota_wait=quota_waited) 

564 

565 

566def _run_api(plan: RunPlan, prompt: str, finish, *, env, opener) -> dict[str, Any]: 

567 """Hosted and OpenAI-compatible vendors: exactly one call through ``api_delegate``.""" 

568 request = plan.request or {} 

569 result = api_delegate.generate( 

570 request["vendor"], 

571 request["model"], 

572 prompt, 

573 endpoint=request.get("endpoint"), 

574 api_key_env=request.get("api_key_env"), 

575 max_tokens=request.get("max_tokens", api_delegate.DEFAULT_MAX_TOKENS), 

576 timeout=request.get("timeout", api_delegate.DEFAULT_TIMEOUT), 

577 extra_payload=request.get("extra_payload") or None, 

578 _env=env, 

579 _opener=opener, 

580 ) 

581 if not result.ok: 

582 # A `bad-response` can still carry the vendor's counts: the call was made. 

583 return finish( 

584 ok=False, error_code=result.error_code, error=result.error, usage=result.usage 

585 ) 

586 return finish(ok=True, text=result.text, usage=result.usage) 

587 

588 

589def _run_ollama(plan: RunPlan, prompt: str, finish, *, opener) -> dict[str, Any]: 

590 """The built-in local vendor: one POST to the hardcoded loopback ``/api/generate``.""" 

591 request = plan.request or {} 

592 body = json.dumps(delegate.ollama_payload(request["model"], prompt)).encode("utf-8") 

593 # The URL is delegate.OLLAMA_GENERATE_URL, a hardcoded constant — never config, never 

594 # the registry, never model or prompt content. 

595 http_request = urllib.request.Request( # nosec B310 

596 delegate.OLLAMA_GENERATE_URL, 

597 data=body, 

598 headers={"content-type": "application/json"}, 

599 method="POST", 

600 ) 

601 client = opener if opener is not None else api_delegate.build_http_only_opener() 

602 try: 

603 with client.open(http_request, timeout=plan.timeout) as response: 

604 raw = response.read(_MAX_RESPONSE_BYTES).decode("utf-8", errors="replace") 

605 status = getattr(response, "status", 200) 

606 except urllib.error.HTTPError as exc: 

607 code = "rate-limit" if exc.code == 429 else "http" 

608 return finish(ok=False, error_code=code, error=f"HTTP {exc.code}") 

609 except Exception as exc: 

610 # Deliberately broad, as in providerprobe: urllib raises URLError, http.client 

611 # exceptions (not OSError subclasses), socket timeouts and the address guard's 

612 # own error. An operator sees "the local server did not answer", not a traceback. 

613 return finish(ok=False, error_code="network", error=str(exc)) 

614 if not 200 <= status < 300: 

615 return finish(ok=False, error_code="http", error=f"HTTP {status}") 

616 try: 

617 data = json.loads(raw) 

618 except ValueError: 

619 return finish(ok=False, error_code="bad-response", error="response is not valid JSON") 

620 text = delegate.parse_ollama_response(data) 

621 if not text: 

622 return finish( 

623 ok=False, error_code="bad-response", error="response carried no completion text" 

624 ) 

625 return finish(ok=True, text=text) 

626 

627 

628def _tail(text: str, limit: int = 400) -> str: 

629 stripped = (text or "").strip() 

630 return stripped[-limit:] 

631 

632 

633# --------------------------------------------------------------------------- detach 

634 

635 

636def load_state(root: str | Path, run_id: str) -> dict[str, Any] | None: 

637 """One run's state document, or ``None`` when it does not exist / cannot be read.""" 

638 try: 

639 path = state_path(root, run_id) 

640 except RunIdError: 

641 return None 

642 try: 

643 return json.loads(path.read_text(encoding="utf-8")) 

644 except (OSError, ValueError): 

645 return None 

646 

647 

648def write_state(root: str | Path, record: dict[str, Any]) -> Path: 

649 """Persist a run's state atomically, so a reader never sees a torn document.""" 

650 path = state_path(root, record["run_id"]) 

651 workspace.write_text_atomic(path, json.dumps(record, indent=2, sort_keys=True)) 

652 return path 

653 

654 

655def list_runs(root: str | Path = ".") -> list[dict[str, Any]]: 

656 """Every readable run record under the state directory, oldest id first.""" 

657 directory = state_dir(root) 

658 records = [] 

659 try: 

660 names = sorted(p.stem for p in directory.glob("*.json")) 

661 except OSError: # pragma: no cover - glob on a readable parent does not fail 

662 return [] 

663 for name in names: 

664 record = run_record(root, name) 

665 if record is not None: 

666 records.append(record) 

667 return records 

668 

669 

670#: Extra wall-clock seconds a detached run gets beyond its own ``--timeout`` before 

671#: ``wait`` calls it abandoned. The child enforces the timeout itself and then has to 

672#: write its result; the grace is the room for that last write. 

673DEADLINE_GRACE_S = 60 

674 

675 

676def start_detached( 

677 argv: list[str], 

678 *, 

679 root: str | Path, 

680 run_id: str, 

681 cwd: str | None = None, 

682 timeout: int | None = None, 

683 provider: str | None = None, 

684 role: str | None = None, 

685 _popen=None, 

686 _clock: Callable[[], datetime.datetime] = _utc_now, 

687) -> dict[str, Any]: 

688 """Spawn ``argv`` as a background child and record it. Never raises. 

689 

690 The record is written **before** the spawn, so a ``wait`` issued immediately 

691 afterwards always finds a file, and the parent never writes it again: the pid goes to 

692 its own ``<run-id>.pid`` file (:func:`pid_path`). That is the fix for a lost update. 

693 A guarded read-check-write from the parent was still a race — the child's terminal 

694 record can land between the read and the write, and the parent would put ``running`` 

695 back over the result the caller is waiting for. Two files, one writer each, and no 

696 window at all. 

697 

698 ``deadline_at`` is stamped from the run's own ``--timeout`` plus 

699 :data:`DEADLINE_GRACE_S`. It is what stops a ``wait`` with no ``--timeout`` of its own 

700 from blocking forever when the child dies without recording anything — a 

701 ``SIGKILL``, an OOM kill, a reboot. 

702 

703 The child gets its own session where the platform has one, so killing the caller's 

704 process group does not take the delegate with it — surviving the caller is the whole 

705 point of ``--detach``. 

706 

707 ``_popen`` defaults to :class:`subprocess.Popen` resolved at call time, for the same 

708 reason :func:`execute`'s ``_run`` does: a default bound at import cannot be patched, 

709 and here the cost of missing the seam is a real agent CLI launched by a unit test. 

710 """ 

711 _popen = subprocess.Popen if _popen is None else _popen 

712 check_run_id(run_id) 

713 started = _clock() 

714 record: dict[str, Any] = { 

715 "schema_version": SCHEMA_VERSION, 

716 "run_id": run_id, 

717 "provider": provider, 

718 "role": role, 

719 "started_at": started.isoformat(), 

720 "timeout": timeout, 

721 "deadline_at": ( 

722 None 

723 if timeout is None 

724 else (started + datetime.timedelta(seconds=timeout + DEADLINE_GRACE_S)).isoformat() 

725 ), 

726 "status": "running", 

727 "argv": list(argv), 

728 "out_path": str(out_path(root, run_id)), 

729 "result": None, 

730 } 

731 # A conditional expression rather than an `if`: the assignment then executes on 

732 # every platform, so the 100 % line gate holds on the Windows runner too, where 

733 # `os.setsid` does not exist and sessions are not a thing. 

734 kwargs: dict[str, Any] = {"start_new_session": True} if hasattr(os, "setsid") else {} 

735 try: 

736 # Creating the directory and opening the log are I/O like any other: an 

737 # unwritable root (a read-only checkout, a root owned by another user) must come 

738 # back as the same fail-soft contract every other failure does, not as a 

739 # PermissionError traceback out of `keel delegate run --detach`. 

740 state_dir(root).mkdir(parents=True, exist_ok=True) 

741 write_state(root, record) 

742 # Same reason the log is opened "w": a reused run id must inherit nothing from 

743 # the run before it. See `clear_sidecars`. 

744 clear_sidecars(root, run_id) 

745 handle = out_path(root, run_id).open("w", encoding="utf-8") 

746 except OSError as exc: 

747 return _failed_spawn_record(root, record, exc) 

748 try: 

749 # argv is keel's own re-invocation (sys.executable -m keel), never a shell. 

750 child = _popen( 

751 list(argv), 

752 cwd=cwd, 

753 stdin=subprocess.DEVNULL, 

754 stdout=handle, 

755 stderr=subprocess.STDOUT, 

756 **kwargs, 

757 ) 

758 except OSError as exc: 

759 return _failed_spawn_record(root, record, exc) 

760 finally: 

761 handle.close() 

762 write_pid(root, run_id, child.pid) 

763 record["pid"] = child.pid 

764 return record 

765 

766 

767def _failed_spawn_record(root: str | Path, record: dict[str, Any], exc: OSError) -> dict[str, Any]: 

768 """Mark a run that never started, persisting the record when the disk allows it. 

769 

770 Best-effort on purpose: the failure being reported may itself be "this directory 

771 cannot be written", and a second write would raise the very traceback the caller is 

772 being spared. The returned document is the contract either way. 

773 """ 

774 record["status"] = "done" 

775 record["pid"] = None 

776 record["result"] = _spawn_failure(record, exc) 

777 with contextlib.suppress(OSError): 

778 write_state(root, record) 

779 return record 

780 

781 

782def _spawn_failure(record: dict[str, Any], exc: OSError) -> dict[str, Any]: 

783 return _detached_failure(record, code="spawn-failed", message=str(exc)) 

784 

785 

786def _detached_failure(record: dict[str, Any], *, code: str, message: str) -> dict[str, Any]: 

787 """A full return contract for a detached run that never produced one itself. 

788 

789 Same keys as :func:`result_document`, so ``keel delegate wait`` hands back one shape 

790 whether the delegate answered, failed, or vanished. 

791 """ 

792 return { 

793 "schema_version": SCHEMA_VERSION, 

794 "ok": False, 

795 "provider": record.get("provider"), 

796 "vendor": None, 

797 "model": None, 

798 "role": record.get("role"), 

799 "transport": None, 

800 "text": "", 

801 "exit_code": None, 

802 "duration_s": 0.0, 

803 "timed_out": code == "lost", 

804 "error_code": code, 

805 "error": message, 

806 "attribution": {}, 

807 "read_only": None, 

808 "read_only_backed": False, 

809 "effort_applied": False, 

810 "warnings": [], 

811 "usage": None, 

812 "quota-wait": 0.0, 

813 "run_id": record.get("run_id"), 

814 "out_path": record.get("out_path"), 

815 } 

816 

817 

818def record_planning_failure( 

819 root: str | Path, 

820 run_id: str, 

821 failure: dict[str, Any], 

822 *, 

823 provider: str | None, 

824 role: str | None, 

825 _clock: Callable[[], datetime.datetime] = _utc_now, 

826) -> dict[str, Any]: 

827 """Persist a ``--detach`` run whose plan could not be made, so ``status`` lists it (#1510). 

828 

829 There is no child and no pid: the record is born ``done`` with the failure document 

830 as its result, which is the shape :func:`_failed_spawn_record` already gives a run 

831 that never started. Best-effort like it, because an unwritable root must not turn a 

832 reported failure into a traceback. Returns the failure document carrying ``run_id``. 

833 """ 

834 check_run_id(run_id) 

835 now = _now_iso(_clock) 

836 result = {**failure, "run_id": run_id, "out_path": None} 

837 record = { 

838 "schema_version": SCHEMA_VERSION, 

839 "run_id": run_id, 

840 "provider": provider, 

841 "role": role, 

842 "started_at": now, 

843 "finished_at": now, 

844 "timeout": None, 

845 "deadline_at": None, 

846 "status": "done", 

847 "pid": None, 

848 "argv": [], 

849 "out_path": None, 

850 "result": result, 

851 } 

852 with contextlib.suppress(OSError): 

853 state_dir(root).mkdir(parents=True, exist_ok=True) 

854 # A reused run id must not pair this record with the previous run's pid or crash 

855 # marker (see `clear_sidecars`). 

856 write_state(root, record) 

857 # Only once the new record exists, so a failed write keeps the old markers. 

858 clear_sidecars(root, run_id) 

859 return result 

860 

861 

862def finish_detached( 

863 root: str | Path, 

864 run_id: str, 

865 result: dict[str, Any], 

866 *, 

867 _clock: Callable[[], datetime.datetime] = _utc_now, 

868) -> Path: 

869 """Record a detached run's result, keeping whatever the parent already wrote.""" 

870 record = load_state(root, run_id) or { 

871 "schema_version": SCHEMA_VERSION, 

872 "run_id": run_id, 

873 "pid": None, 

874 "started_at": _now_iso(_clock), 

875 "argv": [], 

876 "out_path": str(out_path(root, run_id)), 

877 } 

878 record["status"] = "done" 

879 record["finished_at"] = _now_iso(_clock) 

880 record["result"] = result 

881 return write_state(root, record) 

882 

883 

884#: How often :func:`wait` re-reads the state file. 

885DEFAULT_POLL_S = 0.5 

886 

887 

888def process_is_alive(pid: int) -> bool: 

889 """Is ``pid`` still a process we can see? Best-effort, never raises. 

890 

891 ``os.kill(pid, 0)`` sends no signal; it asks the kernel whether the pid exists and 

892 whether we may signal it. ``PermissionError`` therefore means **alive** (it exists and 

893 belongs to someone else), which is the opposite of what a bare ``except`` would say. 

894 

895 Two honest limits, both bounded by the run's own ``deadline_at`` rather than papered 

896 over: a child not yet reaped by its parent is a zombie and still answers "alive", and 

897 a recycled pid can answer "alive" for an unrelated process. Neither can make ``wait`` 

898 return a wrong *result* — only make it wait longer before giving up. 

899 """ 

900 if not hasattr(os, "kill"): # pragma: no cover - POSIX everywhere keel runs 

901 return True 

902 try: 

903 os.kill(pid, 0) 

904 except ProcessLookupError: 

905 return False 

906 except (PermissionError, OSError): 

907 return True 

908 return True 

909 

910 

911def _abandoned( 

912 record: dict[str, Any], 

913 *, 

914 alive: Callable[[int], bool], 

915 clock: Callable[[], datetime.datetime], 

916) -> str | None: 

917 """Why this still-``running`` record can never finish, or ``None`` while it might.""" 

918 pid = record.get("pid") 

919 if isinstance(pid, int) and not alive(pid): 

920 return ( 

921 f"the delegate process (pid {pid}) is gone and recorded no result; " 

922 f"its output is in {record.get('out_path')}" 

923 ) 

924 deadline = record.get("deadline_at") 

925 if isinstance(deadline, str) and deadline: 

926 try: 

927 when = datetime.datetime.fromisoformat(deadline) 

928 except ValueError: 

929 return None 

930 if clock() >= when: 

931 return ( 

932 f"the run passed its own deadline ({deadline}) without recording a " 

933 f"result; its output is in {record.get('out_path')}" 

934 ) 

935 return None 

936 

937 

938def _mark_crashed( 

939 root: str | Path, 

940 run_id: str, 

941 reason: str, 

942 *, 

943 clock: Callable[[], datetime.datetime], 

944) -> dict[str, Any] | None: 

945 """Note that a run vanished, in its own file, and return the composed record. 

946 

947 No read, no check, no write-back: the marker goes to :func:`crashed_path` and 

948 :func:`run_record` decides what it means. So a child that wrote ``done`` while the 

949 liveness probe was running still reports ``done`` — the marker is simply ignored — 

950 and nothing can be lost, because nothing is overwritten. 

951 """ 

952 with contextlib.suppress(OSError): 

953 workspace.write_text_atomic( 

954 crashed_path(root, run_id), 

955 json.dumps({"reason": reason, "finished_at": _now_iso(clock)}, indent=2), 

956 ) 

957 return run_record(root, run_id) 

958 

959 

960def reap_abandoned( 

961 root: str | Path = ".", 

962 *, 

963 _alive: Callable[[int], bool] | None = None, 

964 _clock: Callable[[], datetime.datetime] = _utc_now, 

965) -> list[str]: 

966 """Mark every run that can no longer finish, and return their ids. 

967 

968 The same liveness and deadline test :func:`wait` applies, run across the whole state 

969 directory. Without it a run only stops claiming to be ``running`` when somebody 

970 happens to ``wait`` on it — so ``keel delegate status``, the command an operator uses 

971 precisely because they are *not* waiting, would be the one view that never told the 

972 truth about a killed child. 

973 """ 

974 alive = process_is_alive if _alive is None else _alive 

975 reaped = [] 

976 for record in list_runs(root): 

977 if record.get("status") != "running": 

978 continue 

979 reason = _abandoned(record, alive=alive, clock=_clock) 

980 if reason is not None: 

981 _mark_crashed(root, record["run_id"], reason, clock=_clock) 

982 reaped.append(record["run_id"]) 

983 return reaped 

984 

985 

986def wait( 

987 root: str | Path, 

988 run_id: str, 

989 *, 

990 timeout: float | None = None, 

991 poll: float = DEFAULT_POLL_S, 

992 _sleep: Callable[[float], None] = time.sleep, 

993 _now: Callable[[], float] = time.monotonic, 

994 _clock: Callable[[], datetime.datetime] = _utc_now, 

995 _alive: Callable[[int], bool] | None = None, 

996) -> tuple[dict[str, Any] | None, str | None]: 

997 """Block until ``run_id`` finishes. Returns ``(result, error_code)``. 

998 

999 The state file is the authority — a *result* is only ever read from it, so it survives 

1000 the process that produced it, a new session, and a reboot. 

1001 

1002 Three ways this returns without a result, and all three are bounded: 

1003 

1004 * ``unknown-run`` — fails closed and immediately. An orchestrator that mistypes a run 

1005 id, or asks about a run from another checkout, must not get a wait that can only end 

1006 in a timeout. 

1007 * ``lost`` — the child is gone, or the run passed the deadline stamped at spawn, and 

1008 nothing was recorded. Without this a ``SIGKILL``ed child left ``running`` forever 

1009 and a ``wait`` with no ``--timeout`` blocked indefinitely. The record is marked 

1010 ``crashed`` so ``keel delegate status`` stops claiming it is running. 

1011 * ``timeout`` — the caller's own ``--timeout`` elapsed first. The run may still be 

1012 alive; nothing is marked. 

1013 

1014 A dead pid is re-checked against the file before being called lost, because the child 

1015 exits *after* writing its result and the two are observed in the other order often 

1016 enough to matter. 

1017 """ 

1018 deadline = None if timeout is None else _now() + timeout 

1019 alive = process_is_alive if _alive is None else _alive 

1020 while True: 

1021 record = run_record(root, run_id) 

1022 if record is None: 

1023 return None, "unknown-run" 

1024 status = record.get("status") 

1025 if status == "done": 

1026 return record.get("result"), None 

1027 if status == "crashed": 

1028 return record.get("result"), "lost" 

1029 reason = _abandoned(record, alive=alive, clock=_clock) 

1030 if reason is not None: 

1031 record = _mark_crashed(root, run_id, reason, clock=_clock) 

1032 if record is None: 

1033 return None, "unknown-run" 

1034 if record.get("status") == "done": 

1035 return record.get("result"), None 

1036 return record.get("result"), "lost" 

1037 if deadline is not None and _now() >= deadline: 

1038 return None, "timeout" 

1039 _sleep(poll) 

1040 

1041 

1042def planning_failure( 

1043 provider: str, 

1044 role: str, 

1045 *, 

1046 code: str, 

1047 message: str, 

1048) -> dict[str, Any]: 

1049 """The JSON contract for a run that never started — a plan that could not be made. 

1050 

1051 Same keys as :func:`result_document` with the unresolved halves null, so a caller 

1052 parses one shape whether the provider was unknown, the model token unsafe, or the 

1053 delegate simply exited nonzero. 

1054 """ 

1055 return { 

1056 "schema_version": SCHEMA_VERSION, 

1057 "ok": False, 

1058 "provider": provider, 

1059 "vendor": None, 

1060 "model": None, 

1061 "role": role, 

1062 "transport": None, 

1063 "text": "", 

1064 "exit_code": None, 

1065 "duration_s": 0.0, 

1066 "timed_out": False, 

1067 "error_code": code, 

1068 "error": message, 

1069 "attribution": {}, 

1070 "read_only": None, 

1071 "read_only_backed": False, 

1072 "effort_applied": False, 

1073 "warnings": [], 

1074 "usage": None, 

1075 "quota-wait": 0.0, 

1076 }