Coverage for src/keel/providers.py: 100%
224 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-02 20:26 +0000
1"""The provider vocabulary keel can dispatch to — pure data model + registry (#1011).
3keel advertises agent CLIs, hosted APIs, OpenAI-compatible endpoints, generic CLI
4profiles and local Ollama models as delegates, but nothing in core could tell an
5operator *which* of those are usable on this machine. This module owns the pure half
6of that answer: the :class:`Provider` record, the machine-level **provider registry**
7(``~/.keel/providers.yaml``), the name-clash rules, and :func:`plan_probes`, which
8lists what a probe should look at. The probing itself — subprocess, PATH, HTTP — is
9thin I/O in :mod:`keel.providerprobe`.
11Three sources feed one list, in this precedence:
13``profile`` a project ``knobs.delegate_profiles`` entry (checked into the repo);
14``registry`` a machine-level entry the operator owns and never commits;
15``builtin`` a vendor keel understands with no configuration at all.
17Which providers are usable is a property of the **machine and the person**, not of
18the project — one operator has ``claude``/``codex``/``agy`` logged in and no API key,
19another has only ``XAI_API_KEY``. The registry is where those facts live so they do
20not have to be committed to ``project.yaml``.
22Name resolution is fail-closed and mirrors the existing built-in shadowing rule
23(:func:`keel.config._validate_delegate_profiles`): a registry entry may not take the
24name of a built-in vendor or of a project profile. The project profile keeps working
25and **wins**; the clash is reported as a validation error naming both sources.
27Pure and deterministic: no wall-clock, no randomness, no subprocess, no network. The
28one file read is :func:`load_registry`, whose reader is injectable.
29"""
31from __future__ import annotations
33import os
34from collections.abc import Callable, Iterable, Mapping
35from dataclasses import dataclass, field
36from pathlib import Path
37from typing import Any
39from . import agents
40from . import config as cfg
41from . import yaml_helper as yaml
42from .api_delegate import OPENAI_COMPATIBLE, env_key_name
44#: Env var naming an alternative registry file; otherwise ``~/.keel/providers.yaml``.
45REGISTRY_ENV = "KEEL_PROVIDERS"
47#: Path of the default registry, relative to the operator's home directory.
48DEFAULT_REGISTRY_RELPATH = (".keel", "providers.yaml")
50#: Transports a provider can be reached over. ``cli`` is a local coding-agent binary,
51#: ``api`` a hosted HTTP endpoint keyed by an env var, ``local`` a model served on the
52#: operator's own hardware.
53TRANSPORTS = ("cli", "api", "local")
55#: Documented read-only flag per built-in CLI vendor. Presence here is what
56#: ``read_only_mode`` reports: keel cannot *enforce* read-only for an arbitrary CLI,
57#: so the capability says "a documented flag exists", never "writes are impossible".
58READ_ONLY_FLAGS: dict[str, str] = {
59 "claude": "--disallowed-tools",
60 "codex": "-s read-only",
61 "agy": "--sandbox",
62}
64#: The flag every built-in CLI vendor spells model selection with.
65DEFAULT_MODEL_ARG = "--model"
67#: Ollama's local tag listing. A **hardcoded loopback constant**, like every other URL
68#: keel dials: the probe never reaches an endpoint named by config or by the registry,
69#: which is what keeps the SSRF story of this module trivial.
70OLLAMA_TAGS_URL = "http://127.0.0.1:11434/api/tags"
72#: The command that serves the built-in ``ollama`` local vendor.
73OLLAMA_COMMAND = "ollama"
75#: Distinct review vendors a cross-vendor review panel needs. Two vendors reviewing is
76#: the property `review-vendors` reports; one vendor twice is one opinion twice.
77REVIEW_VENDOR_MINIMUM = 2
79#: Characters an environment variable *name* may contain.
80_ENV_NAME_OK = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_")
82#: Registry keys a provider entry may carry. Anything else is ignored (fail-soft) —
83#: an unknown key is far more likely a typo in a hand-written file than an intent.
84_ENTRY_KEYS = frozenset(
85 {
86 "transport",
87 "command",
88 "endpoint",
89 "api_key_env",
90 "model",
91 "model_arg",
92 "effort",
93 "review_args",
94 "vendor_label",
95 }
96)
99@dataclass(frozen=True)
100class Provider:
101 """One provider keel could dispatch to, from any of the three sources."""
103 name: str
104 #: The delegate vendor recorded in attribution: a built-in vendor name
105 #: (``claude``/``anthropic-api``/…), or ``cli``/``openai-compatible``/``local``
106 #: for a profile or registry entry.
107 vendor: str
108 transport: str
109 command: str | None = None
110 endpoint: str | None = None
111 #: The **name** of the env var holding the key — never the key.
112 api_key_env: str | None = None
113 model: str | None = None
114 #: Vendor-specific reasoning-effort selector, carried through for dispatch (#1012).
115 effort: str | None = None
116 #: Flags that make this provider a reviewer rather than an implementer. Their
117 #: presence is what ``read_only_mode`` reports for a profile/registry entry.
118 review_args: tuple[str, ...] = ()
119 #: ``builtin`` | ``profile`` | ``registry``.
120 source: str = "builtin"
121 #: How a model reaches a CLI (``<model_arg> <model>``); ``None`` when the provider
122 #: exposes no model selection.
123 model_arg: str | None = None
124 #: What ``agent:<vendor>`` should say for a profile or registry entry whose
125 #: :attr:`vendor` is the transport (``cli``) rather than the model's maker (#1129).
126 #: ``None`` — the default and the state of every built-in — leaves the label alone.
127 vendor_label: str | None = None
129 def label_vendor(self) -> str:
130 """The vendor this provider's attribution names: ``vendor_label`` when set.
132 Built-ins never set it, so they keep naming themselves. A ``cli`` entry that
133 does set it stops sharing one ``agent:cli`` with every other ``cli`` entry —
134 which is what ``review-vendor-distinctness`` needs in order to tell two
135 reviewers apart.
136 """
137 return self.vendor_label or self.vendor
139 def capabilities(self) -> dict[str, bool]:
140 """What this provider can do — the three facts a dispatcher needs.
142 ``tools``: can it run git/PR steps itself? Only a CLI can; an ``api`` or
143 ``local`` provider generates text under keel's no-tools contract, where the
144 orchestrator performs every mutation.
146 ``read_only_mode``: is there a documented way to run it without write tools?
147 A built-in CLI has a flag (:data:`READ_ONLY_FLAGS`); a profile or registry
148 entry has it when the operator set ``review_args``. A provider with no tools
149 at all has no such flag, and reports ``False`` — read ``tools`` first.
151 ``model_selection``: can the caller choose the model?
152 """
153 return {
154 "tools": self.transport == "cli",
155 "read_only_mode": self._read_only_mode(),
156 "model_selection": self.transport in ("api", "local")
157 or bool(self.model_arg or self.model),
158 }
160 def _read_only_mode(self) -> bool:
161 if self.source == "builtin":
162 return self.name in READ_ONLY_FLAGS
163 return bool(self.review_args)
165 def as_dict(self) -> dict[str, object]:
166 """JSON-stable record (no secrets: ``api_key_env`` is a name, never a value)."""
167 return {
168 "name": self.name,
169 "vendor": self.vendor,
170 "transport": self.transport,
171 "source": self.source,
172 "command": self.command,
173 "endpoint": self.endpoint,
174 "api_key_env": self.api_key_env,
175 "model": self.model,
176 "effort": self.effort,
177 "review_args": list(self.review_args),
178 "capabilities": self.capabilities(),
179 }
182@dataclass(frozen=True)
183class Registry:
184 """The machine-level provider registry, already parsed and fail-soft."""
186 path: str
187 present: bool = False
188 providers: tuple[Provider, ...] = ()
189 #: Fail-soft parse complaints. A malformed registry never raises: keel degrades to
190 #: the built-ins and says why.
191 warnings: tuple[str, ...] = field(default_factory=tuple)
193 def names(self) -> tuple[str, ...]:
194 return tuple(provider.name for provider in self.providers)
197def registry_path(*, env: Mapping[str, str] | None = None, home: str | Path | None = None) -> Path:
198 """Where the registry lives: ``$KEEL_PROVIDERS``, else ``~/.keel/providers.yaml``.
200 ``home`` is resolved from the **injected** environment before ``Path.home()`` so a
201 test can place a registry without touching the real ``$HOME`` — and so an operator
202 who moved ``HOME`` gets the registry their other tools see.
203 """
204 env = os.environ if env is None else env
205 override = (env.get(REGISTRY_ENV) or "").strip()
206 if override:
207 return Path(override).expanduser()
208 if home is None:
209 home = env.get("HOME") or None
210 base = Path(home).expanduser() if home is not None else Path.home()
211 return base.joinpath(*DEFAULT_REGISTRY_RELPATH)
214def _read_text(path: Path) -> str:
215 return path.read_text(encoding="utf-8")
218def load_registry(
219 path: str | Path | None = None,
220 *,
221 env: Mapping[str, str] | None = None,
222 _read: Callable[[Path], str] = _read_text,
223) -> Registry:
224 """Load and parse the registry. **Never raises** — every failure is a warning.
226 A missing file is not a failure: no registry means no machine-level providers,
227 which is the state of every machine that has not opted in.
228 """
229 env = os.environ if env is None else env
230 target = Path(path) if path is not None else registry_path(env=env)
231 try:
232 text = _read(target)
233 except FileNotFoundError:
234 return Registry(path=str(target), present=False)
235 except OSError as exc:
236 return Registry(
237 path=str(target),
238 present=True,
239 warnings=(f"{target}: cannot be read ({exc})",),
240 )
241 try:
242 data = yaml.load(text)
243 except yaml.YAMLError as exc:
244 return Registry(
245 path=str(target),
246 present=True,
247 warnings=(f"{target}: is not valid YAML ({exc})",),
248 )
249 return parse_registry(data, path=str(target), env=env)
252def parse_registry(
253 data: Any,
254 *,
255 path: str,
256 env: Mapping[str, str] | None = None,
257) -> Registry:
258 """Turn a loaded registry document into providers + fail-soft warnings (pure)."""
259 env = os.environ if env is None else env
260 if data is None:
261 return Registry(path=path, present=True)
262 if not isinstance(data, dict):
263 return Registry(
264 path=path,
265 present=True,
266 warnings=(f"{path}: expected a mapping with a 'providers:' key",),
267 )
268 entries = data.get("providers")
269 if entries is None:
270 return Registry(
271 path=path,
272 present=True,
273 warnings=(f"{path}: has no 'providers:' mapping — nothing to register",),
274 )
275 if not isinstance(entries, dict):
276 return Registry(
277 path=path,
278 present=True,
279 warnings=(f"{path}: 'providers' must be a mapping of name -> entry",),
280 )
281 providers: list[Provider] = []
282 warnings: list[str] = []
283 for name in sorted(entries, key=str):
284 provider, entry_warnings = _parse_entry(name, entries[name], path=path, env=env)
285 warnings.extend(entry_warnings)
286 if provider is not None:
287 providers.append(provider)
288 return Registry(
289 path=path,
290 present=True,
291 providers=tuple(providers),
292 warnings=tuple(warnings),
293 )
296def _parse_entry(
297 name: Any,
298 entry: Any,
299 *,
300 path: str,
301 env: Mapping[str, str],
302) -> tuple[Provider | None, list[str]]:
303 """One registry entry -> a provider, or ``None`` plus the reasons it was skipped."""
304 where = f"{path}: provider {name!r}"
305 if not isinstance(name, str) or not name.strip():
306 return None, [
307 f"{path}: provider name {name!r} is not a non-empty string — quote the key "
308 "(YAML reads a bare on/off/yes/no as a boolean and a bare number as an int)"
309 ]
310 if ":" in name:
311 return None, [
312 f"{where}: a name may not contain ':' — --delegate splits on the first colon "
313 "to separate the provider from a per-run model, so this could never be selected"
314 ]
315 if not isinstance(entry, dict):
316 return None, [f"{where}: entry must be a mapping of fields"]
317 unknown = sorted(set(entry) - _ENTRY_KEYS)
318 warnings = [f"{where}: ignoring unknown field(s) {', '.join(unknown)}"] if unknown else []
319 transport = entry.get("transport")
320 if transport not in TRANSPORTS:
321 warnings.append(f"{where}: unknown transport {transport!r}; valid: {', '.join(TRANSPORTS)}")
322 return None, warnings
323 review_args, arg_warnings = _string_tuple(entry.get("review_args"), where=where)
324 warnings.extend(arg_warnings)
325 # Fail-soft like every other registry rule: a bad label is dropped with a warning
326 # rather than skipping the whole entry. The provider still works; it just keeps the
327 # generic label it had before, which is the state this field improves on.
328 # The **raw** value, not `_text(...)`: `_text` returns None for a bool, an int or a
329 # list, and `vendor_label_errors(None)` is silence — so exactly the YAML case the
330 # project-profile validator reports (an unquoted `on:`/`2:` resolving to a bool/int)
331 # was swallowed here without a warning. Both gate seats found it.
332 raw_label = entry.get("vendor_label")
333 label_errors = cfg.vendor_label_errors(raw_label, where=where)
334 vendor_label = _text(raw_label)
335 if label_errors:
336 warnings.extend(f"{message}; ignoring it" for message in label_errors)
337 vendor_label = None
338 fields = {
339 "name": name,
340 "transport": transport,
341 "model": _text(entry.get("model")),
342 "effort": _text(entry.get("effort")),
343 "review_args": review_args,
344 "vendor_label": vendor_label,
345 "source": "registry",
346 }
347 if transport == "api":
348 endpoint = _text(entry.get("endpoint"))
349 issues = cfg.endpoint_issues(endpoint, where=where, env=env)
350 remote_gate = any(cfg.ALLOW_REMOTE_ENDPOINT_ENV in issue for issue in issues)
351 key_env = _text(entry.get("api_key_env"))
352 issues.extend(_api_key_env_issues(key_env, where=where))
353 if issues:
354 warnings.extend(issues)
355 if remote_gate:
356 # Say it in the registry's own voice. The endpoint rules are the
357 # project profile's, and their message says "not in this file" — which
358 # in a home-directory registry reads as though some *other* file could
359 # grant it. None can: the opt-in is environment-only precisely because
360 # a file must not be able to widen its own reach.
361 warnings.append(
362 f"{where}: not registered. Reaching your own remote endpoint needs "
363 f"{cfg.ALLOW_REMOTE_ENDPOINT_ENV}=1 exported in your shell — no "
364 "registry entry can grant itself that"
365 )
366 return None, warnings
367 return (
368 Provider(vendor=OPENAI_COMPATIBLE, endpoint=endpoint, api_key_env=key_env, **fields),
369 warnings,
370 )
371 command = _text(entry.get("command"))
372 if not command:
373 warnings.append(
374 f"{where}: transport {transport!r} requires a non-empty 'command' — the "
375 "executable keel runs"
376 )
377 return None, warnings
378 vendor = "cli" if transport == "cli" else "local"
379 model_arg = _text(entry.get("model_arg")) or (DEFAULT_MODEL_ARG if transport == "cli" else None)
380 return Provider(vendor=vendor, command=command, model_arg=model_arg, **fields), warnings
383def _api_key_env_issues(key_env: str | None, *, where: str) -> list[str]:
384 """Rules for the **name** of a registry entry's API-key env var.
386 The denylist is shared with ``knobs.delegate_profiles``: a high-privilege system
387 credential (``GITHUB_TOKEN``, ``AWS_*``, ``SSH_AUTH_SOCK``, …) may never become an
388 ``Authorization`` header, wherever the entry was written.
390 The project profile's *allowlist* is deliberately **not** applied here. That list
391 exists because ``project.yaml`` is committed and reviewed by people other than its
392 author — the threat model is a config an attacker influenced through a pull
393 request. This file is not committed and not shared: it sits in the operator's own
394 home directory at the same trust level as their shell profile, and #1011's own
395 example (an operator whose only key is ``XAI_API_KEY``) is exactly the case an
396 allowlist of seven vendor names would refuse. The probe reads presence only — it
397 never reads the value, and never sends it anywhere.
398 """
399 if not key_env:
400 return [
401 f"{where}: transport 'api' requires 'api_key_env' — the *name* of the "
402 "environment variable holding the key, never the key itself"
403 ]
404 if not _ENV_NAME_OK.issuperset(key_env) or key_env[0].isdigit():
405 return [
406 f"{where}: api_key_env {key_env!r} is not a valid environment variable name "
407 "(letters, digits, underscore; not starting with a digit) — this field takes "
408 "a name, not a key"
409 ]
410 if key_env.upper() in cfg.BLOCKED_ENV_KEY_NAMES or key_env.upper().startswith(
411 cfg.BLOCKED_ENV_PREFIXES
412 ):
413 return [
414 f"{where}: api_key_env {key_env!r} names a high-privilege system credential "
415 "and may not be used as a provider key"
416 ]
417 return []
420def _text(value: Any) -> str | None:
421 """A non-blank string, or ``None`` (so a blank field reads as unset)."""
422 return value.strip() if isinstance(value, str) and value.strip() else None
425def _string_tuple(value: Any, *, where: str) -> tuple[tuple[str, ...], list[str]]:
426 if value is None:
427 return (), []
428 if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
429 return (), [f"{where}: 'review_args' must be a list of strings — ignored"]
430 return tuple(value), []
433def builtin_providers() -> tuple[Provider, ...]:
434 """Every vendor keel understands with no configuration at all."""
435 providers = [
436 Provider(
437 name=vendor,
438 vendor=vendor,
439 transport="cli",
440 command=vendor,
441 model_arg=DEFAULT_MODEL_ARG,
442 source="builtin",
443 )
444 for vendor in agents.CLI_VENDORS
445 ]
446 providers.extend(
447 Provider(
448 name=vendor,
449 vendor=vendor,
450 transport="local",
451 command=OLLAMA_COMMAND,
452 endpoint=OLLAMA_TAGS_URL,
453 source="builtin",
454 )
455 for vendor in agents.LOCAL_VENDORS
456 )
457 providers.extend(
458 Provider(
459 name=vendor,
460 vendor=vendor,
461 transport="api",
462 api_key_env=env_key_name(vendor),
463 source="builtin",
464 )
465 for vendor in agents.API_VENDORS
466 )
467 return tuple(providers)
470def profile_providers(config) -> tuple[Provider, ...]:
471 """Providers from a project's ``knobs.delegate_profiles`` (empty when no config)."""
472 if config is None:
473 return ()
474 providers = []
475 for name in sorted(config.knobs.delegate_profiles):
476 profile = config.knobs.delegate_profiles[name]
477 api = profile.vendor == OPENAI_COMPATIBLE
478 providers.append(
479 Provider(
480 name=name,
481 vendor=profile.vendor,
482 transport="api" if api else "cli",
483 command=profile.command,
484 endpoint=profile.endpoint,
485 api_key_env=profile.api_key_env,
486 model=profile.model,
487 review_args=profile.role_args(review=True) if profile.review_args else (),
488 source="profile",
489 model_arg=None if api else (profile.model_arg or None),
490 vendor_label=profile.vendor_label,
491 )
492 )
493 return tuple(providers)
496def registry_clashes(registry: Registry, config) -> list[str]:
497 """Name clashes between the registry and the built-ins / this project's profiles.
499 A registry entry may not shadow either. The message names **both** sources, so an
500 operator editing a file in ``$HOME`` learns which repository file it collided with
501 rather than being told its own name is taken.
502 """
503 builtins = agents.BUILTIN_DELEGATE_VENDORS
504 profiles = {} if config is None else config.knobs.delegate_profiles
505 errors: list[str] = []
506 for name in registry.names():
507 if name in builtins:
508 errors.append(
509 f"{registry.path}: provider {name!r} shadows the built-in delegate vendor "
510 f"{name!r}; built-ins always win and may not be redefined "
511 f"({', '.join(builtins)}) — rename the registry entry"
512 )
513 elif name in profiles:
514 errors.append(
515 f"{registry.path}: provider {name!r} clashes with the project profile "
516 f"knobs.delegate_profiles.{name}; the project profile wins — rename the "
517 "registry entry"
518 )
519 return errors
522def plan_probes(config, registry: Registry | None = None) -> tuple[Provider, ...]:
523 """Everything a probe should look at, in a deterministic order.
525 Built-ins first (dispatch order: CLI, local, hosted API), then this project's
526 profiles, then the machine-level registry. A registry entry whose name clashes is
527 **dropped** here rather than silently overriding: precedence is
528 *built-in > project profile > registry*, and :func:`registry_clashes` reports the
529 clash so the operator sees why the entry is missing. That is the same order
530 :func:`keel.delegate.resolve_provider` dispatches on, and the same invariant
531 :func:`keel.agents.resolve_delegate_profile` states — a built-in vendor always wins
532 and may not be redefined, by a committed profile or by a file in ``$HOME``. Below the
533 built-ins the project's own profiles win over the machine-level registry, so a
534 repository can pin the provider its team shares.
535 """
536 registry = Registry(path="") if registry is None else registry
537 providers = list(builtin_providers())
538 profiles = profile_providers(config)
539 providers.extend(profiles)
540 taken = {provider.name for provider in providers}
541 providers.extend(provider for provider in registry.providers if provider.name not in taken)
542 return tuple(providers)
545def tool_capable(providers: Iterable[Provider]) -> tuple[str, ...]:
546 """Names of the providers that can run tools (i.e. drive git/PR steps themselves)."""
547 return tuple(p.name for p in providers if p.capabilities()["tools"])
550def distinct_vendors(providers: Iterable[Provider]) -> tuple[str, ...]:
551 """Distinct vendors across ``providers``, in first-seen order.
553 A review panel's independence is a property of *vendors*, not of provider entries:
554 two profiles that both shell out to the same CLI are one vendor, and one opinion.
555 """
556 return tuple(dict.fromkeys(p.vendor for p in providers))
559#: Characters a listed model id may contain. Wider than
560#: :data:`keel.agents._MODEL_TOKEN_OK` on purpose — this parses a *listing* keel only
561#: displays (``qwen2.5:7b``, ``anthropic/claude-opus-4-5``), and a token from here is
562#: still re-validated by ``agents.is_safe_model_token`` before it can reach an argv.
563_LISTED_MODEL_OK = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._:-/")
565#: Cap on a parsed model listing. A CLI that decides to print its whole catalogue
566#: (or something that is not a listing at all) must not turn a doctor row into a wall.
567MAX_LISTED_MODELS = 100
570def parse_model_lines(text: str) -> tuple[str, ...]:
571 """Best-effort model ids out of a CLI listing (``agy models``).
573 No agent CLI promises a machine-readable listing, so this reads the shape they all
574 share — one model per line, possibly bulleted, possibly followed by columns of
575 description — and keeps the first token of each line that could be a model id.
576 Headers, rules and prose lines drop out because they carry characters no model id
577 has. Pure, deterministic, and never raises: an unreadable listing yields ``()``,
578 which reads as "this provider exposes no model list", not as an error.
579 """
580 seen: dict[str, None] = {}
581 for raw in (text or "").splitlines():
582 line = raw.strip()
583 if not line or line.startswith("#") or line.endswith(":"):
584 continue
585 line = line.lstrip("-*\u2022 \t")
586 if not line:
587 continue
588 token = line.split()[0]
589 if not _LISTED_MODEL_OK.issuperset(token) or not any(c.isalnum() for c in token):
590 continue
591 seen.setdefault(token, None)
592 if len(seen) >= MAX_LISTED_MODELS:
593 break
594 return tuple(seen)
597def parse_tag_payload(data: Any) -> tuple[str, ...]:
598 """Model names out of an Ollama ``/api/tags`` payload (``()`` when malformed)."""
599 if not isinstance(data, dict):
600 return ()
601 models = data.get("models")
602 if not isinstance(models, list):
603 return ()
604 names: dict[str, None] = {}
605 for entry in models:
606 if not isinstance(entry, dict):
607 continue
608 name = entry.get("name") or entry.get("model")
609 if isinstance(name, str) and name.strip():
610 names.setdefault(name.strip(), None)
611 if len(names) >= MAX_LISTED_MODELS:
612 break
613 return tuple(names)