Coverage for src/keel/team.py: 100%
513 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 19:17 +0000
1"""``knobs.team`` — the per-role / per-tier provider policy (#1014).
3Before this module keel could name *an* implementer (``--delegate``,
4``knobs.implementer_agents``) and *one* reviewer vendor for all N reviewers
5(``--review-delegate``). It could not express the team a real engineering org runs:
6*this role implements with provider X at effort E, every implementation gets one gate
7review from a different vendor, tier-2 gets two reviewers from two vendors, tier-3
8convenes the jury as the review panel.*
10``knobs.team`` is that policy, and this module is its pure half:
12* :class:`Seat` — one occupied chair: a provider, optionally a model and an effort.
13* :class:`TeamPolicy` — the parsed ``knobs.team`` block.
14* :func:`parse_team` / :func:`canonical` — YAML in, typed policy out, and back to the
15 canonical dict that feeds ``config_hash`` and the published contract.
16* :func:`team_issues` — the semantic validation ``keel validate`` runs.
17* :func:`resolve_assignment` — the deterministic answer to *who runs this ship*, which
18 ``keel plan``/``keel ship`` render as ``assignment`` so any host runs the same team.
20A ``provider`` names an entry the provider registry resolves — a built-in vendor, a
21``knobs.delegate_profiles`` entry, or a machine-level ``~/.keel/providers.yaml`` entry —
22with two reserved spellings:
24``subagent:<name>`` the pre-#1014 Claude-subagent semantics ``implementer_agents``
25 values carried, kept explicit so a seat can no longer be read as
26 both a vendor and a subagent name;
27``implementer`` valid in ``fix.provider`` and ``gate.distinct_from`` only: *the
28 provider that implemented this change*, whatever it resolved to.
30Pure and deterministic: no wall-clock, no randomness, no I/O, and exactly one keel
31import — the leaf :mod:`keel.vocab`, which owns the provider and effort vocabulary
32:func:`team_issues` validates against. That vocabulary used to live next to dispatch in
33:mod:`keel.agents` / :mod:`keel.delegate`, which both import :mod:`keel.config`, which
34imports this module for the policy type; validation therefore reached it through a
35function-local import. Moving the vocabulary instead of the import is what removed the
36cycle rather than hiding it (#1050).
37"""
39from __future__ import annotations
41from collections.abc import Iterable, Mapping, Sequence
42from dataclasses import dataclass, field
43from typing import Any
45from .vocab import BUILTIN_DELEGATE_VENDORS, EFFORTS, supports_effort
47#: Prefix that keeps the pre-#1014 Claude-subagent semantics of ``implementer_agents``.
48#: ``subagent:backend-developer`` is a subagent of the host agent, not a vendor keel
49#: dispatches to — which is exactly the ambiguity #1014 opened against.
50SUBAGENT_PREFIX = "subagent:"
52#: Reserved provider value: *whoever implemented this change*. Valid in ``fix.provider``
53#: and ``gate.distinct_from``; anywhere else it would name a provider that does not exist.
54IMPLEMENTER = "implementer"
56#: ``review.by_tier.<n>: jury`` — the cross-vendor panel **is** the review for that tier.
57JURY_PANEL = "jury"
59#: Risk tiers ``review.by_tier`` may key on, as **strings**. A JSON-Schema property name
60#: is a string, and YAML reads a bare ``1:`` as an integer key, so the keys are quoted
61#: (``"1":``) and :func:`team_issues` says so when they are not.
62TIERS = ("1", "2", "3")
64#: How an enabled jury gates. ``gating`` blocks the merge on a blocking verdict;
65#: ``advisory`` reports and never gates.
66JURY_MODES = ("gating", "advisory")
68#: What happens on a jury-panel tier when the panel cannot be staffed *here* (#1066).
69#: ``fallback`` seats a host bench of the same size in its place; ``block`` refuses the
70#: run. A **configured allowance, not a flag**: #1014 round 3 settled that an operator's
71#: preference may not take the panel off, and this does not reopen it — availability is
72#: measured by :func:`keel.juryavail.assess` from the probe keel already runs, never
73#: asserted on a command line.
74JURY_ON_UNAVAILABLE = ("fallback", "block")
76#: The allowance a project that never names one gets. ``fallback`` is the sensible answer
77#: for the single-maintainer case the panel would otherwise wall off; a project whose
78#: product claim *is* cross-vendor review says ``block`` and keeps today's strictness.
79JURY_ON_UNAVAILABLE_DEFAULT = "fallback"
81#: ``reviewer_source`` on a bench seated because the panel could not be. Named rather than
82#: derived from the tier's config path so a reader of the published contract can tell a
83#: fallback bench from a tier that simply never had a panel.
84JURY_FALLBACK_SOURCE = "jury-fallback"
86#: The binary s7 dispatches to convene the panel. Here rather than in
87#: :mod:`keel.juryavail` — which re-exports it — only because :func:`refusal_message` names
88#: it and that module imports this one, never the reverse. It is the same string as
89#: :data:`JURY_PANEL` and deliberately a separate name: one is a *review policy value*, the
90#: other is a *command*, and a project could not rename either without the other.
91JURY_RUNNER_COMMAND = "jury"
94class JuryUnavailableError(RuntimeError):
95 """``on_unavailable: block`` and the panel cannot be staffed — the run refuses.
97 Raised from :func:`_review_seats`, which is reached from exactly one function
98 (:func:`resolve_assignment`), and caught centrally in :func:`keel.cli.main`. Every
99 review-aware surface therefore refuses identically rather than each carrying its own
100 near-copy of the check.
102 **Raised where the panel is resolved, not where it is measured** (#1068). The probe
103 (:func:`keel.providerprobe.jury_availability`) answers one question about the
104 *machine*, and a surface may resolve several benches from that one answer: a swarm
105 scores each cluster's tier while it partitions and asks the probe the widest question
106 it can — *could any tier or band make the panel the review* — before any cluster
107 exists. Refusing there refused a plan of entirely non-panel clusters on an unstaffable
108 host, because "some tier could name the panel" is not "this work does". Here the
109 question is already narrowed to the cluster or command being staffed, so the run
110 refuses exactly when the panel really is its review.
112 Re-exported as ``keel.juryavail.JuryUnavailableError``: this module may not import
113 that one (:mod:`keel.juryavail` imports the policy vocabulary from here), and the
114 refusal has to live on the resolver's side of that edge.
115 """
118#: ``by_difficulty`` bands, lightest first. A band names the bench that staffs work of
119#: that weight; :func:`keel.swarm.score_difficulty` decides which band a cluster is, and
120#: the same resolver seats it. Unlike a tier — which is *how risky the change is* and is
121#: read off the files it touches — a band is *how much work it is*, which is what decides
122#: whether the strong implementer is worth spending on it (#1017).
123DIFFICULTY_BANDS = ("easy", "standard", "hard")
125#: Distinct vendors a jury needs before its verdict can gate, when the policy is silent.
126DEFAULT_MIN_VENDORS = 2
128#: Default agent when neither the policy nor a flag names one: the host agent driving
129#: the run. Mirrors :data:`keel.agents.HOST_DEFAULT`, which cannot be imported here.
130HOST_DEFAULT = "claude"
132#: Vendors that spell reasoning effort as a **model suffix** rather than as its own
133#: argument (``gemini-3.8-flash-high``). An effort on such a seat with no ``model``
134#: beside it has nothing to attach to, so :func:`team_issues` rejects the pair — and
135#: :mod:`keel.wizard` reads the same tuple rather than re-deriving the rule, so the
136#: wizard cannot offer an effort it would then be told to take back.
137EFFORT_MODEL_SUFFIX_VENDORS = ("agy",)
140def effort_needs_model(vendor: str | None) -> bool:
141 """True when an ``effort`` on this vendor's seat requires a ``model`` beside it."""
142 return vendor in EFFORT_MODEL_SUFFIX_VENDORS
145@dataclass(frozen=True)
146class Seat:
147 """One chair on the team: who sits in it, on which model, at which effort."""
149 provider: str
150 model: str | None = None
151 effort: str | None = None
152 #: ``gate`` only: the seat this one may not duplicate (``implementer``).
153 distinct_from: str | None = None
155 @property
156 def kind(self) -> str:
157 """``subagent`` | ``alias`` | ``provider`` — how the provider value is read."""
158 if self.provider.startswith(SUBAGENT_PREFIX):
159 return "subagent"
160 if self.provider == IMPLEMENTER:
161 return "alias"
162 return "provider"
164 @property
165 def name(self) -> str:
166 """The bare name: a subagent seat without its prefix, anything else verbatim."""
167 if self.kind == "subagent":
168 return self.provider[len(SUBAGENT_PREFIX) :]
169 return self.provider
171 def as_dict(self, *, source: str | None = None, slot: str | None = None) -> dict[str, Any]:
172 """JSON-stable seat record; ``source``/``slot`` are the assignment's annotations."""
173 record: dict[str, Any] = {
174 "provider": self.provider,
175 "name": self.name,
176 "kind": self.kind,
177 "model": self.model,
178 "effort": self.effort,
179 }
180 if self.distinct_from is not None:
181 record["distinct_from"] = self.distinct_from
182 if source is not None:
183 record["source"] = source
184 if slot is not None:
185 record["slot"] = slot
186 return record
189@dataclass(frozen=True)
190class Bench:
191 """A named bench: who leads it, who implements on it, who reviews, at what effort.
193 One type, two tables. ``team.by_difficulty`` picks a bench from a cluster's *scored
194 difficulty*; ``team.profiles`` lets an operator pick one by name (``--team``). They
195 are the same thing — "staff this piece of work from this bench" — so they resolve
196 through one code path rather than two that can disagree.
197 """
199 lead: Seat | None = None
200 implement: Seat | None = None
201 #: Reviewer seats, or the literal ``"jury"``; ``None`` leaves the tier's policy alone.
202 review: tuple[Seat, ...] | str | None = None
203 #: Effort for this bench's implementer, when the seat does not name one itself.
204 effort: str | None = None
207@dataclass(frozen=True)
208class TeamPolicy:
209 """A parsed ``knobs.team`` block. ``configured`` is False when there is none."""
211 configured: bool = False
212 implement: Seat | None = None
213 implement_by_role: Mapping[str, Seat] = field(default_factory=dict)
214 gate: Seat | None = None
215 #: ``review.default`` — seats, or the literal ``"jury"``.
216 review: tuple[Seat, ...] | str | None = None
217 #: Tier (``"1"``/``"2"``/``"3"``) -> reviewer seats, or the literal ``"jury"``.
218 review_by_tier: Mapping[str, tuple[Seat, ...] | str] = field(default_factory=dict)
219 jury_mode: str | None = None
220 jury_min_vendors: int | None = None
221 #: ``team.jury.on_unavailable`` — what a jury-panel tier does when the panel cannot be
222 #: staffed here. ``None`` is unset and resolves to
223 #: :data:`JURY_ON_UNAVAILABLE_DEFAULT`; kept tri-state at the config boundary so an
224 #: explicit ``fallback`` stays distinguishable from silence for ``config_hash``.
225 jury_on_unavailable: str | None = None
226 #: ``team.jury.comment_consensus`` — the approving review verdicts a ``COMMENT``
227 #: consensus needs to satisfy ``jury-verdict`` (#1570). ``None`` is off, which is
228 #: the default; when on it is at least :data:`COMMENT_CONSENSUS_MIN_APPROVALS`.
229 jury_comment_consensus: int | None = None
230 #: ``team.jury.base_facts`` — the PR body carries a ``## Base facts`` section and the
231 #: jury is handed it ahead of the diff (#1570). ``None`` is unset, which is off.
232 jury_base_facts: bool | None = None
233 fix: Seat | None = None
234 #: The seat that coordinates a batch of ships — the team lead a swarm cluster or a
235 #: work block reports through. Defaults to the host agent driving the run.
236 lead: Seat | None = None
237 #: Difficulty band (:data:`DIFFICULTY_BANDS`) -> the bench that staffs work of that
238 #: weight.
239 by_difficulty: Mapping[str, Bench] = field(default_factory=dict)
240 #: Operator-selectable benches (``--team <profile>``).
241 profiles: Mapping[str, Bench] = field(default_factory=dict)
243 def benches_for(
244 self, *, difficulty: str | None = None, profile: str | None = None
245 ) -> tuple[tuple[Bench, str], ...]:
246 """Benches that apply to this run, most specific first, each with its config path.
248 A named ``--team`` profile is an operator saying *this bench, for this batch*, so
249 it outranks the band the scorer derived. They are not exclusive: each field
250 resolves down the list on its own, so a profile that names only reviewers still
251 lets the band's implementer stand instead of silently dropping it.
252 """
253 found: list[tuple[Bench, str]] = []
254 if profile is not None and profile in self.profiles:
255 found.append((self.profiles[profile], f"team.profiles.{profile}"))
256 if difficulty is not None and difficulty in self.by_difficulty:
257 found.append((self.by_difficulty[difficulty], f"team.by_difficulty.{difficulty}"))
258 return tuple(found)
260 def review_for(self, tier: int | None) -> tuple[tuple[Seat, ...] | str | None, str | None]:
261 """Reviewer seats (or ``"jury"``) for ``tier``, plus the config path they came from."""
262 key = str(tier)
263 if key in self.review_by_tier:
264 return self.review_by_tier[key], f"team.review.by_tier.{key}"
265 if self.review is not None:
266 return self.review, "team.review.default"
267 return None, None
270def _text(value: Any) -> str | None:
271 """A non-blank string, or ``None`` — so a blank field reads as unset."""
272 return value.strip() if isinstance(value, str) and value.strip() else None
275def _seat(raw: Any) -> Seat | None:
276 """One seat mapping -> a :class:`Seat`; ``None`` when it names no provider.
278 Shape errors are the schema's job and meaning is :func:`team_issues`'; this only has
279 to be total, because :func:`keel.config.parse_config` builds the policy *after*
280 validation and a caller must never get a half-parsed seat.
281 """
282 if not isinstance(raw, Mapping):
283 return None
284 provider = _text(raw.get("provider"))
285 if provider is None:
286 return None
287 return Seat(
288 provider=provider,
289 model=_text(raw.get("model")),
290 effort=_text(raw.get("effort")),
291 distinct_from=_text(raw.get("distinct_from")),
292 )
295def _seats(raw: Any) -> tuple[Seat, ...] | str | None:
296 """A reviewer list, the ``"jury"`` literal, or ``None`` when neither."""
297 if isinstance(raw, str):
298 return raw.strip()
299 if not isinstance(raw, Sequence) or isinstance(raw, (bytes, bytearray)):
300 return None
301 seats = tuple(seat for seat in (_seat(entry) for entry in raw) if seat is not None)
302 return seats
305def _bench(raw: Any) -> Bench | None:
306 """One ``by_difficulty``/``profiles`` entry -> a :class:`Bench`, or ``None``.
308 An entry that names nothing at all is ``None`` rather than an empty bench, so an
309 accidental ``hard: {}`` does not read as *a bench that overrides everything with
310 nothing* — it reads as absent, and the role/tier policy stands.
311 """
312 if not isinstance(raw, Mapping):
313 return None
314 bench = Bench(
315 lead=_seat(raw.get("lead")),
316 implement=_seat(raw.get("implement")),
317 review=_seats(raw.get("review")) if "review" in raw else None,
318 effort=_text(raw.get("effort")),
319 )
320 if bench == Bench():
321 return None
322 return bench
325def _benches(raw: Any) -> dict[str, Bench]:
326 """A ``by_difficulty``/``profiles`` mapping -> named benches, in a stable order."""
327 if not isinstance(raw, Mapping):
328 return {}
329 benches: dict[str, Bench] = {}
330 for name in sorted(raw, key=str):
331 bench = _bench(raw[name])
332 if isinstance(name, str) and bench is not None:
333 benches[name] = bench
334 return benches
337#: The fewest approving review verdicts the ``COMMENT``-consensus rule accepts (#1570):
338#: one approval beside a panel that did not approve is a single opinion, not consensus.
339COMMENT_CONSENSUS_MIN_APPROVALS = 2
342def _comment_consensus(raw: Any) -> int | None:
343 """``team.jury.comment_consensus`` -> its approval count when enabled, else ``None``.
345 Fails closed on shape: anything but ``enabled: true`` is off, and a ``min_approvals``
346 below :data:`COMMENT_CONSENSUS_MIN_APPROVALS` (or not an integer) reads as that floor,
347 so a malformed block can never lower the bar the schema sets.
348 """
349 if not isinstance(raw, Mapping) or raw.get("enabled") is not True:
350 return None
351 count = raw.get("min_approvals")
352 if isinstance(count, bool) or not isinstance(count, int):
353 return COMMENT_CONSENSUS_MIN_APPROVALS
354 return max(count, COMMENT_CONSENSUS_MIN_APPROVALS)
357def parse_team(raw: Any) -> TeamPolicy:
358 """Parse a ``knobs.team`` block (``None``/malformed -> an unconfigured policy)."""
359 if not isinstance(raw, Mapping):
360 return TeamPolicy()
361 implement = raw.get("implement") if isinstance(raw.get("implement"), Mapping) else {}
362 by_role_raw = implement.get("by_role") if isinstance(implement.get("by_role"), Mapping) else {}
363 by_role = {}
364 for role in sorted(by_role_raw, key=str):
365 seat = _seat(by_role_raw[role])
366 if isinstance(role, str) and seat is not None:
367 by_role[role] = seat
368 review = raw.get("review") if isinstance(raw.get("review"), Mapping) else {}
369 by_tier: dict[str, tuple[Seat, ...] | str] = {}
370 by_tier_raw = review.get("by_tier") if isinstance(review.get("by_tier"), Mapping) else {}
371 for tier in sorted(by_tier_raw, key=str):
372 seats = _seats(by_tier_raw[tier])
373 if isinstance(tier, str) and seats is not None:
374 by_tier[tier] = seats
375 default_review = _seats(review.get("default")) if "default" in review else None
376 jury = raw.get("jury") if isinstance(raw.get("jury"), Mapping) else {}
377 min_vendors = jury.get("min_vendors")
378 base_facts = jury.get("base_facts")
379 return TeamPolicy(
380 configured=True,
381 implement=_seat(implement.get("default")),
382 implement_by_role=by_role,
383 gate=_seat(raw.get("gate")),
384 review=default_review,
385 review_by_tier=by_tier,
386 jury_mode=_text(jury.get("mode")),
387 jury_min_vendors=min_vendors if isinstance(min_vendors, int) else None,
388 jury_on_unavailable=_text(jury.get("on_unavailable")),
389 jury_comment_consensus=_comment_consensus(jury.get("comment_consensus")),
390 jury_base_facts=base_facts if isinstance(base_facts, bool) else None,
391 fix=_seat(raw.get("fix")),
392 lead=_seat(raw.get("lead")),
393 by_difficulty=_benches(raw.get("by_difficulty")),
394 profiles=_benches(raw.get("profiles")),
395 )
398def _seat_canonical(seat: Seat) -> dict[str, Any]:
399 record = {"provider": seat.provider, "model": seat.model, "effort": seat.effort}
400 if seat.distinct_from is not None:
401 record["distinct_from"] = seat.distinct_from
402 return record
405def _bench_canonical(bench: Bench) -> dict[str, Any]:
406 record: dict[str, Any] = {}
407 if bench.lead is not None:
408 record["lead"] = _seat_canonical(bench.lead)
409 if bench.implement is not None:
410 record["implement"] = _seat_canonical(bench.implement)
411 if bench.review is not None:
412 record["review"] = (
413 bench.review
414 if isinstance(bench.review, str)
415 else [_seat_canonical(seat) for seat in bench.review]
416 )
417 if bench.effort is not None:
418 record["effort"] = bench.effort
419 return record
422def canonical(policy: TeamPolicy) -> dict[str, Any]:
423 """``{"team": {...}}`` for a configured policy, ``{}`` otherwise.
425 Empty means **absent**, not ``{}``: an added optional field must not rotate
426 ``config_hash`` for the projects that never used it, which is the same treatment
427 :func:`keel.config.delegate_profiles_dict` gives ``delegate_profiles``. The flip
428 side is the guarantee #1014 asks for — ``config_hash`` changes *iff* ``team`` does.
429 """
430 if not policy.configured:
431 return {}
432 team: dict[str, Any] = {}
433 implement: dict[str, Any] = {}
434 if policy.implement is not None:
435 implement["default"] = _seat_canonical(policy.implement)
436 if policy.implement_by_role:
437 implement["by_role"] = {
438 role: _seat_canonical(seat) for role, seat in sorted(policy.implement_by_role.items())
439 }
440 if implement:
441 team["implement"] = implement
442 if policy.gate is not None:
443 team["gate"] = _seat_canonical(policy.gate)
444 review: dict[str, Any] = {}
445 if policy.review is not None:
446 review["default"] = (
447 policy.review
448 if isinstance(policy.review, str)
449 else [_seat_canonical(seat) for seat in policy.review]
450 )
451 if policy.review_by_tier:
452 review["by_tier"] = {
453 tier: value if isinstance(value, str) else [_seat_canonical(s) for s in value]
454 for tier, value in sorted(policy.review_by_tier.items())
455 }
456 if review:
457 team["review"] = review
458 jury = {}
459 if policy.jury_mode is not None:
460 jury["mode"] = policy.jury_mode
461 if policy.jury_min_vendors is not None:
462 jury["min_vendors"] = policy.jury_min_vendors
463 # Absent when unset, like every other optional field here: a project that never names
464 # `on_unavailable` keeps the `config_hash` it had before the setting existed (#1066).
465 if policy.jury_on_unavailable is not None:
466 jury["on_unavailable"] = policy.jury_on_unavailable
467 # Both absent when unset, for the same reason (#1570).
468 if policy.jury_comment_consensus is not None:
469 jury["comment_consensus"] = {
470 "enabled": True,
471 "min_approvals": policy.jury_comment_consensus,
472 }
473 if policy.jury_base_facts is not None:
474 jury["base_facts"] = policy.jury_base_facts
475 if jury:
476 team["jury"] = jury
477 if policy.fix is not None:
478 team["fix"] = _seat_canonical(policy.fix)
479 if policy.lead is not None:
480 team["lead"] = _seat_canonical(policy.lead)
481 for key, benches in (("by_difficulty", policy.by_difficulty), ("profiles", policy.profiles)):
482 if benches:
483 team[key] = {name: _bench_canonical(b) for name, b in sorted(benches.items())}
484 return {"team": team}
487def legacy_seats(
488 implementer_agents: Mapping[str, str],
489 *,
490 provider_names: Iterable[str] = (),
491) -> dict[str, Seat]:
492 """Map the deprecated ``knobs.implementer_agents`` onto ``team.implement.by_role``.
494 ``implementer_agents`` values were documented as vendor strings in
495 ``docs/keel/models.md`` and as Claude subagent names in ``ship.md`` s4, and nothing
496 said which — #1014's opening complaint. The migration reads them the only way that
497 keeps both documented meanings working: a value that names a provider keel can
498 resolve *is* that provider; anything else is a host subagent and gets the explicit
499 ``subagent:`` prefix it always meant.
500 """
501 known = set(provider_names)
502 seats: dict[str, Seat] = {}
503 for role, value in sorted(implementer_agents.items()):
504 name = _text(value)
505 if name is None:
506 continue
507 # Split the same way `--delegate` does before deciding this is a subagent name.
508 # `docs/keel/models.md` documents `frontend: anthropic-api:claude-3-7-sonnet-…`
509 # as a legal value, and treating the whole string as one opaque name turned it
510 # into `subagent:anthropic-api:claude-…` — a host subagent that does not exist,
511 # instead of the hosted-API vendor plus its model.
512 seat = seat_from_token(name)
513 if seat.kind == "subagent" or seat.provider in known:
514 seats[role] = seat
515 else:
516 seats[role] = Seat(provider=f"{SUBAGENT_PREFIX}{name}")
517 return seats
520def seat_from_token(token: str) -> Seat:
521 """A ``--delegate``/``--review-delegate`` token -> a seat.
523 ``vendor:model`` splits on the first colon, exactly as
524 :func:`keel.agents.split_delegate` does — except for ``subagent:<name>``, whose
525 colon separates a *kind* from a name and never a model.
526 """
527 value = (token or "").strip()
528 if value.startswith(SUBAGENT_PREFIX):
529 return Seat(provider=value)
530 provider, sep, model = value.partition(":")
531 return Seat(provider=provider, model=model if (sep and model) else None)
534def _implement_seat(
535 policy: TeamPolicy,
536 *,
537 role: str | None,
538 legacy: Mapping[str, Seat] | None,
539 host_agent: str,
540 benches: Sequence[tuple[Bench, str]] = (),
541) -> tuple[Seat, str]:
542 """The implementer and the config path it came from (bench > policy > legacy > host).
544 A bench outranks ``by_role`` because it is the more specific statement: the role says
545 *what part of the system this is*, the bench says *what this particular piece of work
546 costs*, and "the hard ones go to the strong implementer" is only expressible if the
547 second wins. ``--delegate`` still outranks both — that is the caller's job, above.
548 """
549 for bench, source in benches:
550 if bench.implement is not None:
551 return bench.implement, f"{source}.implement"
552 if role is not None and role in policy.implement_by_role:
553 return policy.implement_by_role[role], f"team.implement.by_role.{role}"
554 if policy.implement is not None:
555 return policy.implement, "team.implement.default"
556 if role is not None and legacy and role in legacy:
557 return legacy[role], f"knobs.implementer_agents.{role} (deprecated)"
558 return Seat(provider=host_agent), "host"
561def jury_on_unavailable(setting: str | None) -> str:
562 """The effective ``knobs.team.jury.on_unavailable`` (#1066).
564 ``None`` — unset — is :data:`JURY_ON_UNAVAILABLE_DEFAULT`. An unrecognised value cannot
565 get past ``keel validate`` (:func:`team_issues` rejects it) but resolves to the default
566 rather than raising: this is read on the resolution path, and a config that reached it
567 must still resolve to *some* policy. The setting stays tri-state at the config boundary
568 so an explicit ``fallback`` remains distinguishable from silence, which is what
569 ``config_hash`` reads.
570 """
571 return setting if setting in JURY_ON_UNAVAILABLE else JURY_ON_UNAVAILABLE_DEFAULT
574def _panel_falls_back(availability: Mapping[str, Any] | None) -> bool:
575 """True when a measured probe says the panel is unstaffable *and* the policy allows it.
577 ``None`` — no probe ran — is False: the panel stands. Nothing here decides the policy;
578 ``decision`` was already resolved by :meth:`keel.juryavail.Availability.decision`, so
579 this module keeps exactly one reading of the operator's configured allowance.
580 """
581 if not isinstance(availability, Mapping):
582 return False
583 return availability.get("decision") == JURY_ON_UNAVAILABLE[0]
586def _panel_refuses(availability: Mapping[str, Any] | None) -> bool:
587 """True when a measured probe says the panel is unstaffable *and* the policy refuses.
589 The mirror of :func:`_panel_falls_back`, written the same way and for the same reason:
590 ``decision`` was already resolved by :meth:`keel.juryavail.Availability.decision`, so
591 the two branches of ``team.jury.on_unavailable`` are read in one module and nowhere
592 else. ``None`` — no probe ran — is False: the panel stands.
593 """
594 if not isinstance(availability, Mapping):
595 return False
596 return availability.get("decision") == JURY_ON_UNAVAILABLE[1]
599def _availability_reason(availability: Mapping[str, Any] | None) -> str:
600 """The probe's own sentence, so the warning names the seats rather than summarising."""
601 reason = availability.get("reason") if isinstance(availability, Mapping) else None
602 return reason if isinstance(reason, str) and reason.strip() else "the probe reported no detail"
605def refusal_message(availability: Mapping[str, Any], *, source: str) -> str:
606 """The message an ``on_unavailable: block`` run refuses with.
608 It names the unavailable seats, because "the panel is unavailable" without them sends
609 the operator to ``keel doctor --providers`` to learn what this run already measured.
611 ``source`` is the config path that made the panel this run's review — the resolver's
612 own reading (:func:`configured_review`), so the overlay that selected the panel is
613 named rather than the tier's policy under it. Re-exported as
614 ``keel.juryavail.refusal_message``.
615 """
616 unavailable = availability.get("unavailable")
617 unavailable = unavailable if isinstance(unavailable, Sequence) else ()
618 seats = [
619 f" - {seat.get('provider')}: {seat.get('reason')}"
620 for seat in unavailable
621 if isinstance(seat, Mapping)
622 ]
623 listed = "\n".join(seats) or " - (no provider was probed)"
624 vendors = availability.get("available_vendors")
625 if not isinstance(vendors, Sequence) or isinstance(vendors, (str, bytes)):
626 vendors = []
627 return (
628 f"{source} makes the cross-vendor jury the review for this tier, and the panel "
629 f"cannot be staffed here: {len(vendors)} vendor(s) available "
630 f"({', '.join(str(v) for v in vendors) or 'none'}), "
631 f"{availability.get('required_vendors')} required.\n"
632 f"Unavailable:\n{listed}\n"
633 "knobs.team.jury.on_unavailable is 'block', so this run refuses rather than "
634 "reviewing with a bench the policy did not ask for. Install or authenticate what "
635 f"is missing — the panel runner answers `{JURY_RUNNER_COMMAND} --doctor` and keel's "
636 "own delegates answer `keel doctor --providers` — or set on_unavailable: fallback "
637 "to let a host bench of the same size review instead."
638 )
641def configured_review(
642 policy: TeamPolicy,
643 *,
644 tier: int | None,
645 benches: Sequence[tuple[Bench, str]] = (),
646) -> tuple[tuple[Seat, ...] | str | None, str | None]:
647 """The review policy in force for this run, and the config path it came from.
649 The tier's own (:meth:`TeamPolicy.review_for`), overlaid by the first bench that names
650 a review — a ``--team`` profile, else the difficulty band — exactly as
651 :meth:`TeamPolicy.benches_for` orders them.
653 **One reading of "what is this run's review", shared by everything that asks.** The
654 resolver that seats the bench (:func:`_review_seats`) and the probe that measures the
655 panel before it (:func:`keel.providerprobe.jury_availability`) had this written twice,
656 and the copies drifted: the probe read ``review_for`` alone, so a project whose
657 ``profiles.strict.review`` or ``by_difficulty.hard.review`` named the panel resolved
658 ``review_panel: jury`` with ``availability: null`` — the panel published on a machine
659 that was never asked whether it could convene one, which is #1066 reached by a second
660 route. A predicate this one depends on may only be written here.
661 """
662 for bench, bench_source in benches:
663 if bench.review is not None:
664 return bench.review, f"{bench_source}.review"
665 return policy.review_for(tier)
668def panel_review_source(
669 policy: TeamPolicy,
670 *,
671 tier: int | None,
672 difficulty: str | None = None,
673 profile: str | None = None,
674 any_difficulty: bool = False,
675) -> str | None:
676 """The config path that makes the cross-vendor panel this run's review, or ``None``.
678 ``None`` means no route to the panel exists for these coordinates, so the panel probe
679 has nothing to measure and a project that never convenes one pays nothing.
681 ``any_difficulty`` is for a caller that cannot name the band yet:
682 :func:`keel.swarm.build_swarm_plan` scores each cluster's difficulty *while* it
683 partitions, so the probe runs before any band exists. It then asks the wider question —
684 *could* any band this policy configures make the panel the review — and the answer is a
685 deliberate superset. That matches what the record already is: availability is a fact
686 about the machine, not about a cluster, which is why
687 :func:`keel.providerprobe.jury_availability_for_any_tier` sweeps tiers the same way.
688 """
689 bands = (None, *DIFFICULTY_BANDS) if any_difficulty else (difficulty,)
690 for band in bands:
691 configured, source = configured_review(
692 policy, tier=tier, benches=policy.benches_for(difficulty=band, profile=profile)
693 )
694 if configured == JURY_PANEL:
695 return source
696 return None
699def _review_seats(
700 policy: TeamPolicy,
701 *,
702 tier: int | None,
703 default_count: int,
704 reviewer_override: int | None,
705 host_agent: str,
706 jury_disabled: bool = False,
707 jury_advisory: bool = False,
708 benches: Sequence[tuple[Bench, str]] = (),
709 jury_availability: Mapping[str, Any] | None = None,
710) -> tuple[tuple[Seat, ...], str, str, list[str]]:
711 """Reviewer seats, the panel, the source, and any warnings.
713 Precedence: a tier whose policy is ``jury`` empties the reviewer bench (the panel
714 *is* the review); otherwise ``--reviewers`` wins over the policy's seat count, which
715 wins over the tier-derived default.
717 ``jury_availability`` is the one exception, and it is not a preference (#1066). It is
718 :func:`keel.juryavail.assess`'s verdict on whether this machine can convene the panel
719 at all, measured from the same probe ``keel doctor --providers`` prints. When it says
720 the panel is unstaffable and ``team.jury.on_unavailable`` is ``fallback``, the tier
721 resolves onto the **tier's own** seat count, staffed from the host, and the record says
722 so. The seat count and the evidence requirement do not move; only who sits does — which
723 is why ``--reviewers`` stays ignored on a fallback bench exactly as it is ignored while
724 the panel sits. A flag that was inert on a staffable panel and *lowered* the tier's
725 requirement the moment the probe failed would make a failed probe a policy change.
726 Under ``block`` the panel stays the panel and the run refuses — *here*, on the first
727 line, with :class:`JuryUnavailableError` (#1068). The probe only measures; it is asked
728 once per surface and may staff many benches, so it cannot know whether this particular
729 cluster's review is the panel. This can, and it is the only place that resolves a
730 bench, so it is the only place the check can neither be forgotten nor over-reach.
732 **The bench is a pure function of config + tier + role + the explicit ``--reviewers``
733 and ``--review-delegate`` overrides, and of nothing else.** In particular it does not
734 depend on the jury flags. It cannot: every surface accepts them since #1043, but
735 nothing makes a *run* pass them uniformly — keel's CI passes ``--no-jury`` to
736 ``evidence-verify`` on every run while passing it to neither ``ship`` nor ``plan``.
737 A bench that moved with that flag would have ``plan`` requiring
738 a jury verdict from zero reviewers while ``evidence-verify`` demanded three host
739 verdicts of the same PR, which is the same contract disagreement in a new place.
741 So ``jury_disabled`` / ``jury_advisory`` are recorded, never applied: on a tier whose
742 review policy is the panel, the panel **is** the review, and a per-run flag does not
743 get to remove the only review that tier has. :func:`keel.ship.resolve_jury` keeps the
744 verdict required for the same reason.
745 """
746 configured, source = configured_review(policy, tier=tier, benches=benches)
747 warnings: list[str] = []
748 if configured == JURY_PANEL and _panel_refuses(jury_availability):
749 # The one place the run refuses (#1068). Not at the probe: the probe answers a
750 # question about the *machine*, and a caller may resolve many benches from one
751 # answer. `keel swarm plan` measures before it has scored a single cluster, asking
752 # the widest question there is — could *any* tier or band name the panel — so a
753 # refusal there refused a swarm of entirely non-panel clusters on an unstaffable
754 # host. By here `configured` is this cluster's own resolved review, so the run
755 # refuses when the panel really is what it would have dispatched, and `source`
756 # names the config path that made it so.
757 raise JuryUnavailableError(refusal_message(jury_availability, source=source))
758 fell_back = configured == JURY_PANEL and _panel_falls_back(jury_availability)
759 if fell_back:
760 # The panel is this tier's review and this machine cannot convene it. Fall through
761 # to the ordinary path with no configured seats, which *is* "a tier without a
762 # panel": the tier's own count, staffed from the host. Reached only from a
763 # measured probe, and the reason travels in `warnings` and in the assignment's
764 # `jury.availability` block so no reader has to re-derive it.
765 warnings.append(
766 f"{source} makes the jury the review for this tier, but the panel cannot be "
767 f"staffed here; knobs.team.jury.on_unavailable is 'fallback', so a host bench "
768 f"of {default_count} seat(s) reviews instead — the same count and the same "
769 f"evidence, different reviewers, and they share one vendor so this review "
770 f"carries no cross-vendor independence claim. "
771 f"{_availability_reason(jury_availability)}"
772 )
773 if reviewer_override is not None:
774 # The same flag, ignored the same way, whether or not the panel could sit.
775 # `--reviewers` is inert on a panel tier — the panel *is* the review — and a
776 # fallback may not turn that inert flag into a live one: a bench sized by
777 # `--reviewers 2` would publish a two-verdict evidence requirement where the
778 # tier asks for three, so a probe failure would have *lowered* the tier's
779 # policy. The fallback changes who sat, never how many.
780 warnings.append(
781 f"--reviewers {reviewer_override} ignored: {source} makes the jury the "
782 f"review panel, and the host bench standing in for it is the tier's own "
783 f"{default_count} seat(s) — a fallback changes who reviews, not how many"
784 )
785 reviewer_override = None
786 configured = ()
787 if configured == JURY_PANEL:
788 if reviewer_override is not None:
789 warnings.append(
790 f"--reviewers {reviewer_override} ignored: {source} makes the jury the "
791 "review panel, so there are no host reviewer slots to size"
792 )
793 ignored = [
794 flag
795 for flag, passed in (("--no-jury", jury_disabled), ("--jury-advisory", jury_advisory))
796 if passed
797 ]
798 if ignored:
799 warnings.append(
800 f"{' and '.join(ignored)} does not apply: this tier's review is the jury "
801 f"panel ({source}). The panel is the review, so its verdict stays required"
802 )
803 return (), JURY_PANEL, source, warnings
804 seats = configured if isinstance(configured, tuple) else ()
805 tier_source = "risk-tier" if tier is not None else "unresolved"
806 if reviewer_override is not None:
807 count, source = reviewer_override, "override"
808 elif seats:
809 count = len(seats)
810 else:
811 count, source = default_count, tier_source
812 if fell_back:
813 # Named, not inherited from the tier's config path: a reader of the published
814 # contract must be able to tell a bench seated because the panel could not be
815 # from a tier that simply never had one.
816 source = JURY_FALLBACK_SOURCE
817 resolved = tuple(
818 seats[index] if index < len(seats) else Seat(provider=host_agent) for index in range(count)
819 )
820 padded = max(0, count - len(seats))
821 if fell_back:
822 # The pad warning's advice ("name the extra seats in knobs.team.review") is wrong
823 # here: this tier *did* name its reviewers — it named the panel. The fallback
824 # warning above already carries the one fact that advice was protecting.
825 padded = 0
826 if padded > 1 or (padded and any(seat.provider == host_agent for seat in seats)):
827 # Two conditions, because there are two ways the pad duplicates. The host may
828 # already be a configured seat (`[claude, codex]` + `--reviewers 3`), or it may
829 # not be and simply get seated twice (`[codex]` + `--reviewers 3` ->
830 # `[codex, claude, claude]`): the second is a duplicate between two *padded*
831 # slots, which a check against the configured seats alone never sees.
832 #
833 # Comparing provider names, not vendors: resolving a name to its vendor needs the
834 # registry, which this module deliberately cannot reach. A repeated *name* is
835 # already a repeated vendor, and `require_distinct_vendors` rejects it at the
836 # evidence gate long after the run.
837 seated = any(seat.provider == host_agent for seat in seats)
838 where = "which is already seated" if seated else "filling more than one slot"
839 warnings.append(
840 f"{padded} reviewer slot(s) padded with the host agent {host_agent!r}, "
841 f"{where}; those reviewers cannot return distinct vendor provenance — name "
842 "the extra seats in knobs.team.review, or lower --reviewers"
843 )
844 if seats and len(seats) > count:
845 warnings.append(
846 f"{len(seats)} reviewer seat(s) configured but only {count} slot(s) are "
847 "staffed; the surplus seats are not dispatched"
848 )
849 return resolved, "reviewers", source or tier_source, warnings
852#: The most reviewer seats a tier may name — keel's reviewer vocabulary is A/B/C, and
853#: :func:`keel.ship.reviewer_focuses` has focus coverage for exactly those three.
854MAX_REVIEW_SEATS = 3
856#: Slot letters, by seat count. Deliberately not ``A``/``B``/``C`` for two seats: keel
857#: merges the B focus into A at that count, so the second reviewer is slot **C**. Mirrors
858#: :func:`keel.ship.reviewer_focuses`, which cannot be imported here (``ship`` imports
859#: this module); ``tests/test_team.py`` asserts the two agree for every valid count.
860_SLOT_LABELS = {0: (), 1: ("A",), 2: ("A", "C"), 3: ("A", "B", "C")}
863def slot_labels(count: int) -> tuple[str, ...]:
864 """Slot letters for ``count`` reviewer seats, matching ``ship.reviewer_focuses``.
866 Total for any non-negative count, including counts keel's own vocabulary does not
867 have a focus for. It is a labelling function: running short here turned an
868 out-of-range reviewer count into an ``IndexError`` from the middle of the resolver
869 rather than the documented ``ValueError`` the caller raises for it.
870 """
871 if count in _SLOT_LABELS:
872 return _SLOT_LABELS[count]
873 return tuple(chr(ord("A") + index) for index in range(max(0, count)))
876def _effective_effort(
877 seat: Seat, *, bench_effort: str | None, flag_effort: str | None
878) -> str | None:
879 """The effort this seat actually runs at: ``--effort`` > the seat's own > the bench's.
881 The flag wins because it is the operator speaking about *this run*; the seat wins over
882 the bench because a seat that names a provider **and** an effort is one statement, and
883 the bench's ``effort`` is the default for seats that did not bother.
884 """
885 return flag_effort or seat.effort or bench_effort
888def _seat_at_effort(seat: Seat, effort: str | None) -> Seat:
889 """``seat`` running at ``effort`` — the same object when nothing changes."""
890 if effort == seat.effort:
891 return seat
892 return Seat(
893 provider=seat.provider,
894 model=seat.model,
895 effort=effort,
896 distinct_from=seat.distinct_from,
897 )
900def _lead_seat(
901 policy: TeamPolicy,
902 *,
903 benches: Sequence[tuple[Bench, str]],
904 host_agent: str,
905) -> tuple[Seat, str]:
906 """Who coordinates this batch of work: bench > ``team.lead`` > the host agent.
908 Always a seat, never ``None``: an unconfigured project still has a lead — the agent
909 driving the run — and a swarm coordinator that had to special-case "no lead" would
910 grow a second answer to a question this one already answers.
911 """
912 for bench, source in benches:
913 if bench.lead is not None:
914 return bench.lead, f"{source}.lead"
915 if policy.lead is not None:
916 return policy.lead, "team.lead"
917 return Seat(provider=host_agent), "host"
920def resolve_assignment(
921 policy: TeamPolicy,
922 *,
923 tier: int | None = None,
924 role: str | None = None,
925 default_count: int = 2,
926 reviewer_override: int | None = None,
927 delegate: str | None = None,
928 review_delegates: Sequence[str] = (),
929 host_agent: str = HOST_DEFAULT,
930 legacy: Mapping[str, Seat] | None = None,
931 jury_disabled: bool = False,
932 jury_advisory: bool = False,
933 difficulty: str | None = None,
934 team_profile: str | None = None,
935 effort: str | None = None,
936 jury_availability: Mapping[str, Any] | None = None,
937) -> dict[str, Any]:
938 """Who runs this ship: implementer, gate, reviewer slots, jury, fix.
940 Deterministic for identical inputs, which is what lets ``keel plan`` and
941 ``keel ship`` render the same team and any host run it. Per-run flags stay per-run
942 overrides: ``--delegate`` replaces the implementer, and each ``--review-delegate``
943 replaces one reviewer slot **positionally** — the first flag is slot A, the second
944 slot B — so a two-vendor panel is expressible from the command line without a config
945 change. A flag past the last slot is reported in ``warnings`` rather than silently
946 dropped or silently growing the panel.
948 ``difficulty`` and ``team_profile`` select a :class:`Bench` (see
949 :meth:`TeamPolicy.benches_for`), which is how a batch runner says *this cluster is
950 hard, give it the strong implementer at high effort; the easy ones go to the cheap
951 model*. They are inputs to this one resolver rather than a second one beside it, so
952 ``keel ship``, ``keel plan`` and ``keel swarm-plan`` cannot disagree about who runs a
953 given issue.
954 """
955 benches = policy.benches_for(difficulty=difficulty, profile=team_profile)
956 implementer, implementer_source = _implement_seat(
957 policy, role=role, legacy=legacy, host_agent=host_agent, benches=benches
958 )
959 if delegate:
960 implementer, implementer_source = seat_from_token(delegate), "flag:--delegate"
961 bench_effort = next((b.effort for b, _ in benches if b.effort is not None), None)
962 implementer = _seat_at_effort(
963 implementer, _effective_effort(implementer, bench_effort=bench_effort, flag_effort=effort)
964 )
965 lead, lead_source = _lead_seat(policy, benches=benches, host_agent=host_agent)
966 seats, panel, review_source, warnings = _review_seats(
967 policy,
968 tier=tier,
969 default_count=default_count,
970 reviewer_override=reviewer_override,
971 host_agent=host_agent,
972 jury_disabled=jury_disabled,
973 jury_advisory=jury_advisory,
974 benches=benches,
975 jury_availability=jury_availability,
976 )
977 if team_profile is not None and team_profile not in policy.profiles:
978 warnings.append(
979 f"--team {team_profile!r} names no knobs.team.profiles entry; this run is "
980 f"staffed from the configured policy instead. Known: "
981 f"{', '.join(sorted(policy.profiles)) or 'none'}"
982 )
983 labels = slot_labels(len(seats))
984 sources = [review_source] * len(seats)
985 overrides = [token for token in review_delegates if (token or "").strip()]
986 for index, token in enumerate(overrides):
987 if index >= len(seats):
988 warnings.append(
989 f"--review-delegate {token!r} names reviewer slot {index + 1}, but only "
990 f"{len(seats)} reviewer slot(s) are staffed; it is not dispatched"
991 )
992 continue
993 seats = seats[:index] + (seat_from_token(token),) + seats[index + 1 :]
994 sources[index] = "flag:--review-delegate"
995 gate = policy.gate
996 gate_record = None
997 if gate is not None:
998 distinct_ok = gate.distinct_from != IMPLEMENTER or gate.name != implementer.name
999 gate_record = gate.as_dict(source="team.gate")
1000 gate_record["distinct_ok"] = distinct_ok
1001 if not distinct_ok:
1002 warnings.append(
1003 f"team.gate.provider {gate.provider!r} is the resolved implementer and "
1004 "gate.distinct_from is 'implementer'; the gate review would be a second "
1005 "opinion from the first opinion"
1006 )
1007 fix_seat = policy.fix if policy.fix is not None else Seat(provider=IMPLEMENTER)
1008 fix_source = "team.fix" if policy.fix is not None else "default"
1009 resolved_fix = implementer if fix_seat.kind == "alias" else fix_seat
1010 fix_record = resolved_fix.as_dict(source=fix_source)
1011 fix_record["alias"] = IMPLEMENTER if fix_seat.kind == "alias" else None
1012 return {
1013 "configured": policy.configured,
1014 "role": role,
1015 "tier": tier,
1016 "difficulty": difficulty,
1017 "team_profile": team_profile,
1018 "bench": [source for _bench, source in benches],
1019 "lead": lead.as_dict(source=lead_source),
1020 "implementer": implementer.as_dict(source=implementer_source),
1021 "effort": implementer.effort,
1022 "gate": gate_record,
1023 "review_panel": panel,
1024 "reviewers": [
1025 seat.as_dict(source=sources[index], slot=labels[index])
1026 for index, seat in enumerate(seats)
1027 ],
1028 "reviewer_count": len(seats),
1029 "reviewer_source": review_source,
1030 "jury": {
1031 "mode": policy.jury_mode,
1032 "min_vendors": policy.jury_min_vendors or DEFAULT_MIN_VENDORS,
1033 "panel_is_review": panel == JURY_PANEL,
1034 # What the tier's policy asked for, before availability had its say. Without
1035 # it a fallback run's assignment is indistinguishable from a tier that never
1036 # configured a panel — which is the silent downgrade #1066 exists to refuse.
1037 "panel_configured": review_source == JURY_FALLBACK_SOURCE or panel == JURY_PANEL,
1038 "on_unavailable": jury_on_unavailable(policy.jury_on_unavailable),
1039 # The approvals a COMMENT consensus needs, or `None` when the rule is off (#1570).
1040 "comment_consensus": policy.jury_comment_consensus,
1041 # `None` until something measured it: `keel plan --no-probe`-shaped callers
1042 # and every non-panel tier resolve without a probe, and an absent measurement
1043 # must not read as "we checked and it was fine".
1044 "availability": (
1045 dict(jury_availability) if isinstance(jury_availability, Mapping) else None
1046 ),
1047 },
1048 "fix": fix_record,
1049 "warnings": warnings,
1050 }
1053def require_distinct_vendors(setting: bool | None) -> bool:
1054 """The effective ``evidence_require_distinct_vendors``.
1056 **Opt-in** (#1065). ``None`` is *unset* and resolves to ``False``: the knob asserts
1057 that the required verdicts came from *independent* opinions, and that is a claim only
1058 the project can make. It is a property a cross-vendor panel provides, not one every
1059 high-tier review has to carry — a person with a single agent CLI installed must still
1060 be able to land a TIER-3 change without configuring anything. A project that wants
1061 the independence claim enforced says so, and the requirement then lives in a file a
1062 reviewer can read rather than in a default nobody chose.
1064 The setting is still tri-state at the config boundary (``None`` unset, ``True``,
1065 ``False``) so an explicit ``false`` stays distinguishable from silence, which is what
1066 ``config_hash`` and the wizard read.
1067 """
1068 return bool(setting)
1071def _seat_paths(policy: TeamPolicy) -> list[tuple[str, Seat]]:
1072 """Every seat in the policy with the config path it sits at, in a stable order."""
1073 paths: list[tuple[str, Seat]] = []
1074 if policy.implement is not None:
1075 paths.append(("implement.default", policy.implement))
1076 paths.extend(
1077 (f"implement.by_role.{role}", seat)
1078 for role, seat in sorted(policy.implement_by_role.items())
1079 )
1080 if policy.gate is not None:
1081 paths.append(("gate", policy.gate))
1082 if isinstance(policy.review, tuple):
1083 paths.extend((f"review.default[{i}]", seat) for i, seat in enumerate(policy.review))
1084 for tier, value in sorted(policy.review_by_tier.items()):
1085 if isinstance(value, tuple):
1086 paths.extend((f"review.by_tier.{tier}[{i}]", seat) for i, seat in enumerate(value))
1087 if policy.fix is not None:
1088 paths.append(("fix", policy.fix))
1089 if policy.lead is not None:
1090 paths.append(("lead", policy.lead))
1091 paths.extend(_bench_seat_paths("by_difficulty", policy.by_difficulty))
1092 paths.extend(_bench_seat_paths("profiles", policy.profiles))
1093 return paths
1096def _bench_seat_paths(key: str, benches: Mapping[str, Bench]) -> list[tuple[str, Seat]]:
1097 """Every seat of every bench in one table, with its config path."""
1098 paths: list[tuple[str, Seat]] = []
1099 for name, bench in sorted(benches.items()):
1100 where = f"{key}.{name}"
1101 if bench.lead is not None:
1102 paths.append((f"{where}.lead", bench.lead))
1103 if bench.implement is not None:
1104 paths.append((f"{where}.implement", bench.implement))
1105 if isinstance(bench.review, tuple):
1106 paths.extend((f"{where}.review[{i}]", seat) for i, seat in enumerate(bench.review))
1107 return paths
1110def team_issues(
1111 raw: Any,
1112 *,
1113 source: str,
1114 profiles: Mapping[str, str] | None = None,
1115 implementer_agents: Mapping[str, str] | None = None,
1116) -> list[str]:
1117 """Semantic errors for ``knobs.team`` (empty == valid).
1119 The schema owns the *shape*; this owns the *meaning* — which providers exist, which
1120 of them can honour an ``effort``, that the mandatory gate review really is a second
1121 opinion, and that a ``review.by_tier`` entry is either reviewer seats or the jury.
1123 ``implementer_agents`` is the deprecated per-role knob. It is read here because it
1124 still resolves implementers (:func:`legacy_seats`), so a ``gate`` declared
1125 ``distinct_from: implementer`` has to be checked against those seats too — otherwise
1126 ``implementer_agents: {core: codex}`` beside ``gate: {provider: codex}`` passes
1127 validation and the "mandatory second opinion" is the first opinion again.
1129 ``profiles`` maps a ``knobs.delegate_profiles`` name to its vendor. A machine-level
1130 ``~/.keel/providers.yaml`` entry is deliberately **not** consulted: validation has to
1131 give the same answer on every machine, and a policy that only validates where its
1132 author's home directory does is a policy the next operator cannot read.
1133 """
1134 if raw is None:
1135 return []
1136 if not isinstance(raw, Mapping):
1137 return [] # the schema already reported the wrong shape
1138 profiles = dict(profiles or {})
1139 known = {**{vendor: vendor for vendor in BUILTIN_DELEGATE_VENDORS}, **profiles}
1140 policy = parse_team(raw)
1141 errors: list[str] = []
1142 errors.extend(_tier_key_issues(raw, source=source))
1143 errors.extend(_band_key_issues(raw, source=source))
1144 for path, seat in _seat_paths(policy):
1145 where = f"{source}.{path}"
1146 vendor = _seat_vendor(seat, path=path, known=known, where=where, errors=errors)
1147 if seat.effort is None:
1148 continue
1149 errors.extend(_effort_issues(seat.effort, seat=seat, vendor=vendor, where=where))
1150 legacy = legacy_seats(implementer_agents or {}, provider_names=known)
1151 errors.extend(_bench_effort_issues(policy, source=source, known=known, legacy=legacy))
1152 errors.extend(_gate_issues(policy, source=source, legacy=legacy))
1153 errors.extend(_review_issues(policy, source=source))
1154 return errors
1157def _effort_issues(
1158 effort: str,
1159 *,
1160 seat: Seat,
1161 vendor: str | None,
1162 where: str,
1163 applies_to: str | None = None,
1164) -> list[str]:
1165 """The ways an ``effort`` is not honourable by ``seat``, worded the same everywhere.
1167 Extracted so a bench-level ``effort`` (:func:`_bench_effort_issues`) is judged by
1168 exactly the rules a seat-level one is, in exactly the same words. Two copies of these
1169 three sentences is how ``by_difficulty.hard.effort`` came to bypass the checks
1170 ``implement.default.effort`` has passed since #1014.
1172 ``applies_to`` names the seat the effort would land on, for the bench case where the
1173 effort and the seat it breaks are written in different places.
1175 These are #1014's three rules and only those, so the two callers cannot drift. The
1176 subagent rule is deliberately *not* here: it applies to a bench effort alone, and
1177 :func:`_bench_effort_issues` says why.
1178 """
1179 tail = "" if applies_to is None else f" (applied to the implementer at {applies_to})"
1180 if effort not in EFFORTS:
1181 return [f"{where}: unknown effort {effort!r}; valid: {', '.join(EFFORTS)}{tail}"]
1182 if vendor is None:
1183 return [] # an unresolvable provider is already reported; do not pile on
1184 if not supports_effort(vendor):
1185 return [
1186 f"{where}: provider {seat.provider!r} ({vendor}) has no spelling for "
1187 f"reasoning effort, so effort {effort!r} would be silently dropped "
1188 f"— drop the field, or name a provider that can honour it{tail}"
1189 ]
1190 if effort_needs_model(vendor) and seat.model is None:
1191 return [
1192 f"{where}: agy spells reasoning effort as a model suffix "
1193 f"(e.g. gemini-3.8-flash-high), so effort {effort!r} needs a "
1194 f"'model' beside it{tail}"
1195 ]
1196 return []
1199def _bench_effort_issues(
1200 policy: TeamPolicy,
1201 *,
1202 source: str,
1203 known: Mapping[str, str],
1204 legacy: Mapping[str, Seat],
1205) -> list[str]:
1206 """A bench ``effort`` has to be honourable by every implementer it could land on.
1208 A seat's own ``effort`` sits next to the provider it applies to, so an operator
1209 reading one line sees both halves. A bench's does not: ``by_difficulty.hard.effort``
1210 lands on whichever implementer resolves for that band, which may be written in
1211 another file's worth of config — action at a distance, and exactly the case where a
1212 silently-dropped effort is invisible. So it is checked against every seat it could
1213 reach, by the same rules and in the same words.
1215 Which seats those are follows the resolution order. A bench naming its own
1216 ``implement`` seat can only land there. One that does not falls through to the
1217 role/default/legacy implementers, and any of them is reachable. The host-agent
1218 fallback is deliberately not checked: the host is a per-run flag, and a policy that
1219 only validates against one operator's default is the kind of rule #1014 refused.
1221 A **subagent** target is an error here and not at seat level, and the difference is
1222 the point. ``fix: {provider: "subagent:x", effort: high}`` pairs the two on one line:
1223 the operator saw both halves and #1014 chose to tolerate it. A bench effort lands on
1224 a seat written somewhere else entirely, so the same pairing is one nobody ever read.
1225 """
1226 errors: list[str] = []
1227 for table, benches in (("by_difficulty", policy.by_difficulty), ("profiles", policy.profiles)):
1228 for name, bench in sorted(benches.items()):
1229 if bench.effort is None:
1230 continue
1231 where = f"{source}.{table}.{name}.effort"
1232 for seat_path, seat in _bench_effort_targets(
1233 policy, table=table, bench=bench, legacy=legacy
1234 ):
1235 # A seat naming its own effort never receives the bench's, so a bench
1236 # effort it could not honour is not a defect — the seat's wins.
1237 if seat.effort is not None:
1238 continue
1239 if seat.kind == "subagent":
1240 errors.append(
1241 f"{where}: {seat.provider!r} is a host subagent, which has no "
1242 f"reasoning-effort dial keel can set, so effort {bench.effort!r} "
1243 "would be silently dropped — drop the field, or name a provider "
1244 f"that can honour it (applied to the implementer at {seat_path})"
1245 )
1246 continue
1247 errors.extend(
1248 _effort_issues(
1249 bench.effort,
1250 seat=seat,
1251 vendor=known.get(seat.provider),
1252 where=where,
1253 applies_to=seat_path,
1254 )
1255 )
1256 return errors
1259def _bench_effort_targets(
1260 policy: TeamPolicy,
1261 *,
1262 table: str,
1263 bench: Bench,
1264 legacy: Mapping[str, Seat],
1265) -> list[tuple[str, Seat]]:
1266 """Implementer seats a bench's ``effort`` could land on, with their config paths.
1268 *Reachable*, not merely *present*. One run resolves exactly one difficulty band and
1269 at most one ``--team`` profile, so two entries of the same table never apply
1270 together: a ``by_difficulty.hard`` effort can never meet ``by_difficulty.easy``'s
1271 implementer. Checking against siblings reported errors for combinations no run can
1272 produce, which is how a validator teaches people to ignore it.
1274 What is left is genuinely reachable. A bench naming its own ``implement`` seat can
1275 only land there. Otherwise the implementer comes from the *other* table (a ``--team``
1276 profile supplying the seat while the band supplies the effort, or the reverse), or
1277 from the role/default/legacy seats every run falls through to.
1278 """
1279 if bench.implement is not None:
1280 return [("this bench's own implement seat", bench.implement)]
1281 other = "profiles" if table == "by_difficulty" else "by_difficulty"
1282 targets = [
1283 (path, seat)
1284 for path, seat in _seat_paths(policy)
1285 if path.startswith("implement")
1286 or (path.startswith(f"{other}.") and path.endswith(".implement"))
1287 ]
1288 targets.extend(
1289 (f"knobs.implementer_agents.{role}", seat) for role, seat in sorted(legacy.items())
1290 )
1291 return targets
1294def _seat_vendor(
1295 seat: Seat,
1296 *,
1297 path: str,
1298 known: Mapping[str, str],
1299 where: str,
1300 errors: list[str],
1301) -> str | None:
1302 """The vendor behind a seat, appending the error when the provider is not resolvable."""
1303 if seat.kind == "subagent":
1304 if not seat.name:
1305 errors.append(
1306 f"{where}: {SUBAGENT_PREFIX!r} needs a subagent name after it, "
1307 "e.g. 'subagent:backend-developer'"
1308 )
1309 return None
1310 if seat.kind == "alias":
1311 if path != "fix":
1312 errors.append(
1313 f"{where}: provider {IMPLEMENTER!r} means 'whoever implemented this "
1314 "change' and is only valid at team.fix.provider"
1315 )
1316 return None
1317 if seat.provider not in known:
1318 errors.append(
1319 f"{where}: unknown provider {seat.provider!r}; name a built-in vendor, a "
1320 f"knobs.delegate_profiles entry, or a host subagent as "
1321 f"'{SUBAGENT_PREFIX}{seat.provider}'. Known: {', '.join(sorted(known))}"
1322 )
1323 return None
1324 return known[seat.provider]
1327def _tier_key_issues(raw: Mapping[str, Any], *, source: str) -> list[str]:
1328 """``review.by_tier`` keys must be the quoted tier strings ``"1"``/``"2"``/``"3"``."""
1329 review = raw.get("review")
1330 by_tier = review.get("by_tier") if isinstance(review, Mapping) else None
1331 if not isinstance(by_tier, Mapping):
1332 return []
1333 errors = []
1334 for key in by_tier:
1335 if key in TIERS:
1336 continue
1337 errors.append(
1338 f"{source}.review.by_tier: {key!r} is not a risk tier; quote the key as "
1339 f'"1", "2" or "3" (YAML reads a bare 1: as an integer key, which a JSON '
1340 "schema cannot describe)"
1341 )
1342 return errors
1345def _band_key_issues(raw: Mapping[str, Any], *, source: str) -> list[str]:
1346 """``by_difficulty`` keys must be difficulty bands the scorer can produce.
1348 A typo here is silent otherwise: ``medium:`` beside ``easy:``/``hard:`` never matches
1349 anything the scorer emits, so the bench an operator wrote is simply never staffed and
1350 the run looks like the table was ignored.
1351 """
1352 by_difficulty = raw.get("by_difficulty")
1353 if not isinstance(by_difficulty, Mapping):
1354 return []
1355 return [
1356 f"{source}.by_difficulty: {key!r} is not a difficulty band; valid: "
1357 f"{', '.join(DIFFICULTY_BANDS)}"
1358 for key in by_difficulty
1359 if key not in DIFFICULTY_BANDS
1360 ]
1363def _gate_issues(
1364 policy: TeamPolicy,
1365 *,
1366 source: str,
1367 legacy: Mapping[str, Seat] | None = None,
1368) -> list[str]:
1369 """The mandatory second opinion must be able to *be* a second opinion."""
1370 gate = policy.gate
1371 if gate is None:
1372 return []
1373 if gate.distinct_from is not None and gate.distinct_from != IMPLEMENTER:
1374 return [
1375 f"{source}.gate: distinct_from {gate.distinct_from!r} is not a seat; the only "
1376 f"supported value is {IMPLEMENTER!r}"
1377 ]
1378 if gate.distinct_from != IMPLEMENTER:
1379 return []
1380 implementers = {
1381 seat.name: f"{source}.{path}"
1382 for path, seat in _seat_paths(policy)
1383 if path.startswith("implement") or path.endswith(".implement")
1384 }
1385 # A role the policy does not name still resolves through the deprecated knob, so a
1386 # gate matching one of those seats is the same defect wearing an older spelling.
1387 for role, seat in sorted((legacy or {}).items()):
1388 implementers.setdefault(seat.name, f"knobs.implementer_agents.{role}")
1389 clash = implementers.get(gate.name)
1390 if clash is None:
1391 return []
1392 return [
1393 f"{source}.gate: provider {gate.provider!r} is also the implementer at "
1394 f"{clash}, and gate.distinct_from is {IMPLEMENTER!r} — a gate review "
1395 "from the vendor that wrote the change is not a second opinion"
1396 ]
1399def _review_issues(policy: TeamPolicy, *, source: str) -> list[str]:
1400 """A tier's review policy is reviewer seats or the ``jury`` literal, nothing else."""
1401 errors = []
1402 entries: list[tuple[str, tuple[Seat, ...] | str]] = [
1403 (f"review.by_tier.{tier}", value) for tier, value in sorted(policy.review_by_tier.items())
1404 ]
1405 if policy.review is not None:
1406 entries.append(("review.default", policy.review)) # seats or the jury literal
1407 for key, benches in (("by_difficulty", policy.by_difficulty), ("profiles", policy.profiles)):
1408 entries.extend(
1409 (f"{key}.{name}.review", bench.review)
1410 for name, bench in sorted(benches.items())
1411 if bench.review is not None
1412 )
1413 for path, value in entries:
1414 if isinstance(value, str) and value != JURY_PANEL:
1415 errors.append(
1416 f"{source}.{path}: {value!r} is neither a list of reviewer seats nor "
1417 f"{JURY_PANEL!r} (the cross-vendor panel as the review)"
1418 )
1419 elif isinstance(value, tuple) and not value:
1420 errors.append(
1421 f"{source}.{path}: an empty reviewer list would leave the change with no "
1422 f"review; use {JURY_PANEL!r} for the panel, or name at least one seat"
1423 )
1424 elif isinstance(value, tuple) and len(value) > MAX_REVIEW_SEATS:
1425 errors.append(
1426 f"{source}.{path}: {len(value)} reviewer seats, but keel dispatches at "
1427 f"most {MAX_REVIEW_SEATS} (slots A/B/C, one focus each); use "
1428 f"{JURY_PANEL!r} for a wider panel"
1429 )
1430 if policy.jury_mode is not None and policy.jury_mode not in JURY_MODES:
1431 errors.append(
1432 f"{source}.jury.mode: unknown mode {policy.jury_mode!r}; valid: {', '.join(JURY_MODES)}"
1433 )
1434 elif policy.jury_mode == "advisory":
1435 # "The panel is the review" plus "the panel does not gate" is a tier with no
1436 # enforceable review at all: there are no host reviewer slots to fall back on, and
1437 # an advisory verdict is not required evidence. Refused here rather than
1438 # discovered as a merge that sailed through the tier the project marked strictest.
1439 panels = sorted(path for path, value in entries if value == JURY_PANEL)
1440 if panels:
1441 errors.append(
1442 f"{source}.jury.mode: 'advisory' leaves {', '.join(panels)} with no "
1443 "enforceable review — that tier has no host reviewers, so an advisory "
1444 "panel requires nothing. Use 'gating' for a jury panel, or name reviewer "
1445 "seats for that tier"
1446 )
1447 if policy.jury_on_unavailable is not None and policy.jury_on_unavailable not in (
1448 JURY_ON_UNAVAILABLE
1449 ):
1450 errors.append(
1451 f"{source}.jury.on_unavailable: unknown policy "
1452 f"{policy.jury_on_unavailable!r}; valid: {', '.join(JURY_ON_UNAVAILABLE)}"
1453 )
1454 return errors