Coverage for src/keel/scaffold.py: 100%
218 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-02 20:26 +0000
1"""`keel init` — scaffold a default `.keel/project.yaml`, or build one with a wizard.
3Pure + deterministic: :func:`detect_stack` is a function of which marker files exist,
4:func:`render_config` renders YAML from explicit values, and :func:`wizard` builds those
5values through an injectable `ask` callback (so the interactive flow is unit-tested
6offline). The CLI supplies the real `input`-based `ask` and does the file I/O.
8Since #1018 the wizard also asks who is on the **team**. The questions come from
9:mod:`keel.wizard`, closed over a catalogue of the providers the probe found on this
10machine, and their answers are rendered here as the ``knobs.team`` block (#1014) — so
11a repository is scaffolded naming seats that actually exist rather than seats copied
12out of somebody else's example.
13"""
15from __future__ import annotations
17import re
18import shutil
19import tomllib
20from collections.abc import Callable
21from pathlib import Path
22from typing import Any
24from . import config as cfg
25from . import consent
26from . import wizard as wizard_core
27from . import yaml_helper as yaml
29#: marker file (checked in order) -> stack name.
30_MARKERS: tuple[tuple[str, str], ...] = (
31 ("pubspec.yaml", "flutter"),
32 ("build.gradle", "android"),
33 ("build.gradle.kts", "android"),
34 ("pom.xml", "java"),
35 ("Cargo.toml", "rust"),
36 ("go.mod", "go"),
37 ("pyproject.toml", "python"),
38 ("setup.py", "python"),
39 ("requirements.txt", "python"),
40 ("Pipfile", "python"),
41 ("package.json", "node"),
42)
44#: per-stack defaults: platform, build cmd, lint cmd (or None), tier-3 globs.
45_TEMPLATES: dict[str, dict] = {
46 "flutter": {
47 "platform": "flutter",
48 "build": "flutter test",
49 "lint": "flutter analyze",
50 "globs": ("lib/**/*.dart",),
51 },
52 "python": {
53 "platform": "python",
54 "build": "make test",
55 "lint": "ruff check .",
56 "globs": ("src/**/*.py",),
57 },
58 "node": {
59 "platform": "node",
60 "build": "npm test",
61 "lint": "npm run lint",
62 "globs": ("src/**/*.ts", "src/**/*.js"),
63 },
64 "android": {
65 "platform": "android",
66 "build": "./gradlew test",
67 "lint": "./gradlew lint",
68 "globs": ("app/src/**",),
69 },
70 "rust": {
71 "platform": "rust",
72 "build": "cargo test",
73 "lint": "cargo clippy",
74 "globs": ("src/**/*.rs",),
75 },
76 "go": {
77 "platform": "go",
78 "build": "go test ./...",
79 "lint": "golangci-lint run",
80 "globs": ("**/*.go",),
81 },
82 "java": {
83 "platform": "java",
84 "build": "mvn test",
85 "lint": "mvn checkstyle:check",
86 "globs": ("src/main/**",),
87 },
88 # No stack, so no command keel can name (#1328): `make test` only when the Makefile
89 # has that rule (see template_for), else none — the build gate is then unconfigured.
90 "generic": {"platform": "generic", "build": None, "lint": None, "globs": ()},
91}
93#: Written above ``knobs:`` when no build command was found (#1328), so the file itself
94#: says what to set.
95_UNSET_BUILD_COMMENT = (
96 "# build_gate_cmd is not set: keel found no test command for this project (no stack",
97 "# detected, and no Makefile `test` rule). Until you set it under knobs, the `build`",
98 "# gate blocks every run. For example:",
99 '# build_gate_cmd: "./scripts/test.sh"',
100)
103# A rule line names its targets before a `:` or `::`. `test := x`, `test ::= x` and
104# `test ?= x` assign a variable instead, and a line starting with a tab is a recipe
105# (#1301 review). Leading spaces are allowed: a rule inside `ifdef` may be indented.
106_MAKE_RULE = re.compile(r"^([^:#=\t][^:#=]*?)[ \t]*::?(?![:=])")
107# `pytest` or a `pytest-…` / `pytest_…` plugin (which depends on it; PEP 503 makes the
108# two spellings one name) as a requirement or a table key — never
109# `flake8-pytest-style`, whose name only contains the word.
110_PYTEST_NAME = re.compile(r"(?i)pytest(?:[-_.][\w.-]*)?")
111_PYTEST_REQUIREMENT = re.compile(r"(?i)^\s*pytest(?:[-_.][\w.-]*)?\s*(?:[\[(<>=!~;@,]|$)")
112_PYTEST_WORD = re.compile(r"(?i)(?<![\w.-])pytest(?![a-z0-9])")
115def _read_text(path: Path) -> str | None:
116 """A file's text, or None when it is absent or cannot be read — no evidence."""
117 try:
118 return path.read_text(encoding="utf-8", errors="replace") if path.is_file() else None
119 except OSError:
120 return None
123def _read_toml(path: Path) -> dict:
124 text = _read_text(path)
125 if text is None:
126 return {}
127 try:
128 return tomllib.loads(text)
129 except tomllib.TOMLDecodeError:
130 return {}
133def _makefile_has_target(root: Path, target: str) -> bool:
134 """Whether the project's Makefile has a rule for ``target``."""
135 for name in ("GNUmakefile", "makefile", "Makefile"):
136 path = root / name
137 if path.is_file():
138 text = _read_text(path) or ""
139 for line in text.splitlines():
140 rule = _MAKE_RULE.match(line)
141 if rule and target in rule.group(1).split():
142 return True
143 return False
144 return False
147def _toml_names_pytest(node: object) -> bool:
148 """Whether a parsed TOML tree names pytest: a ``[tool.pytest…]`` table, a Poetry or
149 Pipfile ``pytest = "…"`` key, or a PEP 508 requirement string anywhere in it.
151 Deliberately not bounded by table: Hatch, PDM, Rye, uv and Flit each keep
152 dependencies somewhere else, and a string that parses as a pytest requirement
153 almost always belongs to a project that runs pytest."""
154 if isinstance(node, dict):
155 return any(
156 _PYTEST_NAME.fullmatch(str(key)) or _toml_names_pytest(value)
157 for key, value in node.items()
158 )
159 if isinstance(node, list):
160 return any(_toml_names_pytest(item) for item in node)
161 return isinstance(node, str) and _PYTEST_REQUIREMENT.match(node) is not None
164def _text_names_pytest(text: str) -> bool:
165 """Whether an INI, ``setup.py`` or requirements text names pytest outside a comment."""
166 return any(_PYTEST_WORD.search(line.split("#", 1)[0]) for line in text.splitlines())
169def _uses_pytest(root: Path) -> bool:
170 """Whether the project runs pytest — configured *or* merely depended on.
172 Most pytest projects keep no pytest config at all: plain ``def test_…``
173 functions and ``pytest`` in their dependencies. ``unittest discover`` finds
174 none of those and, from Python 3.12, exits 5 ("NO TESTS RAN") — the same
175 blocked gate by another route — so a dependency on pytest counts too.
176 """
177 for name in ("pytest.ini", ".pytest.ini", "conftest.py"):
178 if (root / name).is_file():
179 return True
180 if any((root / d / "conftest.py").is_file() for d in ("tests", "test")):
181 return True
182 if any(_toml_names_pytest(_read_toml(root / n)) for n in ("pyproject.toml", "Pipfile")):
183 return True
184 texts = [root / n for n in ("setup.cfg", "tox.ini", "setup.py", "noxfile.py")]
185 patterns = ("*requirements*.txt", "*requirements*.in")
186 for pattern in (*patterns, "requirements/**/*.txt", "requirements/**/*.in"):
187 texts += sorted(root.glob(pattern))
188 return any(_text_names_pytest(_read_text(path) or "") for path in texts)
191def _uses_ruff(root: Path) -> bool:
192 """Whether the project configures ruff: its own file, or a ``[tool.ruff]`` table."""
193 if (root / "ruff.toml").is_file() or (root / ".ruff.toml").is_file():
194 return True
195 tool = _read_toml(root / "pyproject.toml").get("tool")
196 return isinstance(tool, dict) and "ruff" in tool
199def _unittest_command(root: Path, py: str) -> str:
200 """``unittest discover`` pointed where the tests are.
202 Discovery recurses only into packages, so from the root it finds nothing in a
203 ``tests/`` directory without an ``__init__.py`` — the most common layout — and
204 exits 5 (#1301 review). Such a directory is named with ``-s``; a package, or no
205 tests directory, keeps the plain command.
207 Two layouts stay out of reach of any ``unittest`` command: tests in a
208 subdirectory that is not a package (``tests/unit/`` without ``__init__.py``),
209 which discovery never enters, and a ``src/`` layout whose package is not
210 installed — the gate runs after the project's own install, as pytest's would.
211 """
212 for directory in ("tests", "test"):
213 if (root / directory).is_dir() and not (root / directory / "__init__.py").is_file():
214 return f"{py} -m unittest discover -s {directory}"
215 return f"{py} -m unittest discover"
218def _python_interpreter() -> str:
219 """``python`` where it exists, else ``python3``.
221 A venv — and Windows — has ``python``; a macOS or Debian machine outside one has
222 only ``python3``. Writing either one blindly scaffolds a gate that fails with
223 "command not found" on the other, the same broken gate #1297 is about.
224 """
225 return "python" if shutil.which("python") else "python3"
228def _python_gates(root: Path) -> tuple[str, str | None]:
229 """The build and lint commands a Python project can actually run (#1297).
231 The template wrote ``make test`` whatever the project held, so a Python project
232 with no Makefile scaffolded a gate that BLOCKs every ship with
233 ``make: *** No rule to make target `test'``; and ``ruff check .`` whether or not
234 ruff is part of the project. Build: the Makefile's ``test`` rule if there is one,
235 else pytest if the project configures or depends on it, else ``unittest``, which
236 ships with Python. Lint: ruff only when the project configures ruff; otherwise no
237 lint gate, which the operator can add.
238 """
239 py = _python_interpreter()
240 if _makefile_has_target(root, "test"):
241 build = "make test"
242 elif _uses_pytest(root):
243 build = f"{py} -m pytest"
244 else:
245 build = _unittest_command(root, py)
246 return build, ("ruff check ." if _uses_ruff(root) else None)
249def template_for(stack: str, root: str | Path | None = None) -> dict:
250 """The stack's defaults, resolved against the project at ``root`` when given."""
251 t = dict(_TEMPLATES.get(stack, _TEMPLATES["generic"]))
252 if stack == "python" and root is not None:
253 t["build"], t["lint"] = _python_gates(Path(root))
254 elif t["platform"] == "generic" and root is not None:
255 # The one command a stackless project can be shown to have (#1328).
256 if _makefile_has_target(Path(root), "test"):
257 t["build"] = "make test"
258 return t
261def missing_build_gate(text: str) -> bool:
262 """Whether a rendered config leaves ``knobs.build_gate_cmd`` unset (#1328)."""
263 knobs = yaml.load(text).get("knobs") or {}
264 return not knobs.get("build_gate_cmd")
267def detect_stack(root: str | Path) -> str:
268 """Detect the project stack from marker files (``generic`` if none match)."""
269 root = Path(root)
270 for marker, stack in _MARKERS:
271 if (root / marker).exists():
272 return stack
273 return "generic"
276def detect_base_branch(root: str | Path) -> str:
277 """Detect the repository's default base branch (defaults to ``main``)."""
278 root = Path(root)
279 head_file = root / ".git" / "HEAD"
280 if head_file.exists():
281 try:
282 content = head_file.read_text(encoding="utf-8").strip()
283 if content.startswith("ref: refs/heads/"):
284 ref_name = content[len("ref: refs/heads/") :].strip()
285 if ref_name in ("main", "master", "develop", "trunk"):
286 return ref_name
287 except OSError:
288 return "main"
289 return "main"
292def auto_detect_config(
293 root: str | Path,
294 *,
295 repo: str = "my-repo",
296 owner: str | None = None,
297) -> tuple[str, dict]:
298 """Inspect the repository stack and base branch, returning (yaml_text, metadata)."""
299 root = Path(root)
300 stack = detect_stack(root)
301 base_branch = detect_base_branch(root)
302 t = template_for(stack, root)
303 meta = {
304 "stack": stack,
305 "platform": t["platform"],
306 "base_branch": base_branch,
307 "owner": owner,
308 "build_cmd": t["build"],
309 "lint_cmd": t["lint"],
310 "tier3_globs": t["globs"],
311 }
312 text = render_config(
313 repo=repo,
314 owner=owner,
315 base_branch=base_branch,
316 platform=t["platform"],
317 build_cmd=t["build"],
318 lint_cmd=t["lint"],
319 tier3_globs=t["globs"],
320 generator="keel init --auto",
321 )
322 return text, meta
325def render_config(
326 *,
327 repo: str = "my-repo",
328 owner: str | None = None,
329 base_branch: str = "main",
330 platform: str = "generic",
331 build_cmd: str | None = "make test",
332 lint_cmd: str | None = None,
333 tier3_globs: tuple[str, ...] = (),
334 timezone: str | None = None,
335 merge_window: str | None = None,
336 consent_mode: str = "explicit",
337 team: dict[str, Any] | None = None,
338 generator: str = "keel init",
339) -> str:
340 """Render a valid ``project.yaml`` from explicit values (passes ``keel validate``).
342 ``team`` is a ``knobs.team`` block (:meth:`keel.wizard.Resolution.team_block`).
343 ``None`` — the default — writes no block at all, which is *not* the same as writing
344 an empty one: an absent ``team`` leaves ``config_hash`` exactly where it was for
345 every project that never opted in (:func:`keel.team.canonical`).
346 """
347 if consent_mode not in consent.CONSENT_MODES:
348 raise ValueError(
349 f"unknown consent mode {consent_mode!r}; valid: {', '.join(consent.CONSENT_MODES)}"
350 )
351 if bool(timezone) != bool(merge_window):
352 raise ValueError(
353 "timezone and merge_window are all-or-nothing: pass both to configure a merge "
354 "window or neither to configure none — a config with one of them is a config "
355 "`keel validate` refuses (#1076)"
356 )
357 generator_comment = " ".join(str(generator).splitlines())
358 lines = [
359 f"# keel consumer config (generated by `{generator_comment}`)",
360 "extends: keel",
361 'core_version: "^1.0"',
362 f"repo: {_yaml_scalar(repo)}",
363 # `owner` completes the `owner/repo` a live evidence/merge run needs; omitted (not
364 # written blank) when no git remote named one, so `keel validate` still passes and the
365 # operator fills it in (#1247).
366 *([f"owner: {_yaml_scalar(owner)}"] if owner else []),
367 f"base_branch: {_yaml_scalar(base_branch)}",
368 f"platform: {_yaml_scalar(platform)}",
369 f"consent_mode: {_yaml_scalar(consent_mode)}",
370 ]
371 if timezone:
372 lines.append(f"timezone: {_yaml_scalar(timezone)}")
373 if merge_window:
374 lines.append(f"merge_window: {_yaml_scalar(merge_window)}")
375 knobs = [f" build_gate_cmd: {_yaml_scalar(build_cmd)}"] if build_cmd else []
376 if team:
377 knobs.append(" team:")
378 knobs.extend(_render_mapping(team, 2))
379 if lint_cmd:
380 knobs.append(f" lint_cmd: {_yaml_scalar(lint_cmd)}")
381 if tier3_globs:
382 knobs.append(" tier3_globs:")
383 knobs.extend(_render_sequence(list(tier3_globs), 2))
384 lines.append("")
385 if not build_cmd:
386 # No command, but `build` stays in `gates:` below: an unconfigured gate is planned
387 # and blocks with a finding naming the knob, where an absent one would pass (#1328).
388 lines.extend(_UNSET_BUILD_COMMENT)
389 lines += ["knobs:", *knobs] if knobs else ["knobs: {}"]
390 gates = "[build, lint]" if lint_cmd else "[build]"
391 lines += ["", f"gates: {gates}", "extensions: {}", "extensions_dir: .keel/extensions", ""]
392 return "\n".join(lines)
395def _yaml_scalar(value: str) -> str:
396 """Render a scalar as inline YAML so scaffolded values cannot inject new keys."""
397 return yaml.dump(
398 str(value),
399 default_style='"',
400 default_flow_style=True,
401 width=10**6,
402 sort_keys=False,
403 ).strip()
406def _yaml_key(key: str) -> str:
407 """A mapping key, quoted unless it is a plain identifier.
409 The tier keys of ``review.by_tier`` are the reason: ``"1"`` has to stay quoted
410 because YAML reads a bare ``1:`` as an *integer* key, which the schema cannot
411 describe and :func:`keel.team._tier_key_issues` rejects by name.
412 """
413 return key if key.isidentifier() else _yaml_scalar(key)
416def _yaml_value(value: Any) -> str:
417 return str(value) if isinstance(value, int) else _yaml_scalar(value)
420def _render_mapping(data: dict[str, Any], indent: int) -> list[str]:
421 """Block-style YAML for a nested mapping of scalars, mappings and lists."""
422 pad = " " * indent
423 lines: list[str] = []
424 for key, value in data.items():
425 if isinstance(value, dict):
426 lines.append(f"{pad}{_yaml_key(key)}:")
427 lines.extend(_render_mapping(value, indent + 1))
428 elif isinstance(value, list):
429 lines.append(f"{pad}{_yaml_key(key)}:")
430 lines.extend(_render_sequence(value, indent + 1))
431 else:
432 lines.append(f"{pad}{_yaml_key(key)}: {_yaml_value(value)}")
433 return lines
436def _render_sequence(items: list[Any], indent: int) -> list[str]:
437 pad = " " * indent
438 lines: list[str] = []
439 for item in items:
440 if isinstance(item, dict):
441 rendered = _render_mapping(item, indent + 1)
442 lines.append(f"{pad}- {rendered[0].strip()}")
443 lines.extend(rendered[1:])
444 else:
445 lines.append(f"{pad}- {_yaml_value(item)}")
446 return lines
449def default_config(
450 stack: str,
451 *,
452 repo: str = "my-repo",
453 owner: str | None = None,
454 base_branch: str = "main",
455 root: str | Path | None = None,
456) -> str:
457 """Render the default ``project.yaml`` for ``stack`` (non-interactive)."""
458 t = template_for(stack, root)
459 return render_config(
460 repo=repo,
461 owner=owner,
462 base_branch=base_branch,
463 platform=t["platform"],
464 build_cmd=t["build"],
465 lint_cmd=t["lint"],
466 tier3_globs=t["globs"],
467 )
470def _ignore(_message: str) -> None:
471 """Default ``notify``: a caller that did not ask for feedback gets none."""
474def wizard(
475 stack: str,
476 ask: Callable[[str, str], str],
477 *,
478 repo: str = "my-repo",
479 owner: str | None = None,
480 base_default: str = "main",
481 catalog: wizard_core.Catalog | None = None,
482 notify: Callable[[str], None] | None = None,
483 root: str | Path | None = None,
484) -> str:
485 """Build a config by asking for each value, defaulting to the stack template.
487 ``ask(prompt, default)`` returns the chosen value (an empty answer ⇒ the default).
488 Pure given ``ask`` — the CLI passes a real `input`-based implementation.
490 The ``timezone`` + ``merge_window`` pair is one question, not two — see
491 :func:`merge_window_answers` — so no answer can produce the half-configured pair
492 ``parse_config`` refuses (#1082).
494 ``catalog`` is the providers the probe found usable on this machine (#1018). Given
495 one, the wizard adds the **team step**: who implements, who gives the mandatory
496 gate review, who reviews at each risk tier, and how the jury gates — every option
497 drawn from the catalogue, so the scaffolded ``knobs.team`` cannot name a provider
498 this machine has never had. Without one (or with an empty one — a machine where
499 nothing is installed yet) the step is skipped and no ``team`` block is written.
500 """
501 t = template_for(stack, root)
502 report = _ignore if notify is None else notify
503 base = ask("Base branch", base_default)
504 tz, win = merge_window_answers(ask, report)
505 mode = ask("Consent mode (explicit, standing, agent)", "explicit") or "explicit"
506 build_prompt = "Build/test command"
507 if not t["build"]:
508 build_prompt += " (blank leaves it unset: the build gate then blocks until you set it)"
509 build = ask(build_prompt, t["build"] or "")
510 lint = ask("Lint command (blank to skip)", t["lint"] or "")
511 return render_config(
512 repo=repo,
513 owner=owner,
514 base_branch=base,
515 platform=t["platform"],
516 build_cmd=build or None,
517 lint_cmd=lint or None,
518 tier3_globs=t["globs"],
519 timezone=tz,
520 merge_window=win,
521 consent_mode=mode,
522 team=team_block(ask, catalog, notify=report),
523 generator="keel init --wizard",
524 )
527def merge_window_answers(
528 ask: Callable[[str, str], str],
529 notify: Callable[[str], None],
530) -> tuple[str | None, str | None]:
531 """Ask for the ``timezone`` + ``merge_window`` pair as **one** decision (#1082).
533 The two used to be independent prompts, each advertising "blank to skip", which the
534 all-or-nothing rule #1076 made unanswerable: answer one, skip the other, and the
535 wizard wrote a ``project.yaml`` the very next ``keel validate`` rejected. So there
536 is a single yes/no gate now, and on *yes* neither half is skippable.
538 Each answer is checked with the function ``parse_config`` will check it with
539 (:func:`keel.config.timezone_issue` / :func:`keel.config.merge_window_issue`) and a
540 value that will not evaluate is reported and asked for again, at most
541 :data:`keel.wizard.MAX_ATTEMPTS` times — the same bounded re-ask as the team step,
542 because a wizard that argues forever is a hang. Giving up drops the *pair*, never
543 one half of it, so every path out of here is a pair the config layer accepts:
544 ``(zone, window)`` or ``(None, None)``.
545 """
546 if not _asked_yes(ask, "Configure a merge window (timezone + hours)? (y/n)"):
547 return None, None
548 tz = _ask_evaluable(ask, notify, "Timezone (IANA)", "Europe/Istanbul", cfg.timezone_issue)
549 win = (
550 None
551 if tz is None
552 else _ask_evaluable(
553 ask, notify, "Merge window HH:MM-HH:MM", "07:00-01:30", cfg.merge_window_issue
554 )
555 )
556 if tz is None or win is None:
557 notify(
558 "no merge window configured: 'timezone' and 'merge_window' are all-or-nothing, "
559 "so neither was written — re-run the wizard, or add both by hand"
560 )
561 return None, None
562 return tz, win
565def _asked_yes(ask: Callable[[str, str], str], prompt: str, default: str = "y") -> bool:
566 """True if the operator answered yes; a blank answer keeps ``default``.
568 Yes is ``y``/``yes`` in any case, anything else is no. The default is *yes*: the
569 old prompts defaulted to a configured window, and pressing Enter through the wizard
570 must keep scaffolding the night no-merge window it always did.
571 """
572 return ((ask(prompt, default) or "").strip() or default).lower() in ("y", "yes")
575def _ask_evaluable(
576 ask: Callable[[str, str], str],
577 notify: Callable[[str], None],
578 prompt: str,
579 default: str,
580 issue: Callable[[str], str | None],
581) -> str | None:
582 """Ask until ``issue`` finds nothing wrong with the answer (``None`` == gave up)."""
583 for _ in range(wizard_core.MAX_ATTEMPTS):
584 value = (ask(prompt, default) or "").strip() or default
585 problem = issue(value)
586 if problem is None:
587 return value
588 notify(problem)
589 return None
592def team_block(
593 ask: Callable[[str, str], str],
594 catalog: wizard_core.Catalog | None,
595 *,
596 notify: Callable[[str], None] | None = None,
597) -> dict[str, Any] | None:
598 """Ask the team questions and return the ``knobs.team`` block (``None`` to skip)."""
599 # Narrowed before the emptiness check, not after: a machine whose only providers are
600 # registry entries has nothing a committed policy could name, and a wizard that asked
601 # anyway would write a `team` block `keel validate` then refuses.
602 catalog = None if catalog is None else wizard_core.committable(catalog)
603 if catalog is None or not catalog.candidates:
604 return None
605 state = wizard_core.start(catalog, scope=wizard_core.SCOPE_CONFIG)
606 answered = wizard_core.run(state, ask, notify if notify is not None else _ignore)
607 return answered.resolve().team_block()