Fix the known issues: validate checks generated code, Click backend parity, reproducible output, no-extras support

.github/workflows/python-app.yaml

@@ -67,3 +67,21 @@ jobs:
         uses: codecov/codecov-action@v7
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+
+  # cliffy has to work, lint and pass its tests without the optional `rich` and `ai` extras
+  no-extras:
+    name: Without extras (Python 3.11)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Install uv
+        uses: astral-sh/setup-uv@v10.3.0
+        with:
+          python-version: "3.11"
+          enable-cache: true
+      - name: Install library without extras
+        run: uv sync --locked
+      - name: Code quality
+        run: make lint
+      - name: Test with pytest
+        run: make test

Makefile

@@ -1,5 +1,4 @@
 SOURCE_FILES=cliffy tests
-GENERATE_DEPS=requests "six<1.0.0" keyring
 
 check: format lint test clean
 
@@ -50,24 +49,17 @@ docs-check:
 docs-build: docs-check
 	uv run --group docs mkdocs build --strict
 
-# The example manifests declare requirements that are not cliffy dependencies. They have to be
-# installed into the project environment (not a `uv run --with` overlay), because cliffy validates
-# `requires` against `pip freeze`. `uv sync` in generate-cleanup removes them again.
+# examples/generated/*.py are what cliffy generates for the manifests in examples/. Nothing is installed or
+# loaded for this: the manifests are only rendered. tests/test_examples.py fails when a file is stale.
 generate-clis:
-	uv pip install ${GENERATE_DEPS}
-	uv run --no-sync cli load examples/*.yaml
-	cp cliffy/clis/*.py examples/generated/
+	uv run python scripts/examples.py
 
-generate-all: generate-clis generate-schema generate-cleanup
+generate-all: generate-clis generate-schema
 	@echo "~ done"
 
-generate-cleanup:
-	uv run --no-sync cli rm-all
-	uv sync --all-extras --group docs
-
 generate-schema:
 	uv run python -m cliffy.manifest --json-schema > examples/cliffy_schema.json
 
 .PHONY: check install sync test test-cov clean package publish format lint \
 	docs-serve docs-generate docs-check docs-build \
-	generate-clis generate-all generate-cleanup generate-schema
+	generate-clis generate-all generate-schema

README.md

@@ -81,7 +81,7 @@ HELLO, ADA!
 | `cli test <manifest>` | Run the tests defined in the manifest |
 | `cli build <manifest>` | Build one file to ship: a zipapp, a script or a native executable |
 | `cli render <manifest>` | Print the generated Python code |
-| `cli validate <manifest>` | Check a manifest against the schema |
+| `cli validate <manifest>` | Check a manifest and the code generated from it |
 | `cli docs <manifest>` | Generate documentation for the CLI |
 | `cli list`, `info`, `update`, `remove` | Manage loaded CLIs |
 | `cli ai generate`, `cli ai ask` | Draft manifests with an LLM (needs the `ai` extra) |
@@ -93,7 +93,7 @@ Every command and option is described in the
 
 ```console
 $ pip install cliffy               # core
-$ pip install "cliffy[rich]"       # rich-click help for the cli command, needed for the Click backend
+$ pip install "cliffy[rich]"       # rich-click help for the cli command and for Click-backend CLIs
 $ pip install "cliffy[ai]"         # AI commands, built on pydantic-ai
 $ uvx cliffy run hello.yaml greet Ada   # run without installing
 ```

cliffy/ai.py

@@ -278,14 +278,17 @@ def generate_manifest(
     )
     rejections: list[ManifestOutputError] = []
 
-    @agent.output_validator
     def validate_output(output: str) -> str:
         try:
             return validate_manifest_yaml(output, cli_name)
         except ManifestOutputError as e:
             rejections.append(e)
             raise ModelRetry(str(e)) from e
 
+    # registered with a call, not as a decorator: without the `ai` extra the type checker cannot see
+    # pydantic-ai, and an untyped decorator would make this function untyped
+    agent.output_validator(validate_output)
+
     try:
         result = agent.run_sync(description)
     except UnexpectedModelBehavior as e:

cliffy/blueprint.py

@@ -130,6 +130,12 @@ def normalize_reference(reference: str, base_dir: str | os.PathLike[str] | None
     return f"{resolved}:{attr}" if attr else resolved
 
 
+def blueprint_file(reference: str, base_dir: str | os.PathLike[str] | None = None) -> str | None:
+    """Absolute path of the Python file a reference points to, or None when it names a module or a plugin."""
+    target, _ = _split_reference(normalize_reference(reference, base_dir))
+    return target if _is_file_reference(target) else None
+
+
 def resolve_blueprint(reference: str, base_dir: str | os.PathLike[str] | None = None) -> type[Blueprint]:
     """Resolve a blueprint reference to a ``Blueprint`` subclass.
 

cliffy/builder.py

@@ -34,7 +34,12 @@
 MIN_PYTHON = ">=3.11"
 # Package that provides the framework a generated CLI imports
 FRAMEWORK_PACKAGES = {"typer": "typer", "rich_click": "rich-click", "click": "click"}
-GENERATED_HEADER = re.compile(r"\A## Generated (\S+) on .*\n")
+# An import of a framework at the start of a line, at any indentation: generated Click code imports
+# rich_click in a `try` block and falls back to click in the `except` block.
+FRAMEWORK_IMPORT = re.compile(rf"^[ \t]*(?:import|from)[ \t]+({'|'.join(FRAMEWORK_PACKAGES)})\b", re.MULTILINE)
+# Header of code generated by cliffy 0.6 and older, which holds the time of generation. A CLI loaded with
+# such a version is still built from its script. Current headers name the cliffy version and are kept.
+LEGACY_GENERATED_HEADER = re.compile(r"\A## Generated (\S+) on .*\n")
 # Release of PyApp (https://github.com/ofek/pyapp), the launcher a native build compiles
 PYAPP_VERSION = "0.30.0"
 INSTALL_RUST = (
@@ -364,12 +369,12 @@ def build_cli(
 
 
 def get_standalone_code(code: str) -> str:
-    """Generated code as it is shipped: no timestamp, and no import of pybash.
+    """Generated code as it is shipped: no timestamp in the header, and no import of pybash.
 
     A CLI with shell operators calls a helper of pybash. Its source is pasted in where the import
     was, so that a built CLI depends on its framework and never on pybash or cliffy.
     """
-    code = GENERATED_HEADER.sub(r"## Generated \1 with cliffy\n", code)
+    code = LEGACY_GENERATED_HEADER.sub(r"## Generated \1 with cliffy\n", code)
     return code.replace(SHELL_RUNTIME_IMPORT, f"\n\n{pybash.runtime_source()}\n\n", 1)
 
 
@@ -472,9 +477,16 @@ def get_build_requirements(code: str, deps: list[str]) -> list[str]:
         with contextlib.suppress(InvalidRequirement):
             named.add(canonicalize_name(Requirement(dep).name))
 
+    imported = set(FRAMEWORK_IMPORT.findall(code))
+    if imported & {"typer", "rich_click"}:
+        # Typer and rich-click depend on Click and bring a version they work with. Click is only a
+        # requirement of its own for a CLI that imports nothing else. This is also the generated Click
+        # code, which prefers rich_click and falls back to click: the build ships rich-click.
+        imported.discard("click")
+
     requirements = list(deps)
     for module, package in FRAMEWORK_PACKAGES.items():
-        if package in named or not re.search(rf"^\s*(import|from) {module}\b", code, flags=re.MULTILINE):
+        if package in named or module not in imported:
             continue
         try:
             requirements.append(f"{package}=={metadata.version(package)}")

cliffy/checker.py

@@ -0,0 +1,75 @@
+"""Checks the code generated for a manifest and reports problems in the manifest's terms."""
+
+from __future__ import annotations
+
+import re
+import traceback
+
+import typer
+
+from cliffy.commander import CLI
+
+GENERATED_FILE = "<generated CLI>"
+FUNCTION_DEFINITION = re.compile(r"(?:async\s+)?def (\w+)\(")
+MODULE_LEVEL = "module level code (`imports`, `vars`, `types` or `functions`)"
+
+
+class GeneratedCodeError(Exception):
+    """The code generated for a manifest does not compile or does not import.
+
+    The message names the part of the manifest the failing code was generated from.
+    """
+
+
+def locate(cli: CLI, line_number: int) -> str:
+    """Part of the manifest that line `line_number` (1-based) of the generated code was generated from."""
+    lines = cli.code.splitlines()
+    index = min(max(line_number, 1), len(lines)) - 1
+
+    # a decorator belongs to the function below it
+    while index < len(lines) - 1 and lines[index].startswith("@"):
+        index += 1
+
+    for line in reversed(lines[: index + 1]):
+        if match := FUNCTION_DEFINITION.match(line):
+            return cli.functions.get(match.group(1), MODULE_LEVEL)
+    return MODULE_LEVEL
+
+
+def describe(cli: CLI, error: BaseException, line_number: int | None) -> str:
+    where = locate(cli, line_number) if line_number else MODULE_LEVEL
+    message = f"{where}: {type(error).__name__}: {error.msg if isinstance(error, SyntaxError) else error}"
+    lines = cli.code.splitlines()
+    if line_number and 0 < line_number <= len(lines):
+        message += f"\n    {lines[line_number - 1].strip()}"
+    return message
+
+
+def check_generated_code(cli: CLI) -> None:
+    """Compiles and imports the generated code, and builds the command tree of a Typer CLI.
+
+    Importing defines the commands, it does not run them.
+
+    Raises:
+        GeneratedCodeError: the code is not valid Python, or fails while it is imported.
+    """
+    try:
+        # dont_inherit: this module's own `from __future__ import annotations` must not apply to the generated code
+        compiled = compile(cli.code, GENERATED_FILE, "exec", dont_inherit=True)
+    except SyntaxError as e:
+        raise GeneratedCodeError(describe(cli, e, e.lineno)) from e
+
+    try:
+        namespace: dict[str, object] = {"__name__": f"cliffy_check_{cli.name.replace('-', '_')}"}
+        exec(compiled, namespace)
+        app = namespace.get("cli")
+        if isinstance(app, typer.Typer):
+            # Typer reads the signature of a command when the CLI starts. From Python 3.14 on annotations
+            # are evaluated lazily, so a param with an unknown type only fails here
+            typer.main.get_command(app)
+    except (Exception, SystemExit) as e:
+        generated_frames = [
+            frame for frame in traceback.extract_tb(e.__traceback__) if frame.filename == GENERATED_FILE
+        ]
+        line_number = generated_frames[-1].lineno if generated_frames else None
+        raise GeneratedCodeError(describe(cli, e, line_number)) from e

cliffy/cli.py

@@ -17,11 +17,13 @@
     format_size,
     run_cli,
 )
+from cliffy.checker import GeneratedCodeError, check_generated_code
 from cliffy.doc import DocGenerator
 from cliffy.helper import (
     CLI_NAME_PATTERN,
     ManifestOrCLI,
     age_datetime,
+    cliffy_version,
     exit_err,
     indent_block,
     out,
@@ -67,7 +69,8 @@ def show_aliases_callback(ctx: Any, param: Any, val: bool) -> None:
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)
-@click.version_option()
+# the version is passed in: with rich-click installed, the automatic lookup finds rich-click's own version
+@click.version_option(cliffy_version(), package_name="cliffy")
 @click.option("--aliases", type=bool, is_flag=True, is_eager=True, callback=show_aliases_callback)
 def cli(aliases: bool) -> None:
     pass
@@ -340,14 +343,18 @@ def test(manifest: str, exitfirst: bool, blueprint: str | None = None) -> None:
     out("✨ Invoking tests ✨", nl=False)
     total = tester.total_cases
     passed = 0
-    for i, case in enumerate(tester.test_pipeline):
+    case_number = 0
+    for case in tester.test_pipeline:
         if isinstance(case, ShellScript):
+            # a setup or cleanup step: it is run, but it is not one of the counted test cases
             try:
                 tester.invoke_shell(case)
             except SyntaxError as e:
                 exit_err(f"~ error in {manifest}: test step `{case.command}`, column {e.offset}: {e.msg}")
             continue
 
+        case_number += 1
+
         out(f"\n\n🪄 > {tester.T.cli.name} {case.command}\n")
         try:
             test = tester.invoke_test(case.command, case.assert_script)
@@ -356,7 +363,7 @@ def test(manifest: str, exitfirst: bool, blueprint: str | None = None) -> None:
             if result.exception:
                 out(str(result))
             next(test, "")
-            out(f"\n✅ {i + 1} of {total}")
+            out(f"\n✅ {case_number} of {total}")
             passed += 1
         except AssertionError:
             _, _, tb = sys.exc_info()
@@ -390,12 +397,15 @@ def test(manifest: str, exitfirst: bool, blueprint: str | None = None) -> None:
 @click.argument("manifest", type=click.File("rb"), required=True)
 @blueprint_option
 def validate(manifest: TextIO, blueprint: str | None = None) -> None:
-    """Validate the syntax and structure of a CLI manifest"""
+    """Validate a CLI manifest: its structure, its requirements and the code generated from it"""
     try:
-        Transformer(manifest, blueprint=blueprint)
-        out(f"Manifest {manifest.name} is valid", fg="green")
+        T = Transformer(manifest, blueprint=blueprint)
+        check_generated_code(T.cli)
+    except GeneratedCodeError as e:
+        exit_err(f"Manifest {manifest.name} is invalid: the generated code does not work\n  {e}")
     except Exception as e:
         exit_err(f"Manifest {manifest.name} is invalid: {e}")
+    out(f"Manifest {manifest.name} is valid", fg="green")
 
 
 @click.argument("cli_or_manifest", type=ManifestOrCLI())